{
  "id": "addr",
  "title": "Own work email address",
  "question": "Which of these addresses is this person's own work email address?",
  "plain_question": "Which of these addresses is the person's own work email?",
  "matters": "A wrong pick sends your sequence to an address that bounces or reaches someone else.",
  "n": 200,
  "primary_metric": {
    "name": "Correct",
    "definition": "Share of the 200 records where the model's first pick, among four addresses at the person's work domain, was the verified work address. A failed call counts as wrong."
  },
  "guard_definition": "Share of records the model can settle by itself with at most 1 mistake in 100, at the confidence cutoff that works best for it. A record counts when the model gives a top pick. General LLMs state no confidence here, so each settles all of its picks or none.",
  "headline": "Gemini 4 Argon gets 77% of address picks right. Gemini 3.8 Flash and Parallel are too close to call, and GPT-6.1 Terra and mammoth-1 are clearly behind.",
  "leader": "google/gemini-4-argon",
  "rows": [
    {
      "model": "google/gemini-4-argon",
      "provider": "google",
      "kind": "LLM",
      "metric": 0.77,
      "ci_low": 0.71,
      "ci_high": 0.82,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.07,
      "usd_per_1k_cases": 1.187,
      "median_latency_s": 5.65,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Leader."
    },
    {
      "model": "google/gemini-3.8-flash",
      "provider": "google",
      "kind": "LLM",
      "metric": 0.76,
      "ci_low": 0.7,
      "ci_high": 0.81,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.1,
      "usd_per_1k_cases": 1.201,
      "median_latency_s": 4.2,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "parallel/fast",
      "provider": "parallel",
      "kind": "web agent",
      "metric": 0.64,
      "ci_low": 0.57,
      "ci_high": 0.7,
      "guard": {
        "share": 0.04,
        "threshold": 1,
        "passed": 7,
        "wrong": 0
      },
      "ece": 0.12,
      "stability": 0.02,
      "usd_per_1k_cases": 1,
      "median_latency_s": 0.95,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 0.0558)."
    },
    {
      "model": "openai/gpt-6.1-terra",
      "provider": "openai",
      "kind": "LLM",
      "metric": 0.64,
      "ci_low": 0.57,
      "ci_high": 0.7,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.18,
      "usd_per_1k_cases": 0.3367,
      "median_latency_s": 1.49,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "Clearly behind the leader (p Holm 0.0099)."
    },
    {
      "model": "baseten/mammoth-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.59,
      "ci_low": 0.53,
      "ci_high": 0.66,
      "guard": {
        "share": 0.01,
        "threshold": 0.92,
        "passed": 3,
        "wrong": 0
      },
      "ece": 0.09,
      "stability": 0,
      "usd_per_1k_cases": 0.0197,
      "median_latency_s": 0.62,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "Clearly behind the leader (p Holm 0.0004)."
    },
    {
      "model": "cloudflare/clef-omni",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.56,
      "ci_low": 0.49,
      "ci_high": 0.63,
      "guard": {
        "share": 0.01,
        "threshold": 0.94,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.16,
      "stability": 0,
      "usd_per_1k_cases": 0.0357,
      "median_latency_s": 0.71,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "liquid/d1",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.55,
      "ci_low": 0.48,
      "ci_high": 0.62,
      "guard": {
        "share": 0.01,
        "threshold": 0.99,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.17,
      "stability": 0.17,
      "usd_per_1k_cases": 0.003,
      "median_latency_s": 0.59,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "z-ai/glm-5.3-flash",
      "provider": "z-ai",
      "kind": "LLM",
      "metric": 0.55,
      "ci_low": 0.48,
      "ci_high": 0.62,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.32,
      "usd_per_1k_cases": 0.024,
      "median_latency_s": 1.05,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "liquid/d1.1",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.54,
      "ci_low": 0.47,
      "ci_high": 0.61,
      "guard": {
        "share": 0.01,
        "threshold": 0.98,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.14,
      "stability": 0.2,
      "usd_per_1k_cases": 0.003,
      "median_latency_s": 0.59,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "nace-ai/drex-max",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.54,
      "ci_low": 0.47,
      "ci_high": 0.6,
      "guard": {
        "share": 0.01,
        "threshold": 1,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.22,
      "stability": 0.04,
      "usd_per_1k_cases": 0.0113,
      "median_latency_s": 0.76,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "deepseek/deepseek-v4.1-flash-1017",
      "provider": "deepseek",
      "kind": "LLM",
      "metric": 0.54,
      "ci_low": 0.47,
      "ci_high": 0.6,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.1,
      "usd_per_1k_cases": 0.0163,
      "median_latency_s": 0.87,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "typesafe/jev-1.13",
      "provider": "typesafe",
      "kind": "decision model",
      "metric": 0.54,
      "ci_low": 0.47,
      "ci_high": 0.6,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.16,
      "stability": 0.04,
      "usd_per_1k_cases": 0.0183,
      "median_latency_s": 0.57,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "typesafe/jev-1.14",
      "provider": "typesafe",
      "kind": "decision model",
      "metric": 0.53,
      "ci_low": 0.46,
      "ci_high": 0.6,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0.06,
      "usd_per_1k_cases": 0.0183,
      "median_latency_s": 0.57,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "liquid/d1-ultra",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.53,
      "ci_low": 0.46,
      "ci_high": 0.59,
      "guard": {
        "share": 0.01,
        "threshold": 0.95,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.17,
      "stability": 0.14,
      "usd_per_1k_cases": 0.006,
      "median_latency_s": 0.66,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "upstage/solar-decide",
      "provider": "upstage",
      "kind": "decision model",
      "metric": 0.53,
      "ci_low": 0.46,
      "ci_high": 0.59,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.42,
      "stability": 0.19,
      "usd_per_1k_cases": 0.0217,
      "median_latency_s": 9.73,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "perplexity/pplx-decider-v1.2",
      "provider": "perplexity",
      "kind": "decision model",
      "metric": 0.52,
      "ci_low": 0.45,
      "ci_high": 0.58,
      "guard": {
        "share": 0.01,
        "threshold": 0.91,
        "passed": 3,
        "wrong": 0
      },
      "ece": 0.08,
      "stability": 0.03,
      "usd_per_1k_cases": 0.0037,
      "median_latency_s": 0.69,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "scaledown/classify-1",
      "provider": "scaledown",
      "kind": "decision model",
      "metric": 0.51,
      "ci_low": 0.44,
      "ci_high": 0.58,
      "guard": {
        "share": 0.01,
        "threshold": 0.94,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.11,
      "stability": 0.01,
      "usd_per_1k_cases": 0.015,
      "median_latency_s": 0.73,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "the most common local-part pattern in our find key",
      "provider": null,
      "kind": "baseline",
      "metric": 0.51,
      "ci_low": 0.44,
      "ci_high": 0.57,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": null,
      "usd_per_1k_cases": 0,
      "median_latency_s": null,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "Free comparison rule with no model. Not tested against the leader."
    },
    {
      "model": "microsoft/microsoft-decision-1",
      "provider": "microsoft",
      "kind": "decision model",
      "metric": 0.5,
      "ci_low": 0.43,
      "ci_high": 0.57,
      "guard": {
        "share": 0.02,
        "threshold": 0.94,
        "passed": 4,
        "wrong": 0
      },
      "ece": 0.11,
      "stability": 0.03,
      "usd_per_1k_cases": 0.005,
      "median_latency_s": 0.78,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "openai/gpt-6-luna",
      "provider": "openai",
      "kind": "LLM",
      "metric": 0.5,
      "ci_low": 0.43,
      "ci_high": 0.57,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.23,
      "usd_per_1k_cases": 0.0337,
      "median_latency_s": 1.43,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "nace-ai/drex-v2.0",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.49,
      "ci_low": 0.42,
      "ci_high": 0.56,
      "guard": {
        "share": 0.03,
        "threshold": 0.92,
        "passed": 6,
        "wrong": 0
      },
      "ece": 0.12,
      "stability": 0,
      "usd_per_1k_cases": 0.007,
      "median_latency_s": 0.8,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "cloudflare/clef",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.49,
      "ci_low": 0.42,
      "ci_high": 0.56,
      "guard": {
        "share": 0.01,
        "threshold": 0.92,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0.12,
      "usd_per_1k_cases": 0.0527,
      "median_latency_s": 0.87,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "anthropic/claude-haiku-5.5",
      "provider": "anthropic",
      "kind": "LLM",
      "metric": 0.49,
      "ci_low": 0.42,
      "ci_high": 0.56,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.09,
      "usd_per_1k_cases": 0.0623,
      "median_latency_s": 0.97,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "morph/reflex",
      "provider": "morph",
      "kind": "decision model",
      "metric": 0.48,
      "ci_low": 0.41,
      "ci_high": 0.55,
      "guard": {
        "share": 0.03,
        "threshold": 0.95,
        "passed": 6,
        "wrong": 0
      },
      "ece": 0.22,
      "stability": 0.1,
      "usd_per_1k_cases": 0.0083,
      "median_latency_s": 0.78,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "baseten/joey-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.48,
      "ci_low": 0.41,
      "ci_high": 0.55,
      "guard": {
        "share": 0.01,
        "threshold": 0.92,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0,
      "usd_per_1k_cases": 0.002,
      "median_latency_s": 0.6,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "cloudflare/clef-flash",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.48,
      "ci_low": 0.41,
      "ci_high": 0.55,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.21,
      "stability": 0.07,
      "usd_per_1k_cases": 0.0063,
      "median_latency_s": 0.75,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "nace-ai/drex-v1.5",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.47,
      "ci_low": 0.4,
      "ci_high": 0.53,
      "guard": {
        "share": 0.01,
        "threshold": 1,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.37,
      "stability": 0,
      "usd_per_1k_cases": 0.0047,
      "median_latency_s": 0.74,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "minimax/minimax-m3.1-flash",
      "provider": "minimax",
      "kind": "LLM",
      "metric": 0.46,
      "ci_low": 0.39,
      "ci_high": 0.53,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.04,
      "usd_per_1k_cases": 0.0297,
      "median_latency_s": 0.84,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "perplexity/pplx-decider-flash-v1.2",
      "provider": "perplexity",
      "kind": "decision model",
      "metric": 0.45,
      "ci_low": 0.38,
      "ci_high": 0.51,
      "guard": {
        "share": 0.01,
        "threshold": 0.98,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.24,
      "stability": 0,
      "usd_per_1k_cases": 0.002,
      "median_latency_s": 0.56,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "baseten/elephant-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.44,
      "ci_low": 0.37,
      "ci_high": 0.51,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.18,
      "stability": 0,
      "usd_per_1k_cases": 0.0067,
      "median_latency_s": 0.61,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "inception/mercury-decide",
      "provider": "inception",
      "kind": "decision model",
      "metric": 0.4,
      "ci_low": 0.33,
      "ci_high": 0.47,
      "guard": {
        "share": 0.01,
        "threshold": 0.96,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.26,
      "stability": 0.04,
      "usd_per_1k_cases": 0.0037,
      "median_latency_s": 0.69,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "togethercomputer/tev1-4b-experimental",
      "provider": "togethercomputer",
      "kind": "decision model",
      "metric": 0.38,
      "ci_low": 0.31,
      "ci_high": 0.44,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.24,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0083,
      "median_latency_s": 0.64,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "jaredpalmer/kev-4b",
      "provider": "jaredpalmer",
      "kind": "decision model",
      "metric": 0.34,
      "ci_low": 0.28,
      "ci_high": 0.41,
      "guard": {
        "share": 0.01,
        "threshold": 0.92,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.27,
      "stability": 0,
      "usd_per_1k_cases": 0.0047,
      "median_latency_s": 0.71,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "upstage/solar-decide-flash",
      "provider": "upstage",
      "kind": "decision model",
      "metric": 0.34,
      "ci_low": 0.28,
      "ci_high": 0.41,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.24,
      "stability": 0.15,
      "usd_per_1k_cases": 0.0217,
      "median_latency_s": 1.41,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "openai/gpt-6-luna-decisions",
      "provider": "openai",
      "kind": "decision model",
      "metric": 0.21,
      "ci_low": 0.16,
      "ci_high": 0.27,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.24,
      "stability": 0,
      "usd_per_1k_cases": 0.008,
      "median_latency_s": 0.7,
      "unsettled": 0.6,
      "flag": true,
      "tie_group": null,
      "note": "The provider refused 120 of 200 cases at no charge. The model got 0.53 of the 80 it answered. We left it out of the paired tests."
    }
  ],
  "tests": "Exact paired McNemar tests between each pair of the top five models, with Holm correction across all 40 pairs in the four jobs. The leader is the model with the highest right, excluding rules and baselines. tie_group is 1 when the leader is not beaten, meaning p Holm against the leader is 0.05 or higher, and for the leader itself. A tested model with p Holm below 0.05 gets tie_group null and a note. A model outside the top five has no paired test in the source, so it gets tie_group null and a note saying whether its Wilson 95% interval overlaps the leader's. Rules and baselines get tie_group null. We never infer a tie from overlapping intervals.",
  "caveats": [
    "The three wrong options are wrong answers other finders gave, then a name pattern the domain uses. The model sees the person's work domain, and 132 of the 200 records include a LinkedIn URL.",
    "The free baseline, the most common address pattern in our finding test, is right on 51%.",
    "OpenAI's GPT-6 Luna Decisions was refused by its provider on 120 of the 200 questions, the same ones on every try, at no charge. It got 53% of the 80 it answered. We left it out of the comparison.",
    "General LLMs give no confidence on this job, so the share they can settle alone is 0%, even though the best is right on 77%."
  ],
  "not_measured": [
    {
      "item": "Confidence from general LLMs",
      "reason": "They state no confidence on address picks, so we cannot rank which answers to trust."
    },
    {
      "item": "GPT-6 Luna Decisions on 120 of the 200 records",
      "reason": "The provider refused them."
    }
  ]
}