{
  "id": "dup",
  "title": "Duplicate records",
  "question": "Are record_a and record_b the same real person?",
  "plain_question": "Are these two CRM records the same person?",
  "matters": "A missed duplicate means two reps email the same buyer. A wrong merge loses a real contact.",
  "n": 200,
  "primary_metric": {
    "name": "Correct",
    "definition": "Share of the 200 record pairs where the model's first answer to \"same real person?\" matched the answer we already knew. A failed call counts as wrong."
  },
  "guard_definition": "Share of records the model can settle by itself with at most 1 mistake in 100, at the confidence cutoff that works best for it. Only a \"yes\" counts, because a wrong merge costs the most.",
  "headline": "The best model, drex-max, gets 79% of duplicate pairs right and four others are too close to call. A free rule built from the answer key gets 85%.",
  "leader": "nace-ai/drex-max",
  "rows": [
    {
      "model": "names differ or same email domain",
      "provider": null,
      "kind": "rule fitted on this key",
      "metric": 0.85,
      "ci_low": 0.8,
      "ci_high": 0.9,
      "guard": {
        "share": 0.35,
        "threshold": 1,
        "passed": 71,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": null,
      "usd_per_1k_cases": 0,
      "median_latency_s": null,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "Free rule built by looking at this answer key, so its score runs high. Not tested against the leader."
    },
    {
      "model": "nace-ai/drex-max",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.79,
      "ci_low": 0.73,
      "ci_high": 0.84,
      "guard": {
        "share": 0.05,
        "threshold": 0.97,
        "passed": 10,
        "wrong": 0
      },
      "ece": 0.12,
      "stability": 0.03,
      "usd_per_1k_cases": 0.0123,
      "median_latency_s": 0.81,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Leader."
    },
    {
      "model": "liquid/d1-ultra",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.79,
      "ci_low": 0.73,
      "ci_high": 0.84,
      "guard": {
        "share": 0.03,
        "threshold": 0.94,
        "passed": 6,
        "wrong": 0
      },
      "ece": 0.06,
      "stability": 0.06,
      "usd_per_1k_cases": 0.007,
      "median_latency_s": 2.29,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "google/gemini-4-argon",
      "provider": "google",
      "kind": "LLM",
      "metric": 0.78,
      "ci_low": 0.72,
      "ci_high": 0.83,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.21,
      "stability": 0.03,
      "usd_per_1k_cases": 1.1463,
      "median_latency_s": 4.54,
      "unsettled": 0.01,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "perplexity/pplx-decider-v1.2",
      "provider": "perplexity",
      "kind": "decision model",
      "metric": 0.77,
      "ci_low": 0.7,
      "ci_high": 0.82,
      "guard": {
        "share": 0.01,
        "threshold": 0.99,
        "passed": 3,
        "wrong": 0
      },
      "ece": 0.12,
      "stability": 0,
      "usd_per_1k_cases": 0.004,
      "median_latency_s": 0.69,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "google/gemini-3.8-flash",
      "provider": "google",
      "kind": "LLM",
      "metric": 0.75,
      "ci_low": 0.69,
      "ci_high": 0.8,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.19,
      "stability": 0.06,
      "usd_per_1k_cases": 0.351,
      "median_latency_s": 2.47,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "parallel/fast",
      "provider": "parallel",
      "kind": "web agent",
      "metric": 0.74,
      "ci_low": 0.68,
      "ci_high": 0.8,
      "guard": {
        "share": 0.01,
        "threshold": 0.99,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.07,
      "stability": 0.03,
      "usd_per_1k_cases": 1,
      "median_latency_s": 0.97,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "baseten/elephant-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.74,
      "ci_low": 0.68,
      "ci_high": 0.8,
      "guard": {
        "share": 0.14,
        "threshold": 0.79,
        "passed": 28,
        "wrong": 0
      },
      "ece": 0.08,
      "stability": 0,
      "usd_per_1k_cases": 0.008,
      "median_latency_s": 0.59,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "z-ai/glm-5.3-flash",
      "provider": "z-ai",
      "kind": "LLM",
      "metric": 0.72,
      "ci_low": 0.65,
      "ci_high": 0.78,
      "guard": {
        "share": 0.05,
        "threshold": 0.97,
        "passed": 10,
        "wrong": 0
      },
      "ece": 0.16,
      "stability": 0.18,
      "usd_per_1k_cases": 0.0277,
      "median_latency_s": 1.16,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "baseten/mammoth-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.72,
      "ci_low": 0.65,
      "ci_high": 0.78,
      "guard": {
        "share": 0.04,
        "threshold": 0.98,
        "passed": 8,
        "wrong": 0
      },
      "ece": 0.18,
      "stability": 0,
      "usd_per_1k_cases": 0.0227,
      "median_latency_s": 0.6,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "openai/gpt-6-luna-decisions",
      "provider": "openai",
      "kind": "decision model",
      "metric": 0.72,
      "ci_low": 0.65,
      "ci_high": 0.78,
      "guard": {
        "share": 0.01,
        "threshold": 0.99,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0,
      "usd_per_1k_cases": 0.0247,
      "median_latency_s": 0.67,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "morph/reflex",
      "provider": "morph",
      "kind": "decision model",
      "metric": 0.71,
      "ci_low": 0.64,
      "ci_high": 0.77,
      "guard": {
        "share": 0.12,
        "threshold": 0.99,
        "passed": 23,
        "wrong": 0
      },
      "ece": 0.18,
      "stability": 0.04,
      "usd_per_1k_cases": 0.009,
      "median_latency_s": 0.74,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "minimax/minimax-m3.1-flash",
      "provider": "minimax",
      "kind": "LLM",
      "metric": 0.71,
      "ci_low": 0.64,
      "ci_high": 0.77,
      "guard": {
        "share": 0.01,
        "threshold": 0.99,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.21,
      "stability": 0.03,
      "usd_per_1k_cases": 0.031,
      "median_latency_s": 0.82,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "microsoft/microsoft-decision-1",
      "provider": "microsoft",
      "kind": "decision model",
      "metric": 0.7,
      "ci_low": 0.64,
      "ci_high": 0.76,
      "guard": {
        "share": 0.12,
        "threshold": 0.78,
        "passed": 25,
        "wrong": 0
      },
      "ece": 0.17,
      "stability": 0.03,
      "usd_per_1k_cases": 0.005,
      "median_latency_s": 0.79,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "anthropic/claude-haiku-5.5",
      "provider": "anthropic",
      "kind": "LLM",
      "metric": 0.7,
      "ci_low": 0.64,
      "ci_high": 0.76,
      "guard": {
        "share": 0.02,
        "threshold": 0.9,
        "passed": 4,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0.03,
      "usd_per_1k_cases": 0.056,
      "median_latency_s": 0.92,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "togethercomputer/tev1-4b-experimental",
      "provider": "togethercomputer",
      "kind": "decision model",
      "metric": 0.7,
      "ci_low": 0.63,
      "ci_high": 0.76,
      "guard": {
        "share": 0.08,
        "threshold": 0.73,
        "passed": 16,
        "wrong": 0
      },
      "ece": 0.14,
      "stability": 0.04,
      "usd_per_1k_cases": 0.008,
      "median_latency_s": 0.66,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "inception/mercury-decide",
      "provider": "inception",
      "kind": "decision model",
      "metric": 0.69,
      "ci_low": 0.63,
      "ci_high": 0.75,
      "guard": {
        "share": 0.06,
        "threshold": 0.93,
        "passed": 12,
        "wrong": 0
      },
      "ece": 0.18,
      "stability": 0.03,
      "usd_per_1k_cases": 0.0033,
      "median_latency_s": 0.67,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "openai/gpt-6.1-terra",
      "provider": "openai",
      "kind": "LLM",
      "metric": 0.69,
      "ci_low": 0.62,
      "ci_high": 0.75,
      "guard": {
        "share": 0.05,
        "threshold": 0.99,
        "passed": 10,
        "wrong": 0
      },
      "ece": 0.25,
      "stability": 0.05,
      "usd_per_1k_cases": 0.3277,
      "median_latency_s": 1.48,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "upstage/solar-decide",
      "provider": "upstage",
      "kind": "decision model",
      "metric": 0.69,
      "ci_low": 0.62,
      "ci_high": 0.75,
      "guard": {
        "share": 0.03,
        "threshold": 0.92,
        "passed": 6,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0.23,
      "usd_per_1k_cases": 0.0237,
      "median_latency_s": 15.47,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "typesafe/jev-1.14",
      "provider": "typesafe",
      "kind": "decision model",
      "metric": 0.68,
      "ci_low": 0.61,
      "ci_high": 0.74,
      "guard": {
        "share": 0.12,
        "threshold": 0.85,
        "passed": 25,
        "wrong": 0
      },
      "ece": 0.14,
      "stability": 0.04,
      "usd_per_1k_cases": 0.0177,
      "median_latency_s": 0.56,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "liquid/d1.1",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.68,
      "ci_low": 0.61,
      "ci_high": 0.74,
      "guard": {
        "share": 0.07,
        "threshold": 0.99,
        "passed": 13,
        "wrong": 0
      },
      "ece": 0.2,
      "stability": 0.03,
      "usd_per_1k_cases": 0.0033,
      "median_latency_s": 0.57,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "perplexity/pplx-decider-flash-v1.2",
      "provider": "perplexity",
      "kind": "decision model",
      "metric": 0.68,
      "ci_low": 0.61,
      "ci_high": 0.74,
      "guard": {
        "share": 0.11,
        "threshold": 0.87,
        "passed": 22,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0,
      "usd_per_1k_cases": 0.002,
      "median_latency_s": 0.54,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "openai/gpt-6-luna",
      "provider": "openai",
      "kind": "LLM",
      "metric": 0.67,
      "ci_low": 0.6,
      "ci_high": 0.73,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.25,
      "stability": 0.09,
      "usd_per_1k_cases": 0.0327,
      "median_latency_s": 1.06,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "liquid/d1",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.67,
      "ci_low": 0.6,
      "ci_high": 0.73,
      "guard": {
        "share": 0.01,
        "threshold": 1,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.19,
      "stability": 0.1,
      "usd_per_1k_cases": 0.0033,
      "median_latency_s": 0.6,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "scaledown/classify-1",
      "provider": "scaledown",
      "kind": "decision model",
      "metric": 0.64,
      "ci_low": 0.57,
      "ci_high": 0.7,
      "guard": {
        "share": 0.14,
        "threshold": 0.71,
        "passed": 29,
        "wrong": 0
      },
      "ece": 0.16,
      "stability": 0.02,
      "usd_per_1k_cases": 0.012,
      "median_latency_s": 0.71,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "cloudflare/clef",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.64,
      "ci_low": 0.57,
      "ci_high": 0.7,
      "guard": {
        "share": 0.04,
        "threshold": 0.98,
        "passed": 8,
        "wrong": 0
      },
      "ece": 0.29,
      "stability": 0.04,
      "usd_per_1k_cases": 0.0527,
      "median_latency_s": 0.81,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "same folded name and registrable email domain",
      "provider": null,
      "kind": "baseline",
      "metric": 0.62,
      "ci_low": 0.56,
      "ci_high": 0.69,
      "guard": {
        "share": 0.12,
        "threshold": 1,
        "passed": 25,
        "wrong": 0
      },
      "ece": 0.38,
      "stability": null,
      "usd_per_1k_cases": 0,
      "median_latency_s": null,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "Free comparison rule with no model. Not tested against the leader."
    },
    {
      "model": "deepseek/deepseek-v4.1-flash-1017",
      "provider": "deepseek",
      "kind": "LLM",
      "metric": 0.62,
      "ci_low": 0.56,
      "ci_high": 0.69,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.26,
      "stability": 0.04,
      "usd_per_1k_cases": 0.016,
      "median_latency_s": 0.88,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "jaredpalmer/kev-4b",
      "provider": "jaredpalmer",
      "kind": "decision model",
      "metric": 0.61,
      "ci_low": 0.55,
      "ci_high": 0.68,
      "guard": {
        "share": 0.04,
        "threshold": 0.9,
        "passed": 8,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0,
      "usd_per_1k_cases": 0.0053,
      "median_latency_s": 0.68,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "typesafe/jev-1.13",
      "provider": "typesafe",
      "kind": "decision model",
      "metric": 0.57,
      "ci_low": 0.51,
      "ci_high": 0.64,
      "guard": {
        "share": 0.1,
        "threshold": 0.86,
        "passed": 21,
        "wrong": 0
      },
      "ece": 0.2,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0177,
      "median_latency_s": 0.56,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "upstage/solar-decide-flash",
      "provider": "upstage",
      "kind": "decision model",
      "metric": 0.56,
      "ci_low": 0.49,
      "ci_high": 0.62,
      "guard": {
        "share": 0.01,
        "threshold": 0.97,
        "passed": 3,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0.14,
      "usd_per_1k_cases": 0.0237,
      "median_latency_s": 1.33,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "cloudflare/clef-flash",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.53,
      "ci_low": 0.46,
      "ci_high": 0.6,
      "guard": {
        "share": 0.05,
        "threshold": 0.95,
        "passed": 10,
        "wrong": 0
      },
      "ece": 0.35,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0067,
      "median_latency_s": 0.73,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "nace-ai/drex-v1.5",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.53,
      "ci_low": 0.46,
      "ci_high": 0.59,
      "guard": {
        "share": 0.04,
        "threshold": 0.98,
        "passed": 9,
        "wrong": 0
      },
      "ece": 0.34,
      "stability": 0,
      "usd_per_1k_cases": 0.005,
      "median_latency_s": 0.73,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "cloudflare/clef-omni",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.5,
      "ci_low": 0.43,
      "ci_high": 0.57,
      "guard": {
        "share": 0.01,
        "threshold": 0.99,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.43,
      "stability": 0,
      "usd_per_1k_cases": 0.0357,
      "median_latency_s": 0.67,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "baseten/joey-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.49,
      "ci_low": 0.43,
      "ci_high": 0.56,
      "guard": {
        "share": 0.06,
        "threshold": 0.93,
        "passed": 12,
        "wrong": 0
      },
      "ece": 0.3,
      "stability": 0,
      "usd_per_1k_cases": 0.002,
      "median_latency_s": 0.58,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    },
    {
      "model": "nace-ai/drex-v2.0",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.48,
      "ci_low": 0.41,
      "ci_high": 0.55,
      "guard": {
        "share": 0.01,
        "threshold": 0.92,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.25,
      "stability": 0,
      "usd_per_1k_cases": 0.0073,
      "median_latency_s": 0.85,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range does not overlap the leader's."
    }
  ],
  "tests": "Exact paired McNemar tests between each pair of the top five models, with Holm correction across all 40 pairs in the four jobs. The leader is the model with the highest right, excluding rules and baselines. tie_group is 1 when the leader is not beaten, meaning p Holm against the leader is 0.05 or higher, and for the leader itself. A tested model with p Holm below 0.05 gets tie_group null and a note. A model outside the top five has no paired test in the source, so it gets tie_group null and a note saying whether its Wilson 95% interval overlaps the leader's. Rules and baselines get tie_group null. We never infer a tie from overlapping intervals.",
  "caveats": [
    "Half the pairs are the same person. The other half are proven different people, so guessing scores about 50%.",
    "The free rule \"names differ, or the email domains match\" is right on 85% of pairs. It works because of how we drew the pairs, so it shows what the test records give away. Do not ship it.",
    "Models do worst on the 17 pairs with overlapping job titles. The leader, drex-max, gets 59% of them right, and Gemini 3.8 Flash gets 12%.",
    "Gemini 4 Argon returned 3 empty answers across the whole test, 2 of them on the first ask, which is the one we score. A failed call counts as wrong."
  ],
  "not_measured": [
    {
      "item": "Records from other companies",
      "reason": "Every record comes from one company's outbound and CRM history. The signals the free rule uses may be weaker or missing elsewhere."
    },
    {
      "item": "Other question wordings",
      "reason": "We froze the wording before any paid call and ran one version."
    }
  ]
}