{
  "id": "employer",
  "title": "Main employer",
  "question": "Which of these current roles is this person's main job, the employer they mainly work for?",
  "plain_question": "Which of this person's current jobs is their main employer?",
  "matters": "The wrong employer sends your pitch to the wrong company.",
  "n": 102,
  "primary_metric": {
    "name": "Correct",
    "definition": "Share of the 102 records where the model's first pick, among the person's current roles, was the main employer. The answer key is the employer at the domain the person emails from. A failed call counts as wrong."
  },
  "guard_definition": "Share of records the model can settle by itself with at most 1 mistake in 100, at the confidence cutoff that works best for it. A record counts when the model gives a top pick. General LLMs state no confidence here, so each settles all of its picks or none.",
  "headline": "Claude Haiku 5.5 and DeepSeek V4.1 Flash lead at 86% and the top five are too close to call. With 102 records, each score could be about 8 points higher or lower.",
  "leader": "anthropic/claude-haiku-5.5",
  "rows": [
    {
      "model": "anthropic/claude-haiku-5.5",
      "provider": "anthropic",
      "kind": "LLM",
      "metric": 0.86,
      "ci_low": 0.78,
      "ci_high": 0.92,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.02,
      "usd_per_1k_cases": 0.1033,
      "median_latency_s": 0.96,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Leader."
    },
    {
      "model": "deepseek/deepseek-v4.1-flash-1017",
      "provider": "deepseek",
      "kind": "LLM",
      "metric": 0.86,
      "ci_low": 0.78,
      "ci_high": 0.92,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.05,
      "usd_per_1k_cases": 0.031,
      "median_latency_s": 1.01,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "openai/gpt-6-luna",
      "provider": "openai",
      "kind": "LLM",
      "metric": 0.85,
      "ci_low": 0.77,
      "ci_high": 0.91,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.02,
      "usd_per_1k_cases": 0.0597,
      "median_latency_s": 1.51,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "openai/gpt-6-luna-decisions",
      "provider": "openai",
      "kind": "decision model",
      "metric": 0.85,
      "ci_low": 0.77,
      "ci_high": 0.91,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0,
      "usd_per_1k_cases": 0.0413,
      "median_latency_s": 0.68,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "parallel/fast",
      "provider": "parallel",
      "kind": "web agent",
      "metric": 0.84,
      "ci_low": 0.76,
      "ci_high": 0.9,
      "guard": {
        "share": 0.06,
        "threshold": 0.99,
        "passed": 6,
        "wrong": 0
      },
      "ece": 0.05,
      "stability": 0,
      "usd_per_1k_cases": 1,
      "median_latency_s": 1.01,
      "unsettled": 0,
      "flag": false,
      "tie_group": 1,
      "note": "Too close to call with the leader (p Holm 1.0000)."
    },
    {
      "model": "google/gemini-4-argon",
      "provider": "google",
      "kind": "LLM",
      "metric": 0.84,
      "ci_low": 0.76,
      "ci_high": 0.9,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.01,
      "usd_per_1k_cases": 1.7427,
      "median_latency_s": 4.42,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "openai/gpt-6.1-terra",
      "provider": "openai",
      "kind": "LLM",
      "metric": 0.84,
      "ci_low": 0.76,
      "ci_high": 0.9,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.01,
      "usd_per_1k_cases": 0.5967,
      "median_latency_s": 1.41,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "liquid/d1",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.83,
      "ci_low": 0.75,
      "ci_high": 0.89,
      "guard": {
        "share": 0.17,
        "threshold": 1,
        "passed": 17,
        "wrong": 0
      },
      "ece": 0.14,
      "stability": 0.03,
      "usd_per_1k_cases": 0.009,
      "median_latency_s": 0.63,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "nace-ai/drex-max",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.83,
      "ci_low": 0.75,
      "ci_high": 0.89,
      "guard": {
        "share": 0.11,
        "threshold": 0.98,
        "passed": 11,
        "wrong": 0
      },
      "ece": 0.06,
      "stability": 0.01,
      "usd_per_1k_cases": 0.0277,
      "median_latency_s": 0.82,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "minimax/minimax-m3.1-flash",
      "provider": "minimax",
      "kind": "LLM",
      "metric": 0.83,
      "ci_low": 0.75,
      "ci_high": 0.89,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0,
      "usd_per_1k_cases": 0.0673,
      "median_latency_s": 0.98,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "z-ai/glm-5.3-flash",
      "provider": "z-ai",
      "kind": "LLM",
      "metric": 0.83,
      "ci_low": 0.75,
      "ci_high": 0.89,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.09,
      "usd_per_1k_cases": 0.0443,
      "median_latency_s": 1.03,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "perplexity/pplx-decider-v1.2",
      "provider": "perplexity",
      "kind": "decision model",
      "metric": 0.82,
      "ci_low": 0.74,
      "ci_high": 0.89,
      "guard": {
        "share": 0.12,
        "threshold": 0.99,
        "passed": 12,
        "wrong": 0
      },
      "ece": 0.07,
      "stability": 0,
      "usd_per_1k_cases": 0.0093,
      "median_latency_s": 0.67,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "baseten/elephant-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.82,
      "ci_low": 0.74,
      "ci_high": 0.89,
      "guard": {
        "share": 0.02,
        "threshold": 0.99,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.07,
      "stability": 0,
      "usd_per_1k_cases": 0.0163,
      "median_latency_s": 0.59,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "baseten/mammoth-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.82,
      "ci_low": 0.74,
      "ci_high": 0.89,
      "guard": {
        "share": 0.01,
        "threshold": 1,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.07,
      "stability": 0,
      "usd_per_1k_cases": 0.0467,
      "median_latency_s": 0.6,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "google/gemini-3.8-flash",
      "provider": "google",
      "kind": "LLM",
      "metric": 0.82,
      "ci_low": 0.74,
      "ci_high": 0.89,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": null,
      "stability": 0.04,
      "usd_per_1k_cases": 0.6513,
      "median_latency_s": 2.42,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "nace-ai/drex-v2.0",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.81,
      "ci_low": 0.73,
      "ci_high": 0.88,
      "guard": {
        "share": 0.02,
        "threshold": 0.99,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.09,
      "stability": 0,
      "usd_per_1k_cases": 0.023,
      "median_latency_s": 0.84,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "typesafe/jev-1.13",
      "provider": "typesafe",
      "kind": "decision model",
      "metric": 0.81,
      "ci_low": 0.73,
      "ci_high": 0.88,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0,
      "usd_per_1k_cases": 0.0317,
      "median_latency_s": 0.56,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "liquid/d1.1",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.8,
      "ci_low": 0.72,
      "ci_high": 0.87,
      "guard": {
        "share": 0.09,
        "threshold": 0.99,
        "passed": 9,
        "wrong": 0
      },
      "ece": 0.07,
      "stability": 0.08,
      "usd_per_1k_cases": 0.009,
      "median_latency_s": 0.59,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "perplexity/pplx-decider-flash-v1.2",
      "provider": "perplexity",
      "kind": "decision model",
      "metric": 0.8,
      "ci_low": 0.72,
      "ci_high": 0.87,
      "guard": {
        "share": 0.02,
        "threshold": 0.99,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.08,
      "stability": 0,
      "usd_per_1k_cases": 0.0047,
      "median_latency_s": 0.54,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "togethercomputer/tev1-4b-experimental",
      "provider": "togethercomputer",
      "kind": "decision model",
      "metric": 0.8,
      "ci_low": 0.72,
      "ci_high": 0.87,
      "guard": {
        "share": 0.02,
        "threshold": 1,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.12,
      "stability": 0,
      "usd_per_1k_cases": 0.018,
      "median_latency_s": 0.64,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "liquid/d1-ultra",
      "provider": "liquid",
      "kind": "decision model",
      "metric": 0.8,
      "ci_low": 0.72,
      "ci_high": 0.87,
      "guard": {
        "share": 0.01,
        "threshold": 0.99,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.04,
      "stability": 0.03,
      "usd_per_1k_cases": 0.019,
      "median_latency_s": 0.81,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "typesafe/jev-1.14",
      "provider": "typesafe",
      "kind": "decision model",
      "metric": 0.8,
      "ci_low": 0.72,
      "ci_high": 0.87,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0.03,
      "usd_per_1k_cases": 0.0317,
      "median_latency_s": 0.56,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "nace-ai/drex-v1.5",
      "provider": "nace-ai",
      "kind": "decision model",
      "metric": 0.79,
      "ci_low": 0.71,
      "ci_high": 0.86,
      "guard": {
        "share": 0.04,
        "threshold": 1,
        "passed": 4,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0,
      "usd_per_1k_cases": 0.0153,
      "median_latency_s": 0.73,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "morph/reflex",
      "provider": "morph",
      "kind": "decision model",
      "metric": 0.79,
      "ci_low": 0.71,
      "ci_high": 0.86,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.13,
      "stability": 0,
      "usd_per_1k_cases": 0.0197,
      "median_latency_s": 0.73,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "upstage/solar-decide",
      "provider": "upstage",
      "kind": "decision model",
      "metric": 0.78,
      "ci_low": 0.69,
      "ci_high": 0.85,
      "guard": {
        "share": 0.02,
        "threshold": 1,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.18,
      "stability": 0.06,
      "usd_per_1k_cases": 0.0337,
      "median_latency_s": 1.26,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "scaledown/classify-1",
      "provider": "scaledown",
      "kind": "decision model",
      "metric": 0.78,
      "ci_low": 0.69,
      "ci_high": 0.85,
      "guard": {
        "share": 0.01,
        "threshold": 0.98,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.1,
      "stability": 0,
      "usd_per_1k_cases": 0.025,
      "median_latency_s": 0.71,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "inception/mercury-decide",
      "provider": "inception",
      "kind": "decision model",
      "metric": 0.77,
      "ci_low": 0.68,
      "ci_high": 0.84,
      "guard": {
        "share": 0.14,
        "threshold": 1,
        "passed": 14,
        "wrong": 0
      },
      "ece": 0.12,
      "stability": 0,
      "usd_per_1k_cases": 0.0087,
      "median_latency_s": 0.71,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "upstage/solar-decide-flash",
      "provider": "upstage",
      "kind": "decision model",
      "metric": 0.77,
      "ci_low": 0.68,
      "ci_high": 0.84,
      "guard": {
        "share": 0.01,
        "threshold": 1,
        "passed": 1,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0.02,
      "usd_per_1k_cases": 0.0337,
      "median_latency_s": 1.16,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "baseten/joey-1",
      "provider": "baseten",
      "kind": "decision model",
      "metric": 0.77,
      "ci_low": 0.68,
      "ci_high": 0.84,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.1,
      "stability": 0,
      "usd_per_1k_cases": 0.0047,
      "median_latency_s": 0.6,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "cloudflare/clef-omni",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.76,
      "ci_low": 0.67,
      "ci_high": 0.84,
      "guard": {
        "share": 0.03,
        "threshold": 0.99,
        "passed": 3,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0,
      "usd_per_1k_cases": 0.0693,
      "median_latency_s": 0.64,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "cloudflare/clef",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.75,
      "ci_low": 0.66,
      "ci_high": 0.83,
      "guard": {
        "share": 0.04,
        "threshold": 1,
        "passed": 4,
        "wrong": 0
      },
      "ece": 0.16,
      "stability": 0.04,
      "usd_per_1k_cases": 0.1007,
      "median_latency_s": 0.81,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "cloudflare/clef-flash",
      "provider": "cloudflare",
      "kind": "decision model",
      "metric": 0.75,
      "ci_low": 0.65,
      "ci_high": 0.82,
      "guard": {
        "share": 0.04,
        "threshold": 0.99,
        "passed": 4,
        "wrong": 0
      },
      "ece": 0.17,
      "stability": 0,
      "usd_per_1k_cases": 0.0123,
      "median_latency_s": 0.71,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "microsoft/microsoft-decision-1",
      "provider": "microsoft",
      "kind": "decision model",
      "metric": 0.74,
      "ci_low": 0.64,
      "ci_high": 0.81,
      "guard": {
        "share": 0.02,
        "threshold": 1,
        "passed": 2,
        "wrong": 0
      },
      "ece": 0.08,
      "stability": 0.03,
      "usd_per_1k_cases": 0.0153,
      "median_latency_s": 0.72,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "jaredpalmer/kev-4b",
      "provider": "jaredpalmer",
      "kind": "decision model",
      "metric": 0.72,
      "ci_low": 0.62,
      "ci_high": 0.79,
      "guard": {
        "share": 0.13,
        "threshold": 0.98,
        "passed": 13,
        "wrong": 0
      },
      "ece": 0.15,
      "stability": 0,
      "usd_per_1k_cases": 0.0163,
      "median_latency_s": 0.63,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "No paired test, because we only tested the top five. Its likely range overlaps the leader's."
    },
    {
      "model": "the one non-advisory current role, else the role started most recently",
      "provider": null,
      "kind": "baseline",
      "metric": 0.69,
      "ci_low": 0.59,
      "ci_high": 0.77,
      "guard": {
        "share": 0,
        "threshold": null,
        "passed": 0,
        "wrong": 0
      },
      "ece": 0.31,
      "stability": null,
      "usd_per_1k_cases": 0,
      "median_latency_s": null,
      "unsettled": 0,
      "flag": false,
      "tie_group": null,
      "note": "Free comparison rule with no model. Not tested against the leader."
    }
  ],
  "tests": "Exact paired McNemar tests between each pair of the top five models, with Holm correction across all 40 pairs in the four jobs. The leader is the model with the highest right, excluding rules and baselines. tie_group is 1 when the leader is not beaten, meaning p Holm against the leader is 0.05 or higher, and for the leader itself. A tested model with p Holm below 0.05 gets tie_group null and a note. A model outside the top five has no paired test in the source, so it gets tie_group null and a note saying whether its Wilson 95% interval overlaps the leader's. Rules and baselines get tie_group null. We never infer a tie from overlapping intervals.",
  "caveats": [
    "With 102 records, each score could be about 8 points higher or lower.",
    "The records come from 1,439 purchased profiles of people with current roles at two or more employers. Only 6 of the 102 are parent and subsidiary pairs.",
    "The free baseline, \"the one non-advisory current role, else the role started most recently\", is right on 69%.",
    "General LLMs give no confidence on this job, so the share they can settle alone is 0%."
  ],
  "not_measured": [
    {
      "item": "Confidence from general LLMs",
      "reason": "They state no confidence on employer picks, so we cannot rank which answers to trust."
    },
    {
      "item": "Records from other companies",
      "reason": "All records come from one company's outbound and CRM history."
    }
  ]
}