{
 "id": "still",
 "title": "Job change detection",
 "question": "Which source spots who left a company without flagging who stayed?",
 "n": 113,
 "positives": 13,
 "primary_metric": {
  "name": "Youden's J",
  "definition": "Youden's J = share of leavers called 'left' + share of stayers kept 'there' - 1. An unknown verdict counts as neither. Newcombe 95% interval, built from the two Wilson intervals."
 },
 "rows": [
  {
   "vendor": "Forager",
   "n": 44,
   "metric": 0.9459,
   "ci_low": 0.5709,
   "ci_high": 0.985,
   "secondary": {
    "precision": 0.875,
    "recall": 1.0,
    "coverage": 0.9773
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "DeepLine",
     "usd_per_call": 0.03127273,
     "usd_per_1k_correct": 32.76,
     "median_latency_s": 2.9
    }
   ],
   "note": "ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "CPJ",
   "n": 44,
   "metric": 0.9459,
   "ci_low": 0.5709,
   "ci_high": 0.985,
   "secondary": {
    "precision": 0.7778,
    "recall": 1.0,
    "coverage": 1.0
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Clay",
     "usd_per_call": 0.0534,
     "usd_per_1k_correct": 55.94,
     "median_latency_s": null
    }
   ],
   "note": "ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "Enrich CRM",
   "n": 44,
   "metric": 0.9459,
   "ci_low": 0.5709,
   "ci_high": 0.985,
   "secondary": {
    "precision": 0.875,
    "recall": 1.0,
    "coverage": 0.9773
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Clay",
     "usd_per_call": 0.09938864,
     "usd_per_1k_correct": 104.12,
     "median_latency_s": null
    }
   ],
   "note": "ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "MoltSets",
   "n": 107,
   "metric": 0.79,
   "ci_low": 0.4245,
   "ci_high": 0.8583,
   "secondary": {
    "precision": 0.3889,
    "recall": 1.0,
    "coverage": 0.9065
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00925234,
     "usd_per_1k_correct": 11.51,
     "median_latency_s": 1.2
    }
   ],
   "note": "ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "Fiber AI (live)",
   "n": 44,
   "metric": 0.7568,
   "ci_low": 0.3688,
   "ci_high": 0.8664,
   "secondary": {
    "precision": 0.4375,
    "recall": 1.0,
    "coverage": 1.0
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.04,
     "usd_per_1k_correct": 50.29,
     "median_latency_s": 5.9
    }
   ],
   "note": "lower bound: name-only source; ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "HarvestAPI",
   "n": 107,
   "metric": 0.73,
   "ci_low": 0.3633,
   "ci_high": 0.8073,
   "secondary": {
    "precision": 0.2121,
    "recall": 1.0,
    "coverage": 0.9907
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.002,
     "usd_per_1k_correct": 2.67,
     "median_latency_s": 4.4
    }
   ],
   "note": "lower bound: name-only source; ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "AnyAPI",
   "n": 107,
   "metric": 0.73,
   "ci_low": 0.3633,
   "ci_high": 0.8073,
   "secondary": {
    "precision": 0.2121,
    "recall": 1.0,
    "coverage": 0.9907
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.0029,
     "usd_per_1k_correct": 3.88,
     "median_latency_s": 3.9
    }
   ],
   "note": "lower bound: name-only source; ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "Edges",
   "n": 107,
   "metric": 0.73,
   "ci_low": 0.3633,
   "ci_high": 0.8073,
   "secondary": {
    "precision": 0.2121,
    "recall": 1.0,
    "coverage": 0.9907
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "DeepLine",
     "usd_per_call": 0.01715888,
     "usd_per_1k_correct": 22.95,
     "median_latency_s": 9.9
    }
   ],
   "note": "lower bound: name-only source; ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "LiveData",
   "n": 44,
   "metric": 0.7297,
   "ci_low": 0.3411,
   "ci_high": 0.846,
   "secondary": {
    "precision": 0.7778,
    "recall": 1.0,
    "coverage": 0.8182
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Clay",
     "usd_per_call": 0.08320909,
     "usd_per_1k_correct": 107.68,
     "median_latency_s": null
    }
   ],
   "note": "ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "Fetchin",
   "n": 107,
   "metric": 0.72,
   "ci_low": 0.3532,
   "ci_high": 0.7986,
   "secondary": {
    "precision": 0.2059,
    "recall": 1.0,
    "coverage": 0.9907
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00148598,
     "usd_per_1k_correct": 2.01,
     "median_latency_s": 3.8
    }
   ],
   "note": "lower bound: name-only source; ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "Trigify",
   "n": 44,
   "metric": 0.7027,
   "ci_low": 0.3137,
   "ci_high": 0.8251,
   "secondary": {
    "precision": 0.4118,
    "recall": 1.0,
    "coverage": 0.9773
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Clay",
     "usd_per_call": 0.05218636,
     "usd_per_1k_correct": 69.58,
     "median_latency_s": null
    }
   ],
   "note": "lower bound: name-only source; ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "Icypeas",
   "n": 44,
   "metric": 0.6409,
   "ci_low": 0.2392,
   "ci_high": 0.7965,
   "secondary": {
    "precision": 0.6667,
    "recall": 0.8571,
    "coverage": 0.8636
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "DeepLine",
     "usd_per_call": 0.01813636,
     "usd_per_1k_correct": 22.8,
     "median_latency_s": 3.6
    }
   ],
   "note": "ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "Perplexity Sonar",
   "n": 113,
   "metric": 0.6262,
   "ci_low": 0.3427,
   "ci_high": 0.757,
   "secondary": {
    "precision": 0.44,
    "recall": 0.8462,
    "coverage": 0.9204
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "OpenRouter",
     "usd_per_call": 0.00510885,
     "usd_per_1k_correct": 6.49,
     "median_latency_s": 3.9
    }
   ],
   "note": ""
  },
  {
   "vendor": "Reverse Contact",
   "n": 44,
   "metric": 0.6216,
   "ci_low": 0.2326,
   "ci_high": 0.7594,
   "secondary": {
    "precision": 0.3889,
    "recall": 1.0,
    "coverage": 0.9318
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Clay",
     "usd_per_call": 0.04975909,
     "usd_per_1k_correct": 72.98,
     "median_latency_s": null
    }
   ],
   "note": "lower bound: name-only source; ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "BlitzAPI",
   "n": 107,
   "metric": 0.61,
   "ci_low": 0.2424,
   "ci_high": 0.6998,
   "secondary": {
    "precision": 0.2121,
    "recall": 1.0,
    "coverage": 0.8785
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00043486,
     "usd_per_1k_correct": 0.68,
     "median_latency_s": 1.4
    }
   ],
   "note": "ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "Crawl4AI",
   "n": 107,
   "metric": 0.61,
   "ci_low": 0.2424,
   "ci_high": 0.6998,
   "secondary": {
    "precision": 0.2188,
    "recall": 1.0,
    "coverage": 0.8692
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00084112,
     "usd_per_1k_correct": 1.32,
     "median_latency_s": 40.8
    }
   ],
   "note": "lower bound: name-only source; ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "Apollo",
   "n": 50,
   "metric": 0.5572,
   "ci_low": 0.2521,
   "ci_high": 0.7534,
   "secondary": {
    "precision": 0.6429,
    "recall": 0.6923,
    "coverage": 1.0
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.026,
     "usd_per_1k_correct": 31.71,
     "median_latency_s": 1.5
    }
   ],
   "note": "ran on the 50 rows of the frozen 50-row subset"
  },
  {
   "vendor": "Snov.io",
   "n": 44,
   "metric": 0.5328,
   "ci_low": 0.129,
   "ci_high": 0.7063,
   "secondary": {
    "precision": 0.5455,
    "recall": 0.8571,
    "coverage": 0.8182
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Clay",
     "usd_per_call": 0.08320909,
     "usd_per_1k_correct": 118.1,
     "median_latency_s": null
    }
   ],
   "note": "ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "LeadMagic job change",
   "n": 44,
   "metric": 0.5058,
   "ci_low": 0.102,
   "ci_high": 0.6831,
   "secondary": {
    "precision": 0.375,
    "recall": 0.8571,
    "coverage": 0.9091
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.06647727,
     "usd_per_1k_correct": 97.5,
     "median_latency_s": 8.4
    }
   ],
   "note": "ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "ContactOut",
   "n": 113,
   "metric": 0.4654,
   "ci_low": 0.1924,
   "ci_high": 0.6806,
   "secondary": {
    "precision": 0.4211,
    "recall": 0.6154,
    "coverage": 0.9292
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.01858407,
     "usd_per_1k_correct": 22.58,
     "median_latency_s": 1.3
    }
   ],
   "note": ""
  },
  {
   "vendor": "People Data Labs",
   "n": 50,
   "metric": 0.4532,
   "ci_low": 0.1533,
   "ci_high": 0.6777,
   "secondary": {
    "precision": 0.7273,
    "recall": 0.6154,
    "coverage": 0.88
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "DeepLine",
     "usd_per_call": 0.184,
     "usd_per_1k_correct": 235.9,
     "median_latency_s": 3.2
    }
   ],
   "note": "ran on the 50 rows of the frozen 50-row subset"
  },
  {
   "vendor": "Parallel Task",
   "n": 113,
   "metric": 0.3785,
   "ci_low": 0.1175,
   "ci_high": 0.6154,
   "secondary": {
    "precision": 0.3889,
    "recall": 0.5385,
    "coverage": 0.9292
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Parallel",
     "usd_per_call": 0.005,
     "usd_per_1k_correct": 6.21,
     "median_latency_s": null
    }
   ],
   "note": ""
  },
  {
   "vendor": "LeadOcean",
   "n": 107,
   "metric": 0.3743,
   "ci_low": 0.0059,
   "ci_high": 0.595,
   "secondary": {
    "precision": 0.3571,
    "recall": 0.7143,
    "coverage": 0.7477
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00373832,
     "usd_per_1k_correct": 5.63,
     "median_latency_s": 1.0
    }
   ],
   "note": "ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "Bright Data",
   "n": 44,
   "metric": 0.3436,
   "ci_low": -0.0566,
   "ci_high": 0.5376,
   "secondary": {
    "precision": 0.4,
    "recall": 0.8571,
    "coverage": 0.75
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00115909,
     "usd_per_1k_correct": 2.12,
     "median_latency_s": 20.3
    }
   ],
   "note": "lower bound: name-only source; ran on the 44 rows of the frozen 50-row subset that hold a LinkedIn URL"
  },
  {
   "vendor": "Ocean.io",
   "n": 113,
   "metric": 0.1785,
   "ci_low": -0.0872,
   "ci_high": 0.424,
   "secondary": {
    "precision": 0.3043,
    "recall": 0.5385,
    "coverage": 0.7699
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00623628,
     "usd_per_1k_correct": 9.93,
     "median_latency_s": 1.8
    }
   ],
   "note": ""
  },
  {
   "vendor": "Prospeo",
   "n": 50,
   "metric": 0.0603,
   "ci_low": -0.2024,
   "ci_high": 0.3502,
   "secondary": {
    "precision": 0.3846,
    "recall": 0.3846,
    "coverage": 0.84
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.0147,
     "usd_per_1k_correct": 24.5,
     "median_latency_s": 1.2
    }
   ],
   "note": "ran on the 50 rows of the frozen 50-row subset"
  },
  {
   "vendor": "Apify profile scraper",
   "n": 107,
   "metric": 0.3471,
   "ci_low": -0.0353,
   "ci_high": 0.499,
   "secondary": {
    "precision": 0.24,
    "recall": 0.8571,
    "coverage": 0.6916
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Apify",
     "usd_per_call": 0.002206,
     "usd_per_1k_correct": 4.29,
     "median_latency_s": null
    }
   ],
   "note": "lower bound: name-only source; ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "Exa people search",
   "n": 113,
   "metric": -0.0285,
   "ci_low": -0.2774,
   "ci_high": 0.2365,
   "secondary": {
    "precision": 0.2222,
    "recall": 0.4615,
    "coverage": 0.6903
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.007,
     "usd_per_1k_correct": 13.88,
     "median_latency_s": 2.4
    }
   ],
   "note": "lower bound: name-only source"
  },
  {
   "vendor": "Hunter",
   "n": 107,
   "metric": -0.1386,
   "ci_low": -0.4693,
   "ci_high": 0.1481,
   "secondary": {
    "precision": 0.6667,
    "recall": 0.5714,
    "coverage": 0.3364
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.0016486,
     "usd_per_1k_correct": 5.35,
     "median_latency_s": 1.5
    }
   ],
   "note": "ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "Parallel Decisions",
   "n": 113,
   "metric": -0.3592,
   "ci_low": -0.534,
   "ci_high": -0.0703,
   "secondary": {
    "precision": 0.4286,
    "recall": 0.2308,
    "coverage": 0.4248
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Parallel",
     "usd_per_call": 0.001,
     "usd_per_1k_correct": 2.57,
     "median_latency_s": 1.0
    }
   ],
   "note": ""
  },
  {
   "vendor": "Dropleads",
   "n": 113,
   "metric": -0.4862,
   "ci_low": -0.627,
   "ci_high": -0.2004,
   "secondary": {
    "precision": 0.2222,
    "recall": 0.1538,
    "coverage": 0.4248
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.0015292,
     "usd_per_1k_correct": 4.55,
     "median_latency_s": 28.4
    }
   ],
   "note": ""
  },
  {
   "vendor": "ScrapeCreators",
   "n": 107,
   "metric": -0.5743,
   "ci_low": -0.785,
   "ci_high": -0.2097,
   "secondary": {
    "precision": 0.4,
    "recall": 0.2857,
    "coverage": 0.1776
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00033383,
     "usd_per_1k_correct": 2.23,
     "median_latency_s": 3.8
    }
   ],
   "note": "lower bound: name-only source; ran on the 107 rows that hold a LinkedIn URL"
  },
  {
   "vendor": "Datagma job change",
   "n": 50,
   "metric": -0.9459,
   "ci_low": -0.985,
   "ci_high": -0.6868,
   "secondary": {
    "precision": 0.0,
    "recall": 0.0,
    "coverage": 0.12
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "DeepLine",
     "usd_per_call": 0.0084,
     "usd_per_1k_correct": 210.0,
     "median_latency_s": 10.2
    }
   ],
   "note": "ran on the 50 rows of the frozen 50-row subset"
  }
 ],
 "not_measured": [
  {
   "vendor": "Diffbot",
   "reason": "Treg's Diffbot account used up its monthly allowance (402)."
  },
  {
   "vendor": "Aviato",
   "reason": "its answer holds no employment."
  },
  {
   "vendor": "SMARTe",
   "reason": "returns a transaction id only."
  },
  {
   "vendor": "Muse Spark",
   "reason": "outside our model account's data policy."
  }
 ],
 "tests": "Holm-corrected exact paired McNemar against the leader, alpha 0.05, over the rows both answered; a correct row is a decided, right verdict. On 13 left rows no pair is compared in report.py's table either.",
 "caveats": [
  "The key comes from one company's records: 13 people who left and 100 who stayed, read from our own mail over a 90-day window.",
  "The left side is 13 rows, 7 of them with a LinkedIn URL: left caught carries wide intervals, and no paired test has the power to run on it.",
  "A name-only source's employer is matched by name alone and the matcher misses some same-company names, so its 'there kept' and its J are lower bounds.",
  "Three there rows may be wrong in the key: every deciding source calls them left.",
  "Not a row, because it is not a vendor: the free 'assume still there' baseline scores J +0.00 [-0.04, +0.23].",
  "Diffbot, Aviato, SMARTe and Muse Spark were not measured; see not_measured."
 ]
}