{
 "id": "verify",
 "title": "Email verification",
 "question": "Which verifier catches dead addresses without flagging live ones?",
 "n": 172,
 "positives": 82,
 "primary_metric": {
  "name": "Youden's J",
  "definition": "Youden's J = share of dead addresses called dead + share of live addresses kept - 1. An unknown verdict counts as neither. Newcombe 95% interval, built from the two Wilson intervals."
 },
 "rows": [
  {
   "vendor": "BounceBan",
   "n": 172,
   "metric": 0.9035,
   "ci_low": 0.8393,
   "ci_high": 0.9678,
   "secondary": {
    "precision": 0.9868,
    "recall": 0.9146,
    "coverage": 0.9942
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.004,
     "usd_per_1k_correct": 4.2,
     "median_latency_s": 2.8
    }
   ],
   "note": ""
  },
  {
   "vendor": "Enrow",
   "n": 172,
   "metric": 0.8602,
   "ci_low": 0.7838,
   "ci_high": 0.9365,
   "secondary": {
    "precision": 0.9268,
    "recall": 0.9268,
    "coverage": 1.0
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.002175,
     "usd_per_1k_correct": 2.34,
     "median_latency_s": 4.2
    },
    {
     "store": "Clay",
     "usd_per_call": 0.0534,
     "usd_per_1k_correct": 57.41,
     "median_latency_s": null
    }
   ],
   "note": ""
  },
  {
   "vendor": "Tomba",
   "n": 172,
   "metric": 0.8569,
   "ci_low": 0.7798,
   "ci_high": 0.9341,
   "secondary": {
    "precision": 0.9605,
    "recall": 0.8902,
    "coverage": 0.9767
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.0089,
     "usd_per_1k_correct": 9.57,
     "median_latency_s": 7.1
    }
   ],
   "note": ""
  },
  {
   "vendor": "Listmint",
   "n": 172,
   "metric": 0.8558,
   "ci_low": 0.7787,
   "ci_high": 0.9329,
   "secondary": {
    "precision": 0.973,
    "recall": 0.878,
    "coverage": 1.0
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Clay",
     "usd_per_call": 0.02925,
     "usd_per_1k_correct": 31.44,
     "median_latency_s": null
    }
   ],
   "note": ""
  },
  {
   "vendor": "Anymailfinder",
   "n": 172,
   "metric": 0.8425,
   "ci_low": 0.763,
   "ci_high": 0.9221,
   "secondary": {
    "precision": 0.9859,
    "recall": 0.8537,
    "coverage": 0.9709
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.004,
     "usd_per_1k_correct": 4.33,
     "median_latency_s": 3.4
    }
   ],
   "note": ""
  },
  {
   "vendor": "Scrubby",
   "n": 172,
   "metric": 0.8214,
   "ci_low": 0.7362,
   "ci_high": 0.9066,
   "secondary": {
    "precision": 0.9861,
    "recall": 0.8659,
    "coverage": 0.9535
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00767116,
     "usd_per_1k_correct": 8.4,
     "median_latency_s": 1.8
    }
   ],
   "note": ""
  },
  {
   "vendor": "Filtrat catch-all",
   "n": 172,
   "metric": 0.807,
   "ci_low": 0.7201,
   "ci_high": 0.894,
   "secondary": {
    "precision": 0.9714,
    "recall": 0.8293,
    "coverage": 0.9942
   },
   "tie_group": 1,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00050436,
     "usd_per_1k_correct": 0.56,
     "median_latency_s": 3.2
    }
   ],
   "note": ""
  },
  {
   "vendor": "Allegrow",
   "n": 172,
   "metric": 0.7282,
   "ci_low": 0.6259,
   "ci_high": 0.8305,
   "secondary": {
    "precision": 0.9571,
    "recall": 0.8171,
    "coverage": 0.9419
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "DeepLine",
     "usd_per_call": 0.01193023,
     "usd_per_1k_correct": 13.77,
     "median_latency_s": 7.5
    }
   ],
   "note": ""
  },
  {
   "vendor": "LeadMagic",
   "n": 172,
   "metric": 0.7127,
   "ci_low": 0.6099,
   "ci_high": 0.8156,
   "secondary": {
    "precision": 0.9403,
    "recall": 0.7683,
    "coverage": 0.9535
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.0059593,
     "usd_per_1k_correct": 6.93,
     "median_latency_s": 8.1
    }
   ],
   "note": ""
  },
  {
   "vendor": "ZeroBounce",
   "n": 172,
   "metric": 0.6938,
   "ci_low": 0.5862,
   "ci_high": 0.8013,
   "secondary": {
    "precision": 0.9706,
    "recall": 0.8049,
    "coverage": 0.9186
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.01315814,
     "usd_per_1k_correct": 15.5,
     "median_latency_s": 1.3
    }
   ],
   "note": ""
  },
  {
   "vendor": "Limadata",
   "n": 172,
   "metric": 0.6783,
   "ci_low": 0.5701,
   "ci_high": 0.7865,
   "secondary": {
    "precision": 0.9394,
    "recall": 0.7561,
    "coverage": 0.9419
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.006,
     "usd_per_1k_correct": 7.12,
     "median_latency_s": 2.6
    }
   ],
   "note": ""
  },
  {
   "vendor": "Dropcontact",
   "n": 172,
   "metric": 0.6285,
   "ci_low": 0.5163,
   "ci_high": 0.7406,
   "secondary": {
    "precision": 0.9661,
    "recall": 0.6951,
    "coverage": 0.9535
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.03380953,
     "usd_per_1k_correct": 41.24,
     "median_latency_s": 23.4
    }
   ],
   "note": ""
  },
  {
   "vendor": "OrbiSearch",
   "n": 172,
   "metric": 0.4455,
   "ci_low": 0.3257,
   "ci_high": 0.5654,
   "secondary": {
    "precision": 1.0,
    "recall": 0.5122,
    "coverage": 0.7674
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00123279,
     "usd_per_1k_correct": 1.68,
     "median_latency_s": 2.1
    }
   ],
   "note": ""
  },
  {
   "vendor": "Hunter",
   "n": 172,
   "metric": 0.4252,
   "ci_low": 0.2904,
   "ci_high": 0.56,
   "secondary": {
    "precision": 0.9818,
    "recall": 0.6585,
    "coverage": 0.7674
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.01218599,
     "usd_per_1k_correct": 17.04,
     "median_latency_s": 2.0
    }
   ],
   "note": ""
  },
  {
   "vendor": "Enrichley",
   "n": 172,
   "metric": 0.3591,
   "ci_low": 0.2424,
   "ci_high": 0.4757,
   "secondary": {
    "precision": 0.9714,
    "recall": 0.4146,
    "coverage": 0.7791
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Clay",
     "usd_per_call": 0.0534,
     "usd_per_1k_correct": 77.18,
     "median_latency_s": null
    }
   ],
   "note": ""
  },
  {
   "vendor": "Filtrat",
   "n": 172,
   "metric": 0.3209,
   "ci_low": 0.1797,
   "ci_high": 0.462,
   "secondary": {
    "precision": 1.0,
    "recall": 0.6098,
    "coverage": 0.7093
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00017006,
     "usd_per_1k_correct": 0.26,
     "median_latency_s": 3.4
    }
   ],
   "note": ""
  },
  {
   "vendor": "TryKitt",
   "n": 172,
   "metric": 0.3184,
   "ci_low": 0.1776,
   "ci_high": 0.4593,
   "secondary": {
    "precision": 0.9831,
    "recall": 0.7073,
    "coverage": 0.6977
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.0004,
     "usd_per_1k_correct": 0.61,
     "median_latency_s": 2.6
    }
   ],
   "note": ""
  },
  {
   "vendor": "MillionVerifier",
   "n": 172,
   "metric": 0.1699,
   "ci_low": 0.0231,
   "ci_high": 0.3168,
   "secondary": {
    "precision": 1.0,
    "recall": 0.5366,
    "coverage": 0.6279
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.00111767,
     "usd_per_1k_correct": 1.9,
     "median_latency_s": 1.2
    }
   ],
   "note": ""
  },
  {
   "vendor": "Dropleads",
   "n": 172,
   "metric": 0.1699,
   "ci_low": 0.0231,
   "ci_high": 0.3168,
   "secondary": {
    "precision": 0.9778,
    "recall": 0.5366,
    "coverage": 0.6337
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.0011407,
     "usd_per_1k_correct": 1.94,
     "median_latency_s": 1.7
    }
   ],
   "note": ""
  },
  {
   "vendor": "DeBounce",
   "n": 172,
   "metric": 0.0911,
   "ci_low": -0.0578,
   "ci_high": 0.2399,
   "secondary": {
    "precision": 1.0,
    "recall": 0.5244,
    "coverage": 0.5814
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Clay",
     "usd_per_call": 0.0534,
     "usd_per_1k_correct": 97.71,
     "median_latency_s": null
    }
   ],
   "note": ""
  },
  {
   "vendor": "Sixtyfour",
   "n": 172,
   "metric": 0.0645,
   "ci_low": -0.0839,
   "ci_high": 0.2129,
   "secondary": {
    "precision": 1.0,
    "recall": 0.4756,
    "coverage": 0.564
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.0252907,
     "usd_per_1k_correct": 47.28,
     "median_latency_s": 1.9
    }
   ],
   "note": ""
  },
  {
   "vendor": "IPQS",
   "n": 172,
   "metric": 0.0545,
   "ci_low": -0.0945,
   "ci_high": 0.2034,
   "secondary": {
    "precision": 0.9524,
    "recall": 0.4878,
    "coverage": 0.5698
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "DeepLine",
     "usd_per_call": 0.007,
     "usd_per_1k_correct": 13.23,
     "median_latency_s": 2.7
    }
   ],
   "note": ""
  },
  {
   "vendor": "Findymail",
   "n": 172,
   "metric": -0.0444,
   "ci_low": -0.087,
   "ci_high": -0.0019,
   "secondary": {
    "precision": null,
    "recall": 0.0,
    "coverage": 0.564
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "DeepLine",
     "usd_per_call": 0.028,
     "usd_per_1k_correct": 56.0,
     "median_latency_s": 2.6
    }
   ],
   "note": "its \"not verified\" answer counts as unknown, not dead"
  },
  {
   "vendor": "Icypeas",
   "n": 172,
   "metric": -0.5667,
   "ci_low": -0.669,
   "ci_high": -0.4643,
   "secondary": {
    "precision": null,
    "recall": 0.0,
    "coverage": 0.25
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Treg",
     "usd_per_call": 0.0019,
     "usd_per_1k_correct": 8.38,
     "median_latency_s": null
    },
    {
     "store": "Clay",
     "usd_per_call": 0.00502273,
     "usd_per_1k_correct": 17.28,
     "median_latency_s": null
    }
   ],
   "note": ""
  },
  {
   "vendor": "Snov.io",
   "n": 172,
   "metric": -0.7778,
   "ci_low": -0.8637,
   "ci_high": -0.6919,
   "secondary": {
    "precision": null,
    "recall": 0.0,
    "coverage": 0.1512
   },
   "tie_group": 2,
   "stores": [
    {
     "store": "Clay",
     "usd_per_call": 0.02825233,
     "usd_per_1k_correct": 242.97,
     "median_latency_s": null
    }
   ],
   "note": ""
  }
 ],
 "not_measured": [
  {
   "vendor": "Kickbox",
   "reason": "DeepLine's own Kickbox account blocked every call for capacity, refused at $0."
  },
  {
   "vendor": "ContactOut",
   "reason": "Treg's shared ContactOut account is rate-limited by ContactOut. 97 of 172 rows stay unanswered, even at one call at a time under the published 150 a minute."
  }
 ],
 "tests": "Holm-corrected exact paired McNemar against the leader, alpha 0.05, over the rows both answered; a correct row is a decided, right verdict. report.py's printed table tests J with a paired normal interval instead.",
 "caveats": [
  "The key comes from one company's records, skewed to learning and development buyers: 172 addresses, 82 of them dead.",
  "The sample is about half dead by design, not the real-world mix: J and the two per-class rates do not depend on that mix, coverage does.",
  "Not rows, because neither is a vendor: the free MX lookup scores J +0.10 [+0.03, +0.16] and Treg's router +0.43 [+0.30, +0.55].",
  "A verdict the vendor leaves unknown \u2014 catch-all, unrecognised, refused \u2014 counts as neither caught nor kept, so a verifier that abstains everywhere scores -1.",
  "Findymail's 'not verified' answer counts as unknown, not dead."
 ]
}