{
 "assumptions": {
  "analyst_minutes_per_alert": 10,
  "n_days": 60,
  "wave1_day": 25,
  "wave2_day": 45,
  "synthetic": true
 },
 "control_blocks": [
  {
   "control": "DEVICE_TRUST",
   "events_blocked": 5771,
   "attacker_events_blocked": 687,
   "legit_events_blocked": 5084,
   "attacker_share_of_blocks": 0.11904349332871253
  },
  {
   "control": "MFA",
   "events_blocked": 2575,
   "attacker_events_blocked": 426,
   "legit_events_blocked": 2149,
   "attacker_share_of_blocks": 0.1654368932038835
  },
  {
   "control": "LOCKOUT",
   "events_blocked": 193,
   "attacker_events_blocked": 192,
   "legit_events_blocked": 1,
   "attacker_share_of_blocks": 0.9948186528497409
  }
 ],
 "compromises": {
  "total_user_days": 465,
  "by_type": {
   "credential_stuffing": 448,
   "session_hijack": 14,
   "mfa_fatigue": 1,
   "password_spray": 2
  },
  "observed_share": 0.44086021505376344
 },
 "mfa_did": {
  "truth": {
   "w1_pre": 7.9934259674286565,
   "w1_post": 1.7102137767220902,
   "w2_pre": 10.006949270326615,
   "w2_post": 16.140412771211853,
   "naive_w1": -6.283212190706566,
   "did": -12.416675691591804,
   "ci95_did": [
    -15.781258375347104,
    -8.948913467526966
   ]
  },
  "observed": {
   "w1_pre": 2.689376961004034,
   "w1_post": 1.2351543942992873,
   "w2_pre": 3.3356497567755388,
   "w2_post": 8.908096666078674,
   "naive_w1": -1.4542225667047466,
   "did": -7.026669476007882,
   "ci95_did": [
    -9.60943157256009,
    -4.849503013121833
   ]
  },
  "placebo_pre_period": {
   "w1_pre": 5.490315693152357,
   "w1_post": 10.396837018597159,
   "w2_pre": 8.364048766657216,
   "w2_post": 11.586695747001091,
   "naive_w1": 4.906521325444802,
   "did": 1.6838743451009277,
   "ci95_did": [
    -2.785608182615308,
    6.1948579189294035
   ]
  },
  "population_rate_per_1000_user_days": {
   "before": 9.036578341013826,
   "after_wave1_only_window": 9.193614782966655,
   "after_both_waves": 0.7822371983873879
  },
  "weekly_series": [
   {
    "week": 0,
    "w1_user_days": 3677,
    "w2_user_days": 3959,
    "w1_compromises": 18,
    "w1_rate": 4.895295077508838,
    "w2_compromises": 36,
    "w2_rate": 9.093205354887598
   },
   {
    "week": 1,
    "w1_user_days": 3666,
    "w2_user_days": 3938,
    "w1_compromises": 26,
    "w1_rate": 7.092198581560283,
    "w2_compromises": 35,
    "w2_rate": 8.88776028440833
   },
   {
    "week": 2,
    "w1_user_days": 3738,
    "w2_user_days": 4014,
    "w1_compromises": 32,
    "w1_rate": 8.560727661851258,
    "w2_compromises": 35,
    "w2_rate": 8.719481813652218
   },
   {
    "week": 3,
    "w1_user_days": 3677,
    "w2_user_days": 3978,
    "w1_compromises": 36,
    "w1_rate": 9.790590155017677,
    "w2_compromises": 77,
    "w2_rate": 19.35646053293112
   },
   {
    "week": 4,
    "w1_user_days": 3738,
    "w2_user_days": 4025,
    "w1_compromises": 4,
    "w1_rate": 1.0700909577314073,
    "w2_compromises": 75,
    "w2_rate": 18.633540372670808
   },
   {
    "week": 5,
    "w1_user_days": 3687,
    "w2_user_days": 3951,
    "w1_compromises": 4,
    "w1_rate": 1.0848928668294007,
    "w2_compromises": 60,
    "w2_rate": 15.186028853454822
   },
   {
    "week": 6,
    "w1_user_days": 3669,
    "w2_user_days": 3967,
    "w1_compromises": 7,
    "w1_rate": 1.9078768056691195,
    "w2_compromises": 10,
    "w2_rate": 2.5207965717166623
   },
   {
    "week": 7,
    "w1_user_days": 3757,
    "w2_user_days": 4028,
    "w1_compromises": 2,
    "w1_rate": 0.5323396326856534,
    "w2_compromises": 2,
    "w2_rate": 0.49652432969215493
   },
   {
    "week": 8,
    "w1_user_days": 2307,
    "w2_user_days": 2482,
    "w1_compromises": 4,
    "w1_rate": 1.7338534893801474,
    "w2_compromises": 2,
    "w2_rate": 0.8058017727639001
   }
  ],
  "generator_counterfactual": {
   "compromises_averted_total_enforced": 271,
   "of_valid_cred_attempts_under_enforcement": 319,
   "would_have_succeeded_without_mfa": 293,
   "averted_in_window_all_users": 159,
   "true_effect_wave1_per_1000_user_days_in_window": -15.106888361045131
  }
 },
 "friction_weekly": [
  {
   "week": 0,
   "mfa_successes": 4499,
   "latency_hours": 12.224490277777777,
   "legit_mfa_failures": 697
  },
  {
   "week": 1,
   "mfa_successes": 4978,
   "latency_hours": 13.482762777777777,
   "legit_mfa_failures": 186
  },
  {
   "week": 2,
   "mfa_successes": 5013,
   "latency_hours": 13.647520555555555,
   "legit_mfa_failures": 137
  },
  {
   "week": 3,
   "mfa_successes": 5832,
   "latency_hours": 15.821160555555556,
   "legit_mfa_failures": 142
  },
  {
   "week": 4,
   "mfa_successes": 7914,
   "latency_hours": 21.350557777777777,
   "legit_mfa_failures": 162
  },
  {
   "week": 5,
   "mfa_successes": 7925,
   "latency_hours": 21.512014166666667,
   "legit_mfa_failures": 182
  },
  {
   "week": 6,
   "mfa_successes": 9461,
   "latency_hours": 25.699164166666666,
   "legit_mfa_failures": 199
  },
  {
   "week": 7,
   "mfa_successes": 10998,
   "latency_hours": 29.769613055555556,
   "legit_mfa_failures": 249
  },
  {
   "week": 8,
   "mfa_successes": 8143,
   "latency_hours": 22.101557777777778,
   "legit_mfa_failures": 163
  }
 ],
 "friction": {
  "mfa_challenges_succeeded": 64763,
  "user_hours_of_added_latency": 175.6088411111111,
  "legit_mfa_failures": 2117,
  "avg_seconds_per_successful_challenge": 9.761620493182836,
  "compromises_averted_generator_truth": 271,
  "user_hours_of_latency_per_compromise_averted": 0.6480031037310373,
  "legit_mfa_failures_per_compromise_averted": 7.8118081180811805,
  "caveat": "MFA latency is only recorded for MFA-required successes; time lost to failed/re-tried challenges is not modelled beyond counts."
 },
 "detection": {
  "split": {
   "train_days": [
    0,
    39
   ],
   "test_days": [
    40,
    59
   ]
  },
  "test": {
   "user_days": 21867,
   "compromised_user_days": 39,
   "compromised_observed_user_days": 33,
   "attack_touched_user_days": 1491,
   "benign_oddity_user_days": 511
  },
  "train": {
   "user_days": 44391,
   "compromised_user_days": 430,
   "compromised_observed_user_days": 172
  },
  "rules": {
   "impossible_travel": {
    "description": "success from home country and abroad within 2h",
    "vs_truth": {
     "alerts": 200,
     "tp": 15,
     "fp": 185,
     "fn": 24,
     "precision": 0.075,
     "recall": 0.38461538461538464,
     "precision_wilson95": [
      0.045974507057252416,
      0.12004459797332034
     ],
     "recall_wilson95": [
      0.24891162547007065,
      0.5410121868992154
     ]
    },
    "vs_observed_labels": {
     "alerts": 200,
     "tp": 13,
     "fp": 187,
     "fn": 20,
     "precision": 0.065,
     "recall": 0.3939393939393939,
     "precision_wilson95": [
      0.03837598764975976,
      0.10802003749917945
     ],
     "recall_wilson95": [
      0.24682882231508207,
      0.5631685719348962
     ]
    },
    "vs_attack_touched": {
     "alerts": 200,
     "tp": 22,
     "fp": 178,
     "fn": 1469,
     "precision": 0.11,
     "recall": 0.014755197853789403,
     "precision_wilson95": [
      0.07377190160716048,
      0.16092798300912986
     ],
     "recall_wilson95": [
      0.009764039395083786,
      0.02224042181338138
     ]
    },
    "alerts_per_day": 10.0,
    "benign_oddity_fp": 2,
    "analyst_hours_per_day": 1.6666666666666667
   },
   "foreign_success_badip": {
    "description": "success from a foreign country via proxy/hosting IP class",
    "vs_truth": {
     "alerts": 12,
     "tp": 12,
     "fp": 0,
     "fn": 27,
     "precision": 1.0,
     "recall": 0.3076923076923077,
     "precision_wilson95": [
      0.7574992425007574,
      1.0
     ],
     "recall_wilson95": [
      0.1856575344598306,
      0.464215486155646
     ]
    },
    "vs_observed_labels": {
     "alerts": 12,
     "tp": 11,
     "fp": 1,
     "fn": 22,
     "precision": 0.9166666666666666,
     "recall": 0.3333333333333333,
     "precision_wilson95": [
      0.6461140782014047,
      0.9851352905492264
     ],
     "recall_wilson95": [
      0.19750029467803698,
      0.5039241820005003
     ]
    },
    "vs_attack_touched": {
     "alerts": 12,
     "tp": 12,
     "fp": 0,
     "fn": 1479,
     "precision": 1.0,
     "recall": 0.008048289738430584,
     "precision_wilson95": [
      0.7574992425007574,
      1.0
     ],
     "recall_wilson95": [
      0.004609873680393157,
      0.014015243522660334
     ]
    },
    "alerts_per_day": 0.6,
    "benign_oddity_fp": 0,
    "analyst_hours_per_day": 0.1
   },
   "spray_target": {
    "description": "failure from an IP (not corp NAT) that failed >=20 distinct users in that hour",
    "vs_truth": {
     "alerts": 988,
     "tp": 2,
     "fp": 986,
     "fn": 37,
     "precision": 0.0020242914979757085,
     "recall": 0.05128205128205128,
     "precision_wilson95": [
      0.0005552977592657264,
      0.007350803376238169
     ],
     "recall_wilson95": [
      0.014177657646399527,
      0.1688593904563791
     ]
    },
    "vs_observed_labels": {
     "alerts": 988,
     "tp": 2,
     "fp": 986,
     "fn": 31,
     "precision": 0.0020242914979757085,
     "recall": 0.06060606060606061,
     "precision_wilson95": [
      0.0005552977592657264,
      0.007350803376238169
     ],
     "recall_wilson95": [
      0.016780530429070947,
      0.19606581717798194
     ]
    },
    "vs_attack_touched": {
     "alerts": 988,
     "tp": 988,
     "fp": 0,
     "fn": 503,
     "precision": 1.0,
     "recall": 0.6626425217974514,
     "precision_wilson95": [
      0.9961268008924006,
      1.0
     ],
     "recall_wilson95": [
      0.6382522667460249,
      0.6861968253852084
     ]
    },
    "alerts_per_day": 49.4,
    "benign_oddity_fp": 11,
    "analyst_hours_per_day": 8.233333333333333
   },
   "fail_then_success": {
    "description": ">=5 failures then a success from the same IP within 30 min",
    "vs_truth": {
     "alerts": 1,
     "tp": 1,
     "fp": 0,
     "fn": 38,
     "precision": 1.0,
     "recall": 0.02564102564102564,
     "precision_wilson95": [
      0.20654329147389294,
      1.0
     ],
     "recall_wilson95": [
      0.00454058606523737,
      0.13181286478627147
     ]
    },
    "vs_observed_labels": {
     "alerts": 1,
     "tp": 1,
     "fp": 0,
     "fn": 32,
     "precision": 1.0,
     "recall": 0.030303030303030304,
     "precision_wilson95": [
      0.20654329147389294,
      1.0
     ],
     "recall_wilson95": [
      0.005369272089651725,
      0.15319061673168066
     ]
    },
    "vs_attack_touched": {
     "alerts": 1,
     "tp": 1,
     "fp": 0,
     "fn": 1490,
     "precision": 1.0,
     "recall": 0.0006706908115358819,
     "precision_wilson95": [
      0.20654329147389294,
      1.0
     ],
     "recall_wilson95": [
      0.00011840016367674301,
      0.003789438633423899
     ]
    },
    "alerts_per_day": 0.05,
    "benign_oddity_fp": 0,
    "analyst_hours_per_day": 0.008333333333333333
   },
   "mfa_fatigue": {
    "description": ">=5 denied MFA prompts on one user-day",
    "vs_truth": {
     "alerts": 5,
     "tp": 1,
     "fp": 4,
     "fn": 38,
     "precision": 0.2,
     "recall": 0.02564102564102564,
     "precision_wilson95": [
      0.03622316096978745,
      0.6244717358814613
     ],
     "recall_wilson95": [
      0.00454058606523737,
      0.13181286478627147
     ]
    },
    "vs_observed_labels": {
     "alerts": 5,
     "tp": 1,
     "fp": 4,
     "fn": 32,
     "precision": 0.2,
     "recall": 0.030303030303030304,
     "precision_wilson95": [
      0.03622316096978745,
      0.6244717358814613
     ],
     "recall_wilson95": [
      0.005369272089651725,
      0.15319061673168066
     ]
    },
    "vs_attack_touched": {
     "alerts": 5,
     "tp": 5,
     "fp": 0,
     "fn": 1486,
     "precision": 1.0,
     "recall": 0.00335345405767941,
     "precision_wilson95": [
      0.5655085052479191,
      1.0
     ],
     "recall_wilson95": [
      0.0014331986244058522,
      0.007826377774859462
     ]
    },
    "alerts_per_day": 0.25,
    "benign_oddity_fp": 0,
    "analyst_hours_per_day": 0.041666666666666664
   },
   "host_fanout": {
    "description": ">=3 first-seen hosts accessed in one day",
    "vs_truth": {
     "alerts": 109,
     "tp": 22,
     "fp": 87,
     "fn": 17,
     "precision": 0.2018348623853211,
     "recall": 0.5641025641025641,
     "precision_wilson95": [
      0.13723625011246088,
      0.28673505125157506
     ],
     "recall_wilson95": [
      0.40975423474522105,
      0.7069547583829534
     ]
    },
    "vs_observed_labels": {
     "alerts": 109,
     "tp": 19,
     "fp": 90,
     "fn": 14,
     "precision": 0.1743119266055046,
     "recall": 0.5757575757575758,
     "precision_wilson95": [
      0.11452781593206733,
      0.2562716055578796
     ],
     "recall_wilson95": [
      0.4080697369146003,
      0.7276464100497011
     ]
    },
    "vs_attack_touched": {
     "alerts": 109,
     "tp": 29,
     "fp": 80,
     "fn": 1462,
     "precision": 0.26605504587155965,
     "recall": 0.019450033534540577,
     "precision_wilson95": [
      0.1920974014994754,
      0.3559416195707682
     ],
     "recall_wilson95": [
      0.013576016075834965,
      0.027793985936418374
     ]
    },
    "alerts_per_day": 5.45,
    "benign_oddity_fp": 77,
    "analyst_hours_per_day": 0.9083333333333333
   },
   "spray_target_incl_nat_UNTUNED": {
    "description": "same as spray_target but counting corporate-NAT IPs (v0, before the Monday-rush false positives were seen)",
    "vs_truth": {
     "alerts": 1299,
     "tp": 2,
     "fp": 1297,
     "fn": 37,
     "precision": 0.001539645881447267,
     "recall": 0.05128205128205128,
     "precision_wilson95": [
      0.0004223191840078762,
      0.00559652454956649
     ],
     "recall_wilson95": [
      0.014177657646399527,
      0.1688593904563791
     ]
    },
    "vs_observed_labels": {
     "alerts": 1299,
     "tp": 2,
     "fp": 1297,
     "fn": 31,
     "precision": 0.001539645881447267,
     "recall": 0.06060606060606061,
     "precision_wilson95": [
      0.0004223191840078762,
      0.00559652454956649
     ],
     "recall_wilson95": [
      0.016780530429070947,
      0.19606581717798194
     ]
    },
    "vs_attack_touched": {
     "alerts": 1299,
     "tp": 995,
     "fp": 304,
     "fn": 496,
     "precision": 0.7659738260200154,
     "recall": 0.6673373574782026,
     "precision_wilson95": [
      0.7421856402701598,
      0.7881934940774079
     ],
     "recall_wilson95": [
      0.6430180069740508,
      0.6907966259609706
     ]
    },
    "alerts_per_day": 64.95,
    "benign_oddity_fp": 321,
    "analyst_hours_per_day": 10.825
   },
   "ANY_RULE": {
    "description": "union of the six tuned rules",
    "vs_truth": {
     "alerts": 1276,
     "tp": 27,
     "fp": 1249,
     "fn": 12,
     "precision": 0.02115987460815047,
     "recall": 0.6923076923076923,
     "precision_wilson95": [
      0.014582446905903412,
      0.030611895894018082
     ],
     "recall_wilson95": [
      0.535784513844354,
      0.8143424655401694
     ]
    },
    "vs_observed_labels": {
     "alerts": 1276,
     "tp": 23,
     "fp": 1253,
     "fn": 10,
     "precision": 0.018025078369905956,
     "recall": 0.696969696969697,
     "precision_wilson95": [
      0.012040621619937473,
      0.02690294764675928
     ],
     "recall_wilson95": [
      0.5266154602526555,
      0.8262465218545277
     ]
    },
    "vs_attack_touched": {
     "alerts": 1276,
     "tp": 1018,
     "fp": 258,
     "fn": 473,
     "precision": 0.7978056426332288,
     "recall": 0.6827632461435278,
     "precision_wilson95": [
      0.7748891177886543,
      0.8189343678676957
     ],
     "recall_wilson95": [
      0.6586957715625892,
      0.7058913525848789
     ]
    },
    "alerts_per_day": 63.8,
    "benign_oddity_fp": 84,
    "analyst_hours_per_day": 10.633333333333333
   }
  },
  "features": [
   "n",
   "n_fail",
   "fail_rate",
   "n_foreign",
   "n_badclass",
   "n_succ_foreign",
   "n_succ_foreign_badclass",
   "n_unmanaged_succ",
   "n_mfa_denied",
   "n_mfa_notdone",
   "n_locked",
   "n_devblock",
   "n_badpw",
   "n_ips",
   "n_ctry",
   "n_new_hosts",
   "ip_fanin_nonat",
   "fail_then_success",
   "impossible_travel"
  ],
  "logreg": {
   "trained_on_observed_labels": {
    "average_precision_vs_truth": 0.9422470341930569,
    "average_precision_vs_observed": 0.8447864033319283,
    "base_rate_truth": 0.0017835093977225957,
    "budget_5_per_day": {
     "vs_truth": {
      "alerts": 100,
      "tp": 37,
      "fp": 63,
      "fn": 2,
      "precision": 0.37,
      "recall": 0.9487179487179487,
      "precision_wilson95": [
       0.28182212569509163,
       0.46779652424866885
      ],
      "recall_wilson95": [
       0.8311406095436208,
       0.9858223423536004
      ]
     },
     "vs_observed_labels": {
      "alerts": 100,
      "tp": 31,
      "fp": 69,
      "fn": 2,
      "precision": 0.31,
      "recall": 0.9393939393939394,
      "precision_wilson95": [
       0.22779563206132378,
       0.40626239477955695
      ],
      "recall_wilson95": [
       0.8039341828220181,
       0.9832194695709291
      ]
     },
     "vs_attack_touched": {
      "alerts": 100,
      "tp": 95,
      "fp": 5,
      "fn": 1396,
      "precision": 0.95,
      "recall": 0.06371562709590879,
      "precision_wilson95": [
       0.8882480347279118,
       0.9784566385436864
      ],
      "recall_wilson95": [
       0.05240433093697295,
       0.07726934151099749
      ]
     },
     "alerts_per_day": 5.0,
     "benign_oddity_fp": 3,
     "analyst_hours_per_day": 0.8333333333333334
    },
    "budget_10_per_day": {
     "vs_truth": {
      "alerts": 200,
      "tp": 37,
      "fp": 163,
      "fn": 2,
      "precision": 0.185,
      "recall": 0.9487179487179487,
      "precision_wilson95": [
       0.13730116322736063,
       0.2445718205011816
      ],
      "recall_wilson95": [
       0.8311406095436208,
       0.9858223423536004
      ]
     },
     "vs_observed_labels": {
      "alerts": 200,
      "tp": 31,
      "fp": 169,
      "fn": 2,
      "precision": 0.155,
      "recall": 0.9393939393939394,
      "precision_wilson95": [
       0.11139568514638545,
       0.211608058937256
      ],
      "recall_wilson95": [
       0.8039341828220181,
       0.9832194695709291
      ]
     },
     "vs_attack_touched": {
      "alerts": 200,
      "tp": 182,
      "fp": 18,
      "fn": 1309,
      "precision": 0.91,
      "recall": 0.12206572769953052,
      "precision_wilson95": [
       0.8622332538615105,
       0.9423130212854665
      ],
      "recall_wilson95": [
       0.10641323748210157,
       0.1396607278129504
      ]
     },
     "alerts_per_day": 10.0,
     "benign_oddity_fp": 6,
     "analyst_hours_per_day": 1.6666666666666667
    },
    "budget_20_per_day": {
     "vs_truth": {
      "alerts": 400,
      "tp": 38,
      "fp": 362,
      "fn": 1,
      "precision": 0.095,
      "recall": 0.9743589743589743,
      "precision_wilson95": [
       0.0699962171700798,
       0.12770902171566156
      ],
      "recall_wilson95": [
       0.8681871352137286,
       0.9954594139347627
      ]
     },
     "vs_observed_labels": {
      "alerts": 400,
      "tp": 32,
      "fp": 368,
      "fn": 1,
      "precision": 0.08,
      "recall": 0.9696969696969697,
      "precision_wilson95": [
       0.057235397942962354,
       0.11075522016076943
      ],
      "recall_wilson95": [
       0.8468093832683193,
       0.9946307279103482
      ]
     },
     "vs_attack_touched": {
      "alerts": 400,
      "tp": 341,
      "fp": 59,
      "fn": 1150,
      "precision": 0.8525,
      "recall": 0.22870556673373574,
      "precision_wilson95": [
       0.8143991454822668,
       0.8838944428949584
      ],
      "recall_wilson95": [
       0.2080998215946697,
       0.25070571338642794
      ]
     },
     "alerts_per_day": 20.0,
     "benign_oddity_fp": 8,
     "analyst_hours_per_day": 3.3333333333333335
    },
    "coefficients_standardised": {
     "n": -0.085,
     "n_fail": -0.303,
     "fail_rate": 0.199,
     "n_foreign": -0.027,
     "n_badclass": 0.722,
     "n_succ_foreign": -0.075,
     "n_succ_foreign_badclass": -0.079,
     "n_unmanaged_succ": 0.444,
     "n_mfa_denied": -0.214,
     "n_mfa_notdone": -0.149,
     "n_locked": -0.335,
     "n_devblock": 0.349,
     "n_badpw": -0.269,
     "n_ips": 0.218,
     "n_ctry": 0.805,
     "n_new_hosts": 0.19,
     "ip_fanin_nonat": -0.219,
     "fail_then_success": 0.0,
     "impossible_travel": -0.025
    }
   },
   "trained_on_true_labels": {
    "average_precision_vs_truth": 0.9117705904069677,
    "average_precision_vs_observed": 0.8247438499577363,
    "base_rate_truth": 0.0017835093977225957,
    "budget_5_per_day": {
     "vs_truth": {
      "alerts": 100,
      "tp": 38,
      "fp": 62,
      "fn": 1,
      "precision": 0.38,
      "recall": 0.9743589743589743,
      "precision_wilson95": [
       0.2909744920013438,
       0.47790426179289663
      ],
      "recall_wilson95": [
       0.8681871352137286,
       0.9954594139347627
      ]
     },
     "vs_observed_labels": {
      "alerts": 100,
      "tp": 32,
      "fp": 68,
      "fn": 1,
      "precision": 0.32,
      "recall": 0.9696969696969697,
      "precision_wilson95": [
       0.2366901081545688,
       0.4166280225367919
      ],
      "recall_wilson95": [
       0.8468093832683193,
       0.9946307279103482
      ]
     },
     "vs_attack_touched": {
      "alerts": 100,
      "tp": 91,
      "fp": 9,
      "fn": 1400,
      "precision": 0.91,
      "recall": 0.06103286384976526,
      "precision_wilson95": [
       0.8377362530335634,
       0.9519280048361151
      ],
      "recall_wilson95": [
       0.04997292730028549,
       0.07434900754552025
      ]
     },
     "alerts_per_day": 5.0,
     "benign_oddity_fp": 5,
     "analyst_hours_per_day": 0.8333333333333334
    },
    "budget_10_per_day": {
     "vs_truth": {
      "alerts": 200,
      "tp": 38,
      "fp": 162,
      "fn": 1,
      "precision": 0.19,
      "recall": 0.9743589743589743,
      "precision_wilson95": [
       0.14167093792456506,
       0.250013585744794
      ],
      "recall_wilson95": [
       0.8681871352137286,
       0.9954594139347627
      ]
     },
     "vs_observed_labels": {
      "alerts": 200,
      "tp": 32,
      "fp": 168,
      "fn": 1,
      "precision": 0.16,
      "recall": 0.9696969696969697,
      "precision_wilson95": [
       0.11567342208544587,
       0.2171418619390124
      ],
      "recall_wilson95": [
       0.8468093832683193,
       0.9946307279103482
      ]
     },
     "vs_attack_touched": {
      "alerts": 200,
      "tp": 160,
      "fp": 40,
      "fn": 1331,
      "precision": 0.8,
      "recall": 0.10731052984574112,
      "precision_wilson95": [
       0.7391436038882958,
       0.8495487925607118
      ],
      "recall_wilson95": [
       0.09259703356167455,
       0.12404237492147176
      ]
     },
     "alerts_per_day": 10.0,
     "benign_oddity_fp": 9,
     "analyst_hours_per_day": 1.6666666666666667
    },
    "budget_20_per_day": {
     "vs_truth": {
      "alerts": 400,
      "tp": 38,
      "fp": 362,
      "fn": 1,
      "precision": 0.095,
      "recall": 0.9743589743589743,
      "precision_wilson95": [
       0.0699962171700798,
       0.12770902171566156
      ],
      "recall_wilson95": [
       0.8681871352137286,
       0.9954594139347627
      ]
     },
     "vs_observed_labels": {
      "alerts": 400,
      "tp": 32,
      "fp": 368,
      "fn": 1,
      "precision": 0.08,
      "recall": 0.9696969696969697,
      "precision_wilson95": [
       0.057235397942962354,
       0.11075522016076943
      ],
      "recall_wilson95": [
       0.8468093832683193,
       0.9946307279103482
      ]
     },
     "vs_attack_touched": {
      "alerts": 400,
      "tp": 267,
      "fp": 133,
      "fn": 1224,
      "precision": 0.6675,
      "recall": 0.1790744466800805,
      "precision_wilson95": [
       0.6199304437921472,
       0.7118828216069598
      ],
      "recall_wilson95": [
       0.16044479662780758,
       0.1993535927132437
      ]
     },
     "alerts_per_day": 20.0,
     "benign_oddity_fp": 22,
     "analyst_hours_per_day": 3.3333333333333335
    },
    "coefficients_standardised": {
     "n": -0.13,
     "n_fail": -0.341,
     "fail_rate": -0.031,
     "n_foreign": 0.071,
     "n_badclass": 1.205,
     "n_succ_foreign": -0.164,
     "n_succ_foreign_badclass": 1.063,
     "n_unmanaged_succ": 0.414,
     "n_mfa_denied": -0.366,
     "n_mfa_notdone": -0.206,
     "n_locked": -0.564,
     "n_devblock": 0.686,
     "n_badpw": -0.497,
     "n_ips": 0.361,
     "n_ctry": 0.764,
     "n_new_hosts": 0.432,
     "ip_fanin_nonat": -0.366,
     "fail_then_success": 0.0,
     "impossible_travel": 0.02
    }
   }
  },
  "rules_count_score_average_precision_vs_truth": 0.384684303552261,
  "prevalence_note": "Base rate is ~0.5% of user-days; precision is only interpretable next to it.",
  "by_attack_type_test": {
   "credential_stuffing": {
    "n": 31,
    "caught_by_rules": 21,
    "caught_by_logreg_budget10": 31
   },
   "session_hijack": {
    "n": 7,
    "caught_by_rules": 5,
    "caught_by_logreg_budget10": 5
   },
   "mfa_fatigue": {
    "n": 1,
    "caught_by_rules": 1,
    "caught_by_logreg_budget10": 1
   }
  }
 }
}