← All the evidencesentinel-interruption-heldout-summary.json
SHA-256 c46fd5d45dd79f2aadb4fceb4b141d21551426325df46e1d58f4c9f2522fa019 · download the original
{
"stage": "heldout",
"worlds_per_condition": 24,
"events_per_world": 3200,
"audit_assumption": "3.5% independent opt-in retrospective outcome score, noisy and delayed; unavailable by default in real world",
"identifiability": {
"identical_observed_logs": true,
"unobserved_counterfactual_a": 7,
"unobserved_counterfactual_b": -7,
"consequence": "No method using only these logs can identify the unobserved utility of interrupting."
},
"metrics": {
"biased_audit": {
"silent": {
"worlds": 24,
"situations": 76800,
"recall_pct": 0.0,
"precision_pct": null,
"false_per_1000": 0.0,
"alerts_per_1000": 0.0,
"rare_recall_pct": 0.0,
"net_per_1000": 0.0,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 0
},
"fixed": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.17,
"precision_pct": 65.35,
"false_per_1000": 27.24,
"alerts_per_1000": 78.62,
"rare_recall_pct": 12.92,
"net_per_1000": 60.05,
"late_recall_pct": 14.06,
"late_net_per_1000": 59.52,
"audits_per_1000": 0.0,
"feedback_count": 2882,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"naive_silence_negative": {
"worlds": 24,
"situations": 76800,
"recall_pct": 2.16,
"precision_pct": 78.36,
"false_per_1000": 2.16,
"alerts_per_1000": 9.99,
"rare_recall_pct": 1.82,
"net_per_1000": 20.01,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 390,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"bandit_explore": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.3,
"precision_pct": 64.85,
"false_per_1000": 28.1,
"alerts_per_1000": 79.95,
"rare_recall_pct": 14.38,
"net_per_1000": 54.81,
"late_recall_pct": 13.96,
"late_net_per_1000": 50.25,
"audits_per_1000": 0.0,
"feedback_count": 2872,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_only": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.83,
"precision_pct": 69.49,
"false_per_1000": 23.61,
"alerts_per_1000": 77.37,
"rare_recall_pct": 12.01,
"net_per_1000": 78.01,
"late_recall_pct": 15.06,
"late_net_per_1000": 75.53,
"audits_per_1000": 15.35,
"feedback_count": 2855,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"prior_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 16.36,
"precision_pct": 80.17,
"false_per_1000": 14.67,
"alerts_per_1000": 74.0,
"rare_recall_pct": 11.1,
"net_per_1000": 130.92,
"late_recall_pct": 16.15,
"late_net_per_1000": 126.32,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"feedback_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.32,
"precision_pct": 75.71,
"false_per_1000": 16.65,
"alerts_per_1000": 68.57,
"rare_recall_pct": 13.47,
"net_per_1000": 131.67,
"late_recall_pct": 14.36,
"late_net_per_1000": 146.51,
"audits_per_1000": 0.0,
"feedback_count": 2641,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.28,
"precision_pct": 74.11,
"false_per_1000": 16.82,
"alerts_per_1000": 64.97,
"rare_recall_pct": 11.28,
"net_per_1000": 117.37,
"late_recall_pct": 13.44,
"late_net_per_1000": 129.64,
"audits_per_1000": 15.35,
"feedback_count": 2476,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_stratified": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.66,
"precision_pct": 69.73,
"false_per_1000": 23.07,
"alerts_per_1000": 76.22,
"rare_recall_pct": 17.93,
"net_per_1000": 84.69,
"late_recall_pct": 14.82,
"late_net_per_1000": 89.63,
"audits_per_1000": 33.22,
"feedback_count": 2787,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_plus_probe": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.68,
"precision_pct": 68.73,
"false_per_1000": 24.22,
"alerts_per_1000": 77.46,
"rare_recall_pct": 12.19,
"net_per_1000": 75.71,
"late_recall_pct": 14.95,
"late_net_per_1000": 77.62,
"audits_per_1000": 15.35,
"feedback_count": 2861,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"oracle": {
"worlds": 24,
"situations": 76800,
"recall_pct": 22.06,
"precision_pct": 100.0,
"false_per_1000": 0.0,
"alerts_per_1000": 80.0,
"rare_recall_pct": 24.39,
"net_per_1000": 166.46,
"late_recall_pct": 22.03,
"late_net_per_1000": 166.95,
"audits_per_1000": 0.0,
"feedback_count": 3610,
"permission_violations": 0,
"max_alerts_in_100": 8
}
},
"delayed": {
"silent": {
"worlds": 24,
"situations": 76800,
"recall_pct": 0.0,
"precision_pct": null,
"false_per_1000": 0.0,
"alerts_per_1000": 0.0,
"rare_recall_pct": 0.0,
"net_per_1000": 0.0,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 0
},
"fixed": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.27,
"precision_pct": 66.18,
"false_per_1000": 26.59,
"alerts_per_1000": 78.61,
"rare_recall_pct": 14.65,
"net_per_1000": 67.58,
"late_recall_pct": 14.27,
"late_net_per_1000": 67.91,
"audits_per_1000": 0.0,
"feedback_count": 1481,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"naive_silence_negative": {
"worlds": 24,
"situations": 76800,
"recall_pct": 1.57,
"precision_pct": 81.48,
"false_per_1000": 1.3,
"alerts_per_1000": 7.03,
"rare_recall_pct": 1.36,
"net_per_1000": 13.81,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 136,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"bandit_explore": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.24,
"precision_pct": 64.97,
"false_per_1000": 27.98,
"alerts_per_1000": 79.88,
"rare_recall_pct": 12.65,
"net_per_1000": 53.73,
"late_recall_pct": 13.98,
"late_net_per_1000": 46.34,
"audits_per_1000": 0.0,
"feedback_count": 1423,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_only": {
"worlds": 24,
"situations": 76800,
"recall_pct": 15.04,
"precision_pct": 72.07,
"false_per_1000": 21.25,
"alerts_per_1000": 76.08,
"rare_recall_pct": 15.65,
"net_per_1000": 94.82,
"late_recall_pct": 15.26,
"late_net_per_1000": 95.94,
"audits_per_1000": 34.67,
"feedback_count": 1444,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"prior_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 16.38,
"precision_pct": 80.79,
"false_per_1000": 14.19,
"alerts_per_1000": 73.88,
"rare_recall_pct": 13.47,
"net_per_1000": 137.75,
"late_recall_pct": 16.31,
"late_net_per_1000": 140.84,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"feedback_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 15.28,
"precision_pct": 77.32,
"false_per_1000": 16.34,
"alerts_per_1000": 72.04,
"rare_recall_pct": 13.19,
"net_per_1000": 138.07,
"late_recall_pct": 14.48,
"late_net_per_1000": 146.16,
"audits_per_1000": 0.0,
"feedback_count": 1408,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.16,
"precision_pct": 75.73,
"false_per_1000": 15.38,
"alerts_per_1000": 63.36,
"rare_recall_pct": 16.65,
"net_per_1000": 112.64,
"late_recall_pct": 13.41,
"late_net_per_1000": 125.26,
"audits_per_1000": 34.67,
"feedback_count": 1180,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_stratified": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.88,
"precision_pct": 71.21,
"false_per_1000": 21.93,
"alerts_per_1000": 76.16,
"rare_recall_pct": 17.38,
"net_per_1000": 89.59,
"late_recall_pct": 15.19,
"late_net_per_1000": 88.32,
"audits_per_1000": 32.47,
"feedback_count": 1386,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_plus_probe": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.9,
"precision_pct": 70.75,
"false_per_1000": 22.45,
"alerts_per_1000": 76.74,
"rare_recall_pct": 15.1,
"net_per_1000": 88.41,
"late_recall_pct": 15.16,
"late_net_per_1000": 89.55,
"audits_per_1000": 34.67,
"feedback_count": 1450,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"oracle": {
"worlds": 24,
"situations": 76800,
"recall_pct": 21.95,
"precision_pct": 100.0,
"false_per_1000": 0.0,
"alerts_per_1000": 80.0,
"rare_recall_pct": 20.93,
"net_per_1000": 162.22,
"late_recall_pct": 22.05,
"late_net_per_1000": 154.84,
"audits_per_1000": 0.0,
"feedback_count": 1808,
"permission_violations": 0,
"max_alerts_in_100": 8
}
},
"drift": {
"silent": {
"worlds": 24,
"situations": 76800,
"recall_pct": 0.0,
"precision_pct": null,
"false_per_1000": 0.0,
"alerts_per_1000": 0.0,
"rare_recall_pct": 0.0,
"net_per_1000": 0.0,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 0
},
"fixed": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.57,
"precision_pct": 59.86,
"false_per_1000": 31.61,
"alerts_per_1000": 78.76,
"rare_recall_pct": 13.42,
"net_per_1000": 48.83,
"late_recall_pct": 14.56,
"late_net_per_1000": 30.78,
"audits_per_1000": 0.0,
"feedback_count": 2739,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"naive_silence_negative": {
"worlds": 24,
"situations": 76800,
"recall_pct": 2.46,
"precision_pct": 81.55,
"false_per_1000": 1.8,
"alerts_per_1000": 9.74,
"rare_recall_pct": 1.18,
"net_per_1000": 20.26,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 406,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"bandit_explore": {
"worlds": 24,
"situations": 76800,
"recall_pct": 15.37,
"precision_pct": 62.3,
"false_per_1000": 30.09,
"alerts_per_1000": 79.82,
"rare_recall_pct": 12.15,
"net_per_1000": 43.7,
"late_recall_pct": 17.06,
"late_net_per_1000": 42.89,
"audits_per_1000": 0.0,
"feedback_count": 2714,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_only": {
"worlds": 24,
"situations": 76800,
"recall_pct": 15.31,
"precision_pct": 69.12,
"false_per_1000": 22.12,
"alerts_per_1000": 71.64,
"rare_recall_pct": 11.06,
"net_per_1000": 75.32,
"late_recall_pct": 16.82,
"late_net_per_1000": 86.48,
"audits_per_1000": 35.66,
"feedback_count": 2612,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"prior_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 17.73,
"precision_pct": 77.39,
"false_per_1000": 16.76,
"alerts_per_1000": 74.13,
"rare_recall_pct": 11.15,
"net_per_1000": 111.54,
"late_recall_pct": 18.83,
"late_net_per_1000": 90.22,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"feedback_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 15.09,
"precision_pct": 74.07,
"false_per_1000": 17.08,
"alerts_per_1000": 65.89,
"rare_recall_pct": 12.96,
"net_per_1000": 109.1,
"late_recall_pct": 16.57,
"late_net_per_1000": 111.67,
"audits_per_1000": 0.0,
"feedback_count": 2490,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.23,
"precision_pct": 71.87,
"false_per_1000": 16.74,
"alerts_per_1000": 59.53,
"rare_recall_pct": 10.52,
"net_per_1000": 86.82,
"late_recall_pct": 14.51,
"late_net_per_1000": 95.44,
"audits_per_1000": 35.66,
"feedback_count": 2224,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_stratified": {
"worlds": 24,
"situations": 76800,
"recall_pct": 15.98,
"precision_pct": 69.27,
"false_per_1000": 22.93,
"alerts_per_1000": 74.62,
"rare_recall_pct": 17.86,
"net_per_1000": 81.66,
"late_recall_pct": 18.13,
"late_net_per_1000": 93.43,
"audits_per_1000": 32.34,
"feedback_count": 2717,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_plus_probe": {
"worlds": 24,
"situations": 76800,
"recall_pct": 15.22,
"precision_pct": 67.17,
"false_per_1000": 24.06,
"alerts_per_1000": 73.29,
"rare_recall_pct": 12.33,
"net_per_1000": 70.09,
"late_recall_pct": 16.81,
"late_net_per_1000": 83.7,
"audits_per_1000": 35.66,
"feedback_count": 2639,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"oracle": {
"worlds": 24,
"situations": 76800,
"recall_pct": 24.73,
"precision_pct": 100.0,
"false_per_1000": 0.0,
"alerts_per_1000": 80.0,
"rare_recall_pct": 25.84,
"net_per_1000": 160.35,
"late_recall_pct": 27.82,
"late_net_per_1000": 157.4,
"audits_per_1000": 0.0,
"feedback_count": 3607,
"permission_violations": 0,
"max_alerts_in_100": 8
}
},
"miscalibrated": {
"silent": {
"worlds": 24,
"situations": 76800,
"recall_pct": 0.0,
"precision_pct": null,
"false_per_1000": 0.0,
"alerts_per_1000": 0.0,
"rare_recall_pct": 0.0,
"net_per_1000": 0.0,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 0
},
"fixed": {
"worlds": 24,
"situations": 76800,
"recall_pct": 12.48,
"precision_pct": 56.69,
"false_per_1000": 34.21,
"alerts_per_1000": 78.98,
"rare_recall_pct": 12.28,
"net_per_1000": 18.69,
"late_recall_pct": 10.67,
"late_net_per_1000": -19.13,
"audits_per_1000": 0.0,
"feedback_count": 2745,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"naive_silence_negative": {
"worlds": 24,
"situations": 76800,
"recall_pct": 2.42,
"precision_pct": 80.24,
"false_per_1000": 2.14,
"alerts_per_1000": 10.81,
"rare_recall_pct": 5.35,
"net_per_1000": 22.67,
"late_recall_pct": 0.17,
"late_net_per_1000": 2.89,
"audits_per_1000": 0.0,
"feedback_count": 428,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"bandit_explore": {
"worlds": 24,
"situations": 76800,
"recall_pct": 12.79,
"precision_pct": 58.76,
"false_per_1000": 32.2,
"alerts_per_1000": 78.07,
"rare_recall_pct": 15.96,
"net_per_1000": 24.47,
"late_recall_pct": 11.5,
"late_net_per_1000": 7.15,
"audits_per_1000": 0.0,
"feedback_count": 2585,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_only": {
"worlds": 24,
"situations": 76800,
"recall_pct": 12.99,
"precision_pct": 63.84,
"false_per_1000": 26.41,
"alerts_per_1000": 73.02,
"rare_recall_pct": 17.46,
"net_per_1000": 49.5,
"late_recall_pct": 11.36,
"late_net_per_1000": 13.61,
"audits_per_1000": 35.6,
"feedback_count": 2552,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"prior_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.9,
"precision_pct": 71.6,
"false_per_1000": 21.2,
"alerts_per_1000": 74.64,
"rare_recall_pct": 12.11,
"net_per_1000": 81.93,
"late_recall_pct": 13.33,
"late_net_per_1000": 36.56,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"feedback_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 11.91,
"precision_pct": 67.94,
"false_per_1000": 20.17,
"alerts_per_1000": 62.9,
"rare_recall_pct": 15.79,
"net_per_1000": 74.8,
"late_recall_pct": 10.56,
"late_net_per_1000": 37.42,
"audits_per_1000": 0.0,
"feedback_count": 2284,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 10.91,
"precision_pct": 65.52,
"false_per_1000": 20.6,
"alerts_per_1000": 59.74,
"rare_recall_pct": 18.95,
"net_per_1000": 59.7,
"late_recall_pct": 9.74,
"late_net_per_1000": 23.87,
"audits_per_1000": 35.6,
"feedback_count": 2110,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_stratified": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.17,
"precision_pct": 63.83,
"false_per_1000": 26.76,
"alerts_per_1000": 73.98,
"rare_recall_pct": 23.6,
"net_per_1000": 51.86,
"late_recall_pct": 11.29,
"late_net_per_1000": 20.34,
"audits_per_1000": 32.81,
"feedback_count": 2612,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_plus_probe": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.0,
"precision_pct": 62.51,
"false_per_1000": 27.97,
"alerts_per_1000": 74.6,
"rare_recall_pct": 15.96,
"net_per_1000": 43.75,
"late_recall_pct": 11.53,
"late_net_per_1000": 10.46,
"audits_per_1000": 35.6,
"feedback_count": 2582,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"oracle": {
"worlds": 24,
"situations": 76800,
"recall_pct": 22.3,
"precision_pct": 100.0,
"false_per_1000": 0.0,
"alerts_per_1000": 80.0,
"rare_recall_pct": 21.75,
"net_per_1000": 162.48,
"late_recall_pct": 22.35,
"late_net_per_1000": 160.66,
"audits_per_1000": 0.0,
"feedback_count": 3563,
"permission_violations": 0,
"max_alerts_in_100": 8
}
},
"ordinary": {
"silent": {
"worlds": 24,
"situations": 76800,
"recall_pct": 0.0,
"precision_pct": null,
"false_per_1000": 0.0,
"alerts_per_1000": 0.0,
"rare_recall_pct": 0.0,
"net_per_1000": 0.0,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 0
},
"fixed": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.13,
"precision_pct": 65.38,
"false_per_1000": 27.3,
"alerts_per_1000": 78.88,
"rare_recall_pct": 12.11,
"net_per_1000": 62.17,
"late_recall_pct": 13.97,
"late_net_per_1000": 65.13,
"audits_per_1000": 0.0,
"feedback_count": 2908,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"naive_silence_negative": {
"worlds": 24,
"situations": 76800,
"recall_pct": 2.13,
"precision_pct": 82.66,
"false_per_1000": 1.63,
"alerts_per_1000": 9.39,
"rare_recall_pct": 5.08,
"net_per_1000": 22.04,
"late_recall_pct": 0.13,
"late_net_per_1000": 1.9,
"audits_per_1000": 0.0,
"feedback_count": 363,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"bandit_explore": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.13,
"precision_pct": 64.83,
"false_per_1000": 27.99,
"alerts_per_1000": 79.6,
"rare_recall_pct": 10.86,
"net_per_1000": 50.34,
"late_recall_pct": 14.49,
"late_net_per_1000": 56.71,
"audits_per_1000": 0.0,
"feedback_count": 2818,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_only": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.37,
"precision_pct": 70.85,
"false_per_1000": 21.59,
"alerts_per_1000": 74.05,
"rare_recall_pct": 14.16,
"net_per_1000": 83.25,
"late_recall_pct": 15.29,
"late_net_per_1000": 94.14,
"audits_per_1000": 35.55,
"feedback_count": 2805,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"prior_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 16.35,
"precision_pct": 81.24,
"false_per_1000": 13.78,
"alerts_per_1000": 73.45,
"rare_recall_pct": 10.6,
"net_per_1000": 132.29,
"late_recall_pct": 16.34,
"late_net_per_1000": 141.51,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"feedback_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.12,
"precision_pct": 76.19,
"false_per_1000": 16.11,
"alerts_per_1000": 67.64,
"rare_recall_pct": 8.99,
"net_per_1000": 126.38,
"late_recall_pct": 14.62,
"late_net_per_1000": 144.21,
"audits_per_1000": 0.0,
"feedback_count": 2638,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 12.91,
"precision_pct": 74.89,
"false_per_1000": 15.81,
"alerts_per_1000": 62.96,
"rare_recall_pct": 14.78,
"net_per_1000": 109.07,
"late_recall_pct": 14.45,
"late_net_per_1000": 137.96,
"audits_per_1000": 35.55,
"feedback_count": 2461,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_stratified": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.73,
"precision_pct": 70.58,
"false_per_1000": 22.42,
"alerts_per_1000": 76.21,
"rare_recall_pct": 17.01,
"net_per_1000": 86.96,
"late_recall_pct": 15.2,
"late_net_per_1000": 95.39,
"audits_per_1000": 32.33,
"feedback_count": 2840,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_plus_probe": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.32,
"precision_pct": 69.05,
"false_per_1000": 23.44,
"alerts_per_1000": 75.73,
"rare_recall_pct": 14.34,
"net_per_1000": 77.84,
"late_recall_pct": 15.03,
"late_net_per_1000": 88.82,
"audits_per_1000": 35.55,
"feedback_count": 2833,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"oracle": {
"worlds": 24,
"situations": 76800,
"recall_pct": 21.91,
"precision_pct": 100.0,
"false_per_1000": 0.0,
"alerts_per_1000": 80.0,
"rare_recall_pct": 20.39,
"net_per_1000": 161.41,
"late_recall_pct": 21.99,
"late_net_per_1000": 161.16,
"audits_per_1000": 0.0,
"feedback_count": 3618,
"permission_violations": 0,
"max_alerts_in_100": 8
}
},
"personal": {
"silent": {
"worlds": 24,
"situations": 76800,
"recall_pct": 0.0,
"precision_pct": null,
"false_per_1000": 0.0,
"alerts_per_1000": 0.0,
"rare_recall_pct": 0.0,
"net_per_1000": 0.0,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 0
},
"fixed": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.3,
"precision_pct": 47.71,
"false_per_1000": 41.17,
"alerts_per_1000": 78.74,
"rare_recall_pct": 13.69,
"net_per_1000": 30.41,
"late_recall_pct": 14.29,
"late_net_per_1000": 27.66,
"audits_per_1000": 0.0,
"feedback_count": 2560,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"naive_silence_negative": {
"worlds": 24,
"situations": 76800,
"recall_pct": 5.99,
"precision_pct": 66.39,
"false_per_1000": 7.97,
"alerts_per_1000": 23.71,
"rare_recall_pct": 6.06,
"net_per_1000": 85.33,
"late_recall_pct": 4.94,
"late_net_per_1000": 81.15,
"audits_per_1000": 0.0,
"feedback_count": 853,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"bandit_explore": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.07,
"precision_pct": 47.62,
"false_per_1000": 37.79,
"alerts_per_1000": 72.14,
"rare_recall_pct": 10.85,
"net_per_1000": 31.97,
"late_recall_pct": 13.21,
"late_net_per_1000": 39.14,
"audits_per_1000": 0.0,
"feedback_count": 2176,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_only": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.74,
"precision_pct": 59.07,
"false_per_1000": 25.01,
"alerts_per_1000": 61.11,
"rare_recall_pct": 9.78,
"net_per_1000": 90.09,
"late_recall_pct": 14.72,
"late_net_per_1000": 93.43,
"audits_per_1000": 34.84,
"feedback_count": 2077,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"prior_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.87,
"precision_pct": 53.16,
"false_per_1000": 34.43,
"alerts_per_1000": 73.5,
"rare_recall_pct": 12.12,
"net_per_1000": 89.11,
"late_recall_pct": 15.38,
"late_net_per_1000": 103.41,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"feedback_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.56,
"precision_pct": 64.52,
"false_per_1000": 21.04,
"alerts_per_1000": 59.31,
"rare_recall_pct": 12.51,
"net_per_1000": 173.3,
"late_recall_pct": 14.44,
"late_net_per_1000": 210.1,
"audits_per_1000": 0.0,
"feedback_count": 2031,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 12.4,
"precision_pct": 65.04,
"false_per_1000": 17.51,
"alerts_per_1000": 50.09,
"rare_recall_pct": 9.58,
"net_per_1000": 145.58,
"late_recall_pct": 13.84,
"late_net_per_1000": 184.44,
"audits_per_1000": 34.84,
"feedback_count": 1781,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_stratified": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.06,
"precision_pct": 57.58,
"false_per_1000": 27.23,
"alerts_per_1000": 64.18,
"rare_recall_pct": 17.4,
"net_per_1000": 93.86,
"late_recall_pct": 14.48,
"late_net_per_1000": 97.04,
"audits_per_1000": 32.28,
"feedback_count": 2140,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_plus_probe": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.77,
"precision_pct": 57.28,
"false_per_1000": 26.99,
"alerts_per_1000": 63.18,
"rare_recall_pct": 11.63,
"net_per_1000": 84.19,
"late_recall_pct": 14.79,
"late_net_per_1000": 87.31,
"audits_per_1000": 34.84,
"feedback_count": 2088,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"oracle": {
"worlds": 24,
"situations": 76800,
"recall_pct": 28.96,
"precision_pct": 100.0,
"false_per_1000": 0.0,
"alerts_per_1000": 76.11,
"rare_recall_pct": 38.22,
"net_per_1000": 248.46,
"late_recall_pct": 28.69,
"late_net_per_1000": 254.97,
"audits_per_1000": 0.0,
"feedback_count": 3406,
"permission_violations": 0,
"max_alerts_in_100": 8
}
},
"personal_drift": {
"silent": {
"worlds": 24,
"situations": 76800,
"recall_pct": 0.0,
"precision_pct": null,
"false_per_1000": 0.0,
"alerts_per_1000": 0.0,
"rare_recall_pct": 0.0,
"net_per_1000": 0.0,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 0
},
"fixed": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.17,
"precision_pct": 52.44,
"false_per_1000": 37.43,
"alerts_per_1000": 78.71,
"rare_recall_pct": 14.72,
"net_per_1000": 56.88,
"late_recall_pct": 14.58,
"late_net_per_1000": 69.88,
"audits_per_1000": 0.0,
"feedback_count": 2651,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"naive_silence_negative": {
"worlds": 24,
"situations": 76800,
"recall_pct": 5.71,
"precision_pct": 62.7,
"false_per_1000": 9.88,
"alerts_per_1000": 26.5,
"rare_recall_pct": 4.45,
"net_per_1000": 82.84,
"late_recall_pct": 2.71,
"late_net_per_1000": 50.76,
"audits_per_1000": 0.0,
"feedback_count": 918,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"bandit_explore": {
"worlds": 24,
"situations": 76800,
"recall_pct": 12.71,
"precision_pct": 48.4,
"false_per_1000": 39.47,
"alerts_per_1000": 76.48,
"rare_recall_pct": 10.87,
"net_per_1000": 34.76,
"late_recall_pct": 12.77,
"late_net_per_1000": 40.82,
"audits_per_1000": 0.0,
"feedback_count": 2355,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_only": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.97,
"precision_pct": 57.74,
"false_per_1000": 29.78,
"alerts_per_1000": 70.47,
"rare_recall_pct": 10.87,
"net_per_1000": 100.49,
"late_recall_pct": 13.59,
"late_net_per_1000": 97.78,
"audits_per_1000": 34.58,
"feedback_count": 2430,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"prior_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.69,
"precision_pct": 57.49,
"false_per_1000": 31.63,
"alerts_per_1000": 74.4,
"rare_recall_pct": 15.12,
"net_per_1000": 127.01,
"late_recall_pct": 15.45,
"late_net_per_1000": 156.26,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"feedback_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.23,
"precision_pct": 62.1,
"false_per_1000": 23.5,
"alerts_per_1000": 62.02,
"rare_recall_pct": 12.94,
"net_per_1000": 147.43,
"late_recall_pct": 12.25,
"late_net_per_1000": 148.42,
"audits_per_1000": 0.0,
"feedback_count": 2192,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 12.76,
"precision_pct": 64.09,
"false_per_1000": 20.82,
"alerts_per_1000": 57.98,
"rare_recall_pct": 11.46,
"net_per_1000": 146.45,
"late_recall_pct": 13.01,
"late_net_per_1000": 169.37,
"audits_per_1000": 34.58,
"feedback_count": 2072,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_stratified": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.81,
"precision_pct": 56.69,
"false_per_1000": 30.73,
"alerts_per_1000": 70.95,
"rare_recall_pct": 16.5,
"net_per_1000": 91.9,
"late_recall_pct": 13.84,
"late_net_per_1000": 98.0,
"audits_per_1000": 31.45,
"feedback_count": 2386,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_plus_probe": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.75,
"precision_pct": 55.89,
"false_per_1000": 31.61,
"alerts_per_1000": 71.67,
"rare_recall_pct": 10.97,
"net_per_1000": 90.58,
"late_recall_pct": 13.55,
"late_net_per_1000": 85.78,
"audits_per_1000": 34.58,
"feedback_count": 2413,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"oracle": {
"worlds": 24,
"situations": 76800,
"recall_pct": 26.86,
"precision_pct": 100.0,
"false_per_1000": 0.0,
"alerts_per_1000": 78.22,
"rare_recall_pct": 32.11,
"net_per_1000": 284.35,
"late_recall_pct": 25.17,
"late_net_per_1000": 249.34,
"audits_per_1000": 0.0,
"feedback_count": 3513,
"permission_violations": 0,
"max_alerts_in_100": 8
}
},
"rare": {
"silent": {
"worlds": 24,
"situations": 76800,
"recall_pct": 0.0,
"precision_pct": null,
"false_per_1000": 0.0,
"alerts_per_1000": 0.0,
"rare_recall_pct": 0.0,
"net_per_1000": 0.0,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 0
},
"fixed": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.41,
"precision_pct": 66.07,
"false_per_1000": 26.81,
"alerts_per_1000": 79.02,
"rare_recall_pct": 15.21,
"net_per_1000": 61.12,
"late_recall_pct": 14.84,
"late_net_per_1000": 63.22,
"audits_per_1000": 0.0,
"feedback_count": 2973,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"naive_silence_negative": {
"worlds": 24,
"situations": 76800,
"recall_pct": 2.43,
"precision_pct": 80.26,
"false_per_1000": 2.16,
"alerts_per_1000": 10.95,
"rare_recall_pct": 2.3,
"net_per_1000": 23.43,
"late_recall_pct": 0.0,
"late_net_per_1000": 0.0,
"audits_per_1000": 0.0,
"feedback_count": 471,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"bandit_explore": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.31,
"precision_pct": 64.89,
"false_per_1000": 28.06,
"alerts_per_1000": 79.91,
"rare_recall_pct": 10.37,
"net_per_1000": 51.94,
"late_recall_pct": 14.31,
"late_net_per_1000": 53.16,
"audits_per_1000": 0.0,
"feedback_count": 2891,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_only": {
"worlds": 24,
"situations": 76800,
"recall_pct": 15.29,
"precision_pct": 71.31,
"false_per_1000": 22.3,
"alerts_per_1000": 77.73,
"rare_recall_pct": 10.83,
"net_per_1000": 87.95,
"late_recall_pct": 15.56,
"late_net_per_1000": 94.78,
"audits_per_1000": 36.16,
"feedback_count": 2887,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"prior_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 16.68,
"precision_pct": 80.9,
"false_per_1000": 14.27,
"alerts_per_1000": 74.71,
"rare_recall_pct": 11.98,
"net_per_1000": 129.92,
"late_recall_pct": 16.61,
"late_net_per_1000": 131.64,
"audits_per_1000": 0.0,
"feedback_count": 0,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"feedback_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.49,
"precision_pct": 76.13,
"false_per_1000": 16.47,
"alerts_per_1000": 69.0,
"rare_recall_pct": 5.07,
"net_per_1000": 130.1,
"late_recall_pct": 13.62,
"late_net_per_1000": 132.77,
"audits_per_1000": 0.0,
"feedback_count": 2688,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_paced": {
"worlds": 24,
"situations": 76800,
"recall_pct": 13.27,
"precision_pct": 75.17,
"false_per_1000": 15.89,
"alerts_per_1000": 63.97,
"rare_recall_pct": 7.37,
"net_per_1000": 109.75,
"late_recall_pct": 13.22,
"late_net_per_1000": 118.84,
"audits_per_1000": 36.16,
"feedback_count": 2480,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_stratified": {
"worlds": 24,
"situations": 76800,
"recall_pct": 14.78,
"precision_pct": 70.16,
"false_per_1000": 22.77,
"alerts_per_1000": 76.33,
"rare_recall_pct": 13.59,
"net_per_1000": 80.04,
"late_recall_pct": 15.14,
"late_net_per_1000": 79.03,
"audits_per_1000": 31.16,
"feedback_count": 2862,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"audit_plus_probe": {
"worlds": 24,
"situations": 76800,
"recall_pct": 15.13,
"precision_pct": 70.09,
"false_per_1000": 23.4,
"alerts_per_1000": 78.23,
"rare_recall_pct": 10.83,
"net_per_1000": 81.68,
"late_recall_pct": 15.13,
"late_net_per_1000": 85.23,
"audits_per_1000": 36.16,
"feedback_count": 2900,
"permission_violations": 0,
"max_alerts_in_100": 8
},
"oracle": {
"worlds": 24,
"situations": 76800,
"recall_pct": 22.07,
"precision_pct": 100.0,
"false_per_1000": 0.0,
"alerts_per_1000": 80.0,
"rare_recall_pct": 20.51,
"net_per_1000": 162.21,
"late_recall_pct": 22.05,
"late_net_per_1000": 165.32,
"audits_per_1000": 0.0,
"feedback_count": 3673,
"permission_violations": 0,
"max_alerts_in_100": 8
}
}
}
}