{
 "study": "formula-round1",
 "run_id": "20261030T025631Z",
 "date": "2026-10-30",
 "prereg": {
  "path": "PREREG.txt",
  "sha256": "e86e7bdf7f96dd06bf882c52147252681c20cae0c67b6b6d53fae37c2e664f35",
  "anchors": [
   "sidecar",
   "lock"
  ],
  "plan_sha256": "7e10c1d2720bad332feb9c416e10836895031a61c1ac8e1d74a8f9d5ed8f18ab",
  "locked_at": "2026-10-30T02:55:43+00:00"
 },
 "hashes": {
  "result.json": "8b24682e592b5283e408c57ec77f47667b8fd006a2709424b58613246c97c7c8",
  "power-snapshot.csv": "1afe9cd946cd1e2a0d41d2a2784f85fe3c961cc9f3a26b43fd9e9bf7a8e5b48f",
  "sites.json": "efcbee35d7b266126c599e96628f2dcc6f9a30b75862eefdd2bb73657a75d334",
  "0008_westhoff_ts.tsv": "b47dac880030d62bb9f25acddba4c76a8acaff9909a5f5bc5c28e6067cfecb56",
  "0018_bailon_ts.tsv": "a9f6296cea7b747577f3a5b959348237a5486c42a55b47ad529557916779fd6f",
  "0062_neubauer_ts.tsv": "1c27ae6f25d972f870c836791a9906493cab4ce395c2d70754329750c4019293",
  "0076_bayer_ts.tsv": "5923ffffa062b749dd1b0ec84b624d6c7c242b31627b07ed5b9a1efad98ae4e4",
  "fr_units.py": "7e1063e626df37c57d020d4a06c9292868c7cbbdf1e124fcdfcb414eb7ad9463",
  "fr_score.py": "4054f8903351ac4866c4691f0449d5eb6478bd0a95151ced40f9be60489c498d",
  "fr_stats.py": "4f91fde43495ef1fa43b21fb8408fb37ea333c93088f02eb9048eac1e4329312",
  "fr_pairs.py": "9e3c5b10212b639af382d4fdfc29314fb75e918cb3a296ec62e6dd7026e84743",
  "fr_c1.py": "c3147168328a60b01440a2677fc1626e8e060092781385d7330be24944c6100f",
  "fr_c2.py": "d4a5810ce3c9f50ec4594606468a21b812a02ef4f95bb1ad3839162ef2905c1b",
  "fr_c3.py": "98c5096f53229ed09b2ef0991de696f5b60cf7a822e588c5798f97bad93d92c5",
  "fr_c4.py": "20c967518ddb7c5e39349f64b67bbe81e9b3bd999caa478293be0ca0da8fc1dc",
  "fr_c5.py": "ddd75d70ef2da85d2854968d1be74b056f9d27f20fb13f9d05562f43974185fe",
  "smoke.py": "761ade02b4d1ed916146fa303605178fa88760e90043d204f853d94fdad20f64",
  "make_spec.py": "f64156ab46e5d5a603b86db07b1d4263575041c464e22050d00d0d92ec3791f1",
  "prereg_text.py": "e02b57ca8f2e931119875859e9fa94524f1428d15ee1d9cef39434ed8f094002",
  "uw_lens.py": "b1c58df8aab37eaeac237d3218503f3be4c404f5e44246c0e5f7e204593660eb",
  "uw_game.py": "12697f7eddec273f0de71eb1613952969f19a0f65eebd52b822003eb915d78ed",
  "uw_bridge.py": "5e62af7f3b3cffebbf2a86f3095586112340f28e3f89b7e29e147e047131d670",
  "uw_sims.py": "0543af942ba404aa13df26997e5a8a8ca145dcbccc1d292fce89c96d485e8f5f",
  "uw_units.py": "10a96d4ffce01b7ad6018ed22857889cfd1fec598b2f2b0b67a3270975775ab9",
  "it_stats.py": "6833fe67ab776f567434eca504d4ae6af947399597a0e38800646495db6b2b6f",
  "fc_core.py": "ac2c2b430b04890d19d231ab7c0a49bbceb22577c5feaa1894062da6a5a279d6",
  "fc_analysis.py": "bd4141729f9dab1b6ba9b45e65c0788d57f81bd919ccc7c2c627c3ab3b01f9a1",
  "fc_data.py": "4894710b888e6d8aae4b661f7a840430b584d16f576f9e0bdf2815717483337f",
  "ls_spectra.py": "76ac5fecd57ddb4842a7a5d0ab9f8e6f744572aa8a664691edd0496c740fe080",
  "ls_series.py": "9cd8cb76a33e6f2ec31ce4adc5bee47fd94d001a02a5cd92e7edebd6190c8744",
  "ls_common.py": "a82b3a9980908932f9e21538775c0e3d4ad776535026558f45400b3385b80b9b",
  "twolines.py": "9875b29f5108f6209ba0d8cf2acd8805cf7d64b4b2f09a4b10befd5b0f1c61e5",
  "__init__.py": "388ead5feaeee2d8b99abdb1144ea94dd7466c0a66f51c6de29d5d0cbef1a31d",
  "sim.py": "17b1ff27f4396253c62273b1c56342be68298f3abd89bc251161bf6041a03a90",
  "strategies.py": "3260af99f91527e712943478fe060f62b1c5a5f15426cb869d47e69d66fed58b",
  "stats.py": "4eb93a09654384c14d37cc614a57fb522085c05a4d420a273199d7d415367d27"
 },
 "analysis": {
  "c5": {
   "tol": 0.059746794344808965,
   "configs": [
    {
     "n": 100,
     "p0": 0.1,
     "m": 20,
     "old": 0.1735,
     "holm": 0.045,
     "bonferroni": 0.045,
     "bh": 0.0465
    },
    {
     "n": 100,
     "p0": 0.1,
     "m": 200,
     "old": 0.8655,
     "holm": 0.019,
     "bonferroni": 0.019,
     "bh": 0.0195
    },
    {
     "n": 100,
     "p0": 0.1,
     "m": 1000,
     "old": 1.0,
     "holm": 0.034,
     "bonferroni": 0.034,
     "bh": 0.034
    },
    {
     "n": 100,
     "p0": 0.25,
     "m": 20,
     "old": 0.276,
     "holm": 0.022,
     "bonferroni": 0.022,
     "bh": 0.022
    },
    {
     "n": 100,
     "p0": 0.25,
     "m": 200,
     "old": 0.9605,
     "holm": 0.026,
     "bonferroni": 0.026,
     "bh": 0.027
    },
    {
     "n": 100,
     "p0": 0.25,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0245,
     "bonferroni": 0.0245,
     "bh": 0.025
    },
    {
     "n": 100,
     "p0": 0.5,
     "m": 20,
     "old": 0.42,
     "holm": 0.0295,
     "bonferroni": 0.0295,
     "bh": 0.0295
    },
    {
     "n": 100,
     "p0": 0.5,
     "m": 200,
     "old": 0.9965,
     "holm": 0.038,
     "bonferroni": 0.038,
     "bh": 0.0395
    },
    {
     "n": 100,
     "p0": 0.5,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0425,
     "bonferroni": 0.0425,
     "bh": 0.0445
    },
    {
     "n": 100,
     "p0": 0.8,
     "m": 20,
     "old": 0.6105,
     "holm": 0.043,
     "bonferroni": 0.043,
     "bh": 0.043
    },
    {
     "n": 100,
     "p0": 0.8,
     "m": 200,
     "old": 1.0,
     "holm": 0.015,
     "bonferroni": 0.015,
     "bh": 0.017
    },
    {
     "n": 100,
     "p0": 0.8,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0185,
     "bonferroni": 0.0185,
     "bh": 0.0195
    },
    {
     "n": 100,
     "p0": 0.95,
     "m": 20,
     "old": 0.915,
     "holm": 0.0,
     "bonferroni": 0.0,
     "bh": 0.0
    },
    {
     "n": 100,
     "p0": 0.95,
     "m": 200,
     "old": 1.0,
     "holm": 0.0,
     "bonferroni": 0.0,
     "bh": 0.0
    },
    {
     "n": 100,
     "p0": 0.95,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0,
     "bonferroni": 0.0,
     "bh": 0.0
    },
    {
     "n": 500,
     "p0": 0.1,
     "m": 20,
     "old": 0.3,
     "holm": 0.036,
     "bonferroni": 0.036,
     "bh": 0.0375
    },
    {
     "n": 500,
     "p0": 0.1,
     "m": 200,
     "old": 0.9735,
     "holm": 0.036,
     "bonferroni": 0.036,
     "bh": 0.04
    },
    {
     "n": 500,
     "p0": 0.1,
     "m": 1000,
     "old": 1.0,
     "holm": 0.039,
     "bonferroni": 0.039,
     "bh": 0.039
    },
    {
     "n": 500,
     "p0": 0.25,
     "m": 20,
     "old": 0.369,
     "holm": 0.037,
     "bonferroni": 0.037,
     "bh": 0.0375
    },
    {
     "n": 500,
     "p0": 0.25,
     "m": 200,
     "old": 0.992,
     "holm": 0.0385,
     "bonferroni": 0.0385,
     "bh": 0.0395
    },
    {
     "n": 500,
     "p0": 0.25,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0485,
     "bonferroni": 0.0485,
     "bh": 0.049
    },
    {
     "n": 500,
     "p0": 0.5,
     "m": 20,
     "old": 0.4095,
     "holm": 0.0515,
     "bonferroni": 0.0515,
     "bh": 0.052
    },
    {
     "n": 500,
     "p0": 0.5,
     "m": 200,
     "old": 0.996,
     "holm": 0.0345,
     "bonferroni": 0.0345,
     "bh": 0.035
    },
    {
     "n": 500,
     "p0": 0.5,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0435,
     "bonferroni": 0.0435,
     "bh": 0.044
    },
    {
     "n": 500,
     "p0": 0.8,
     "m": 20,
     "old": 0.4705,
     "holm": 0.047,
     "bonferroni": 0.047,
     "bh": 0.0485
    },
    {
     "n": 500,
     "p0": 0.8,
     "m": 200,
     "old": 0.9975,
     "holm": 0.034,
     "bonferroni": 0.034,
     "bh": 0.0355
    },
    {
     "n": 500,
     "p0": 0.8,
     "m": 1000,
     "old": 1.0,
     "holm": 0.039,
     "bonferroni": 0.039,
     "bh": 0.04
    },
    {
     "n": 500,
     "p0": 0.95,
     "m": 20,
     "old": 0.676,
     "holm": 0.0235,
     "bonferroni": 0.0235,
     "bh": 0.025
    },
    {
     "n": 500,
     "p0": 0.95,
     "m": 200,
     "old": 1.0,
     "holm": 0.0355,
     "bonferroni": 0.0355,
     "bh": 0.0365
    },
    {
     "n": 500,
     "p0": 0.95,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0165,
     "bonferroni": 0.0165,
     "bh": 0.0165
    },
    {
     "n": 2000,
     "p0": 0.1,
     "m": 20,
     "old": 0.3545,
     "holm": 0.048,
     "bonferroni": 0.048,
     "bh": 0.05
    },
    {
     "n": 2000,
     "p0": 0.1,
     "m": 200,
     "old": 0.9865,
     "holm": 0.039,
     "bonferroni": 0.039,
     "bh": 0.0395
    },
    {
     "n": 2000,
     "p0": 0.1,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0505,
     "bonferroni": 0.0505,
     "bh": 0.0505
    },
    {
     "n": 2000,
     "p0": 0.25,
     "m": 20,
     "old": 0.3975,
     "holm": 0.0395,
     "bonferroni": 0.0395,
     "bh": 0.0425
    },
    {
     "n": 2000,
     "p0": 0.25,
     "m": 200,
     "old": 0.994,
     "holm": 0.046,
     "bonferroni": 0.046,
     "bh": 0.047
    },
    {
     "n": 2000,
     "p0": 0.25,
     "m": 1000,
     "old": 1.0,
     "holm": 0.049,
     "bonferroni": 0.049,
     "bh": 0.0505
    },
    {
     "n": 2000,
     "p0": 0.5,
     "m": 20,
     "old": 0.4195,
     "holm": 0.0455,
     "bonferroni": 0.0455,
     "bh": 0.047
    },
    {
     "n": 2000,
     "p0": 0.5,
     "m": 200,
     "old": 0.9915,
     "holm": 0.05,
     "bonferroni": 0.05,
     "bh": 0.051
    },
    {
     "n": 2000,
     "p0": 0.5,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0345,
     "bonferroni": 0.0345,
     "bh": 0.035
    },
    {
     "n": 2000,
     "p0": 0.8,
     "m": 20,
     "old": 0.436,
     "holm": 0.035,
     "bonferroni": 0.035,
     "bh": 0.036
    },
    {
     "n": 2000,
     "p0": 0.8,
     "m": 200,
     "old": 0.9985,
     "holm": 0.0395,
     "bonferroni": 0.0395,
     "bh": 0.0415
    },
    {
     "n": 2000,
     "p0": 0.8,
     "m": 1000,
     "old": 1.0,
     "holm": 0.048,
     "bonferroni": 0.048,
     "bh": 0.048
    },
    {
     "n": 2000,
     "p0": 0.95,
     "m": 20,
     "old": 0.4965,
     "holm": 0.04,
     "bonferroni": 0.04,
     "bh": 0.043
    },
    {
     "n": 2000,
     "p0": 0.95,
     "m": 200,
     "old": 0.999,
     "holm": 0.0385,
     "bonferroni": 0.0385,
     "bh": 0.0385
    },
    {
     "n": 2000,
     "p0": 0.95,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0225,
     "bonferroni": 0.0225,
     "bh": 0.0225
    }
   ],
   "mixed_null": {
    "old": 1.0,
    "holm": 0.04,
    "bonferroni": 0.04,
    "bh": 0.0415
   },
   "power": [
    {
     "n": 100,
     "p0": 0.1,
     "m": 20,
     "old": 0.978,
     "holm": 0.912,
     "bonferroni": 0.912,
     "bh": 0.916
    },
    {
     "n": 100,
     "p0": 0.1,
     "m": 200,
     "old": 0.9611999999999985,
     "holm": 0.7083999999999993,
     "bonferroni": 0.7083999999999993,
     "bh": 0.8917999999999985
    },
    {
     "n": 100,
     "p0": 0.1,
     "m": 1000,
     "old": 0.9618399999999997,
     "holm": 0.6288000000000005,
     "bonferroni": 0.6288000000000005,
     "bh": 0.8999999999999981
    },
    {
     "n": 100,
     "p0": 0.25,
     "m": 20,
     "old": 0.994,
     "holm": 0.96,
     "bonferroni": 0.96,
     "bh": 0.96
    },
    {
     "n": 100,
     "p0": 0.25,
     "m": 200,
     "old": 0.9935999999999995,
     "holm": 0.8569999999999988,
     "bonferroni": 0.8569999999999988,
     "bh": 0.9583999999999981
    },
    {
     "n": 100,
     "p0": 0.25,
     "m": 1000,
     "old": 0.9928000000000009,
     "holm": 0.7327600000000003,
     "bonferroni": 0.7327600000000003,
     "bh": 0.9478799999999985
    },
    {
     "n": 100,
     "p0": 0.5,
     "m": 20,
     "old": 1.0,
     "holm": 0.994,
     "bonferroni": 0.994,
     "bh": 0.994
    },
    {
     "n": 100,
     "p0": 0.5,
     "m": 200,
     "old": 0.9995999999999999,
     "holm": 0.9561999999999982,
     "bonferroni": 0.9561999999999982,
     "bh": 0.9901999999999994
    },
    {
     "n": 100,
     "p0": 0.5,
     "m": 1000,
     "old": 0.9998400000000001,
     "holm": 0.8979199999999983,
     "bonferroni": 0.8979199999999983,
     "bh": 0.991840000000001
    },
    {
     "n": 100,
     "p0": 0.8,
     "m": 20,
     "old": 1.0,
     "holm": 1.0,
     "bonferroni": 1.0,
     "bh": 1.0
    },
    {
     "n": 100,
     "p0": 0.8,
     "m": 200,
     "old": 1.0,
     "holm": 1.0,
     "bonferroni": 1.0,
     "bh": 1.0
    },
    {
     "n": 100,
     "p0": 0.8,
     "m": 1000,
     "old": 1.0,
     "holm": 1.0,
     "bonferroni": 1.0,
     "bh": 1.0
    },
    {
     "n": 100,
     "p0": 0.95,
     "m": 20,
     "old": 1.0,
     "holm": 0.0,
     "bonferroni": 0.0,
     "bh": 0.002
    },
    {
     "n": 100,
     "p0": 0.95,
     "m": 200,
     "old": 1.0,
     "holm": 0.0,
     "bonferroni": 0.0,
     "bh": 0.0
    },
    {
     "n": 100,
     "p0": 0.95,
     "m": 1000,
     "old": 1.0,
     "holm": 0.0,
     "bonferroni": 0.0,
     "bh": 0.0
    },
    {
     "n": 500,
     "p0": 0.1,
     "m": 20,
     "old": 0.988,
     "holm": 0.928,
     "bonferroni": 0.928,
     "bh": 0.934
    },
    {
     "n": 500,
     "p0": 0.1,
     "m": 200,
     "old": 0.989799999999999,
     "holm": 0.8397999999999992,
     "bonferroni": 0.8397999999999992,
     "bh": 0.9495999999999981
    },
    {
     "n": 500,
     "p0": 0.1,
     "m": 1000,
     "old": 0.9898400000000014,
     "holm": 0.7208800000000002,
     "bonferroni": 0.7208800000000002,
     "bh": 0.9438399999999981
    },
    {
     "n": 500,
     "p0": 0.25,
     "m": 20,
     "old": 0.996,
     "holm": 0.976,
     "bonferroni": 0.976,
     "bh": 0.976
    },
    {
     "n": 500,
     "p0": 0.25,
     "m": 200,
     "old": 0.9979999999999998,
     "holm": 0.9005999999999981,
     "bonferroni": 0.8815999999999984,
     "bh": 0.9711999999999982
    },
    {
     "n": 500,
     "p0": 0.25,
     "m": 1000,
     "old": 0.9968400000000007,
     "holm": 0.7986399999999985,
     "bonferroni": 0.7986399999999985,
     "bh": 0.97108
    },
    {
     "n": 500,
     "p0": 0.5,
     "m": 20,
     "old": 0.996,
     "holm": 0.98,
     "bonferroni": 0.98,
     "bh": 0.98
    },
    {
     "n": 500,
     "p0": 0.5,
     "m": 200,
     "old": 0.9989999999999999,
     "holm": 0.9287999999999973,
     "bonferroni": 0.9287999999999973,
     "bh": 0.9881999999999986
    },
    {
     "n": 500,
     "p0": 0.5,
     "m": 1000,
     "old": 0.9990400000000004,
     "holm": 0.866959999999998,
     "bonferroni": 0.866959999999998,
     "bh": 0.9863600000000008
    },
    {
     "n": 500,
     "p0": 0.8,
     "m": 20,
     "old": 1.0,
     "holm": 0.996,
     "bonferroni": 0.996,
     "bh": 0.996
    },
    {
     "n": 500,
     "p0": 0.8,
     "m": 200,
     "old": 1.0,
     "holm": 0.9783999999999987,
     "bonferroni": 0.9783999999999987,
     "bh": 0.9981999999999999
    },
    {
     "n": 500,
     "p0": 0.8,
     "m": 1000,
     "old": 0.9998,
     "holm": 0.944119999999999,
     "bonferroni": 0.944119999999999,
     "bh": 0.9970400000000006
    },
    {
     "n": 500,
     "p0": 0.95,
     "m": 20,
     "old": 1.0,
     "holm": 1.0,
     "bonferroni": 1.0,
     "bh": 1.0
    },
    {
     "n": 500,
     "p0": 0.95,
     "m": 200,
     "old": 1.0,
     "holm": 1.0,
     "bonferroni": 1.0,
     "bh": 1.0
    },
    {
     "n": 500,
     "p0": 0.95,
     "m": 1000,
     "old": 1.0,
     "holm": 1.0,
     "bonferroni": 1.0,
     "bh": 1.0
    },
    {
     "n": 2000,
     "p0": 0.1,
     "m": 20,
     "old": 0.996,
     "holm": 0.974,
     "bonferroni": 0.974,
     "bh": 0.974
    },
    {
     "n": 2000,
     "p0": 0.1,
     "m": 200,
     "old": 0.9949999999999998,
     "holm": 0.8879999999999977,
     "bonferroni": 0.8879999999999977,
     "bh": 0.9689999999999989
    },
    {
     "n": 2000,
     "p0": 0.1,
     "m": 1000,
     "old": 0.9954800000000015,
     "holm": 0.7968000000000002,
     "bonferroni": 0.7968000000000002,
     "bh": 0.9696799999999999
    },
    {
     "n": 2000,
     "p0": 0.25,
     "m": 20,
     "old": 0.996,
     "holm": 0.976,
     "bonferroni": 0.976,
     "bh": 0.978
    },
    {
     "n": 2000,
     "p0": 0.25,
     "m": 200,
     "old": 0.9985999999999999,
     "holm": 0.9093999999999979,
     "bonferroni": 0.9093999999999979,
     "bh": 0.9751999999999987
    },
    {
     "n": 2000,
     "p0": 0.25,
     "m": 1000,
     "old": 0.9975600000000007,
     "holm": 0.8372799999999987,
     "bonferroni": 0.8372799999999987,
     "bh": 0.978480000000001
    },
    {
     "n": 2000,
     "p0": 0.5,
     "m": 20,
     "old": 1.0,
     "holm": 0.978,
     "bonferroni": 0.978,
     "bh": 0.978
    },
    {
     "n": 2000,
     "p0": 0.5,
     "m": 200,
     "old": 0.9993999999999998,
     "holm": 0.9355999999999987,
     "bonferroni": 0.9351999999999986,
     "bh": 0.986599999999999
    },
    {
     "n": 2000,
     "p0": 0.5,
     "m": 1000,
     "old": 0.9992000000000003,
     "holm": 0.8623599999999988,
     "bonferroni": 0.8623599999999988,
     "bh": 0.9869600000000016
    },
    {
     "n": 2000,
     "p0": 0.8,
     "m": 20,
     "old": 1.0,
     "holm": 0.992,
     "bonferroni": 0.992,
     "bh": 0.992
    },
    {
     "n": 2000,
     "p0": 0.8,
     "m": 200,
     "old": 0.9994,
     "holm": 0.9585999999999986,
     "bonferroni": 0.9585999999999986,
     "bh": 0.9929999999999994
    },
    {
     "n": 2000,
     "p0": 0.8,
     "m": 1000,
     "old": 0.9995200000000002,
     "holm": 0.9015599999999979,
     "bonferroni": 0.9015599999999979,
     "bh": 0.9920000000000009
    },
    {
     "n": 2000,
     "p0": 0.95,
     "m": 20,
     "old": 1.0,
     "holm": 0.998,
     "bonferroni": 0.998,
     "bh": 0.998
    },
    {
     "n": 2000,
     "p0": 0.95,
     "m": 200,
     "old": 1.0,
     "holm": 0.9861999999999995,
     "bonferroni": 0.9861999999999995,
     "bh": 0.9982
    },
    {
     "n": 2000,
     "p0": 0.95,
     "m": 1000,
     "old": 0.99992,
     "holm": 0.9544799999999993,
     "bonferroni": 0.9544799999999993,
     "bh": 0.9979200000000005
    }
   ],
   "max_fwer_holm": 0.0515,
   "max_fwer_bh": 0.052,
   "max_fwer_bonferroni": 0.0515,
   "max_fwer_old": 1.0,
   "mean_fwer_old": 0.8105555555555555,
   "mean_power_holm": 0.8545235555555547,
   "mean_power_bh": 0.911348444444444,
   "mean_power_old": 0.9957617777777779,
   "mcnemar_b": 69865,
   "mcnemar_c": 0,
   "b_minus_c_sub": 15580.0,
   "p": 0.0
  },
  "c3": {
   "chosen": 0.0,
   "dev_score": {
    "0.0": 0.02,
    "1e-06": 0.02,
    "0.001": 0.02,
    "0.05": 0.02
   },
   "rates": {
    "u_true": {
     "old": 1.0,
     "new": 1.0,
     "new_wilson95": [
      0.9936380666717896,
      1.0000000000000002
     ]
    },
    "linear": {
     "old": 0.006666666666666667,
     "new": 0.006666666666666667,
     "new_wilson95": [
      0.0025954592764893545,
      0.017014981607345086
     ]
    },
    "concave": {
     "old": 0.011666666666666667,
     "new": 0.011666666666666667,
     "new_wilson95": [
      0.005662511553023055,
      0.02388430999752928
     ]
    },
    "floor": {
     "old": 0.37666666666666665,
     "new": 0.0,
     "new_wilson95": [
      0.0,
      0.006361933328210577
     ]
    },
    "floor_binary": {
     "old": 0.36,
     "new": 0.0,
     "new_wilson95": [
      0.0,
      0.006361933328210577
     ]
    },
    "log_trap": {
     "old": 0.0016666666666666668,
     "new": 0.0016666666666666668,
     "new_wilson95": [
      0.0002942606305477184,
      0.009379799586568826
     ]
    }
   },
   "mcnemar_b": 442,
   "mcnemar_c": 0,
   "b_minus_c": 442,
   "p": 8.805254571710335e-134,
   "n_test_per_shape": 600,
   "max_null_false_u_new": 0.011666666666666667,
   "power_old": 1.0,
   "power_new": 1.0
  },
  "c4": {
   "T": {
    "A": {
     "c": 0.515630745637003,
     "mae_old": 0.15530744451956227,
     "mae_new": 0.041949386658105846,
     "gain": {
      "mean": 0.11335805786145643,
      "lo": 0.09718770084227224,
      "hi": 0.1305058284735135
     },
     "p": 0.0001999600079984003,
     "bias_new": -0.016008754594561844,
     "pass_rule": true
    },
    "B": {
     "G0": 0.854855975247807,
     "d6_dev": 0.715212902400016,
     "C": 0.02327384547463183,
     "per_Q": {
      "3": {
       "observed": 0.9335341171190336,
       "ci95": [
        0.8761926691443468,
        1.0121714985146444
       ],
       "n_boot_with_crossing": 500,
       "predicted": 0.7850344388239114,
       "null": 0.715212902400016,
       "err_new": 0.14849967829512212,
       "err_null": 0.21832121471901755,
       "pass": false,
       "G0_test_run": 0.8472130289802441
      },
      "12": {
       "observed": 0.4464227702162991,
       "ci95": [
        0.39801884984854796,
        0.5000483645004129
       ],
       "n_boot_with_crossing": 500,
       "predicted": 0.575569829552225,
       "null": 0.715212902400016,
       "err_new": 0.1291470593359259,
       "err_null": 0.2687901321837169,
       "pass": false,
       "G0_test_run": 0.835773292294396
      }
     },
     "pass": false
    }
   },
   "R1": {
    "scored": true,
    "A": {
     "c": 0.8091744026197183,
     "mae_old": 0.15530744451956227,
     "mae_new": 0.02564164178207663,
     "gain": {
      "mean": 0.12966580273748565,
      "lo": 0.11228737265039485,
      "hi": 0.14728749041935868
     },
     "p": 0.0001999600079984003,
     "bias_new": -0.00287034411166246,
     "pass_rule": true
    },
    "B": {
     "G0": 0.854855975247807,
     "d6_dev": 0.715212902400016,
     "C": 0.025410984359578344,
     "per_Q": {
      "3": {
       "observed": 0.9335341171190336,
       "ci95": [
        0.8701235934966594,
        1.014359534098274
       ],
       "n_boot_with_crossing": 500,
       "predicted": 0.7819232847223442,
       "null": 0.715212902400016,
       "err_new": 0.15161083239668938,
       "err_null": 0.21832121471901755,
       "pass": false,
       "G0_test_run": 0.8472130289802441
      },
      "12": {
       "observed": 0.4464227702162991,
       "ci95": [
        0.40034494039875346,
        0.5019667861807346
       ],
       "n_boot_with_crossing": 500,
       "predicted": 0.5983809092650628,
       "null": 0.715212902400016,
       "err_new": 0.1519581390487637,
       "err_null": 0.2687901321837169,
       "pass": false,
       "G0_test_run": 0.835773292294396
      }
     },
     "pass": false
    }
   }
  },
  "c1": {
   "chosen": "fact|1.0|0.02",
   "dev_mae": {
    "fact|0.5|0.0": 0.029161706349206354,
    "fact|0.5|0.02": 0.02766071428571429,
    "fact|0.5|0.05": 0.028524801587301585,
    "fact|1.0|0.0": 0.02909623015873017,
    "fact|1.0|0.02": 0.027524801587301594,
    "fact|1.0|0.05": 0.028450396825396834,
    "flat|0.5|None": 0.039030753968253964,
    "flat|1.0|None": 0.03916666666666667
   },
   "n_dev": 126,
   "n_test": 168,
   "mae_old": 0.15530744451956227,
   "mae_new": 0.034621279761904766,
   "gain": {
    "mean": 0.12068616475765752,
    "lo": 0.1035377757378873,
    "hi": 0.13852815060349
   },
   "p": 0.0001999600079984003,
   "mae_old_calibrated": 0.029120617076251958,
   "mae_oracle": 0.038660714285714284,
   "rho_new": 0.9327502993053276,
   "rho_old": 0.9656814298183747,
   "bias_new": 0.022581101190476193,
   "bias_old": 0.15414077857839606,
   "per_kind_mae_new": {
    "identical": 0.104328125,
    "random": 0.0405484375,
    "scrambled": 0.021723437500000005
   },
   "per_kind_mae_old": {
    "identical": 0.27515774167536994,
    "random": 0.21224822279667416,
    "scrambled": 0.08638163652686962
   }
  },
  "c2": {
   "chosen": "M0",
   "dev": {
    "M0": {
     "rho": 0.4137010259044157,
     "viol": 0.061633281972265024
    },
    "M1": {
     "rho": 0.3747450164399317,
     "viol": 0.06317411402157165
    },
    "M2": {
     "rho": 0.38486112757299196,
     "viol": 0.05546995377503852
    },
    "M3": {
     "rho": 0.3500507903897735,
     "viol": 0.040061633281972264
    },
    "M4": {
     "rho": 0.3977430980820812,
     "viol": 0.05084745762711865
    }
   },
   "n_dev": 649,
   "n_test": 585,
   "rho_old": 0.4037230376026019,
   "rho_new": 0.4037230376026019,
   "rho_diff": 0.0,
   "rho_diff_ci": [
    0.0,
    0.0
   ],
   "p": 1.0,
   "viol_old": {
    "rate": 0.06153846153846154,
    "wilson95": [
     0.04477876409743006,
     0.0840192012637129
    ],
    "count": 36,
    "n": 585
   },
   "viol_new": {
    "rate": 0.06153846153846154,
    "wilson95": [
     0.04477876409743006,
     0.0840192012637129
    ],
    "count": 36,
    "n": 585
   },
   "surrogate_rho_old": 0.5949852608158279,
   "surrogate_rho_new": 0.5949852608158279,
   "per_group": {
    "tree_rings": {
     "rho_old": 0.3947935662660471,
     "rho_new": 0.3947935662660471
    },
    "esm_mood": {
     "rho_old": 0.0704209065679926,
     "rho_new": 0.0704209065679926
    }
   }
  },
  "bh": {
   "C1": {
    "p": 0.0001999600079984003,
    "p_bh": 0.00023995200959808037,
    "reject": true
   },
   "C2": {
    "p": 1.0,
    "p_bh": 1.0,
    "reject": false
   },
   "C3": {
    "p": 8.805254571710335e-134,
    "p_bh": 2.6415763715131004e-133,
    "reject": true
   },
   "C4": {
    "p": 0.0001999600079984003,
    "p_bh": 0.00023995200959808037,
    "reject": true
   },
   "C5": {
    "p": 0.0,
    "p_bh": 0.0,
    "reject": true
   },
   "C4R": {
    "p": 0.0001999600079984003,
    "p_bh": 0.00023995200959808037,
    "reject": true
   }
  },
  "ok": {
   "C1": 1,
   "C2": 0,
   "C3": 1,
   "C4": 0,
   "C5": 1,
   "C4R": 0
  },
  "f79_reproduction": {
   "I": 0.0,
   "oracle": 0.0,
   "learned": 0.0
  },
  "timing_s": {
   "c5": 51.0699847000069,
   "c3": 494.309840300004,
   "c1_c4": 1691.0975791999954,
   "c2": 1768.1855537999945
  },
  "test_pairs": [
   {
    "kind": "random",
    "I": 2.3862238102843705,
    "oracle": 0.80425,
    "learned": 0.66375
   },
   {
    "kind": "scrambled",
    "I": 0.19380236606989587,
    "oracle": 0.37975,
    "learned": 0.32175
   },
   {
    "kind": "random",
    "I": 2.2703197960845913,
    "oracle": 0.78775,
    "learned": 0.65175
   },
   {
    "kind": "scrambled",
    "I": 0.056755271886220544,
    "oracle": 0.31275,
    "learned": 0.29025
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.24675,
    "learned": 0.24525
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.24425,
    "learned": 0.25575
   },
   {
    "kind": "random",
    "I": 1.6224608332826365,
    "oracle": 0.63375,
    "learned": 0.62575
   },
   {
    "kind": "scrambled",
    "I": 0.16300957276850092,
    "oracle": 0.35675,
    "learned": 0.356
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.252,
    "learned": 0.248
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.24975,
    "learned": 0.251
   },
   {
    "kind": "random",
    "I": 0.5264396682416352,
    "oracle": 0.41425,
    "learned": 0.40975
   },
   {
    "kind": "scrambled",
    "I": 0.018423185580758318,
    "oracle": 0.286,
    "learned": 0.28025
   },
   {
    "kind": "random",
    "I": 0.6676928696866213,
    "oracle": 0.44975,
    "learned": 0.45175
   },
   {
    "kind": "scrambled",
    "I": 0.020010993520564145,
    "oracle": 0.29525,
    "learned": 0.27
   },
   {
    "kind": "random",
    "I": 3.5202391409310643,
    "oracle": 0.90475,
    "learned": 0.725
   },
   {
    "kind": "scrambled",
    "I": 0.47633821473986526,
    "oracle": 0.459,
    "learned": 0.306
   },
   {
    "kind": "random",
    "I": 1.6704402657316955,
    "oracle": 0.69675,
    "learned": 0.6785
   },
   {
    "kind": "scrambled",
    "I": 0.03935214286054186,
    "oracle": 0.31525,
    "learned": 0.29275
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.254,
    "learned": 0.2535
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.24675,
    "learned": 0.2435
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.24725,
    "learned": 0.253
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.25375,
    "learned": 0.2495
   },
   {
    "kind": "random",
    "I": 1.5140157366102627,
    "oracle": 0.63175,
    "learned": 0.60325
   },
   {
    "kind": "scrambled",
    "I": 0.10275883595966806,
    "oracle": 0.3435,
    "learned": 0.278
   },
   {
    "kind": "random",
    "I": 1.1658865607556719,
    "oracle": 0.58275,
    "learned": 0.5565
   },
   {
    "kind": "scrambled",
    "I": 0.10273392918590994,
    "oracle": 0.3195,
    "learned": 0.3065
   },
   {
    "kind": "random",
    "I": 1.5518769303073603,
    "oracle": 0.6535,
    "learned": 0.6365
   },
   {
    "kind": "scrambled",
    "I": 0.11107252231519693,
    "oracle": 0.3425,
    "learned": 0.3075
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.26125,
    "learned": 0.25275
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.2505,
    "learned": 0.25525
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.2545,
    "learned": 0.2575
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.24725,
    "learned": 0.243
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.2445,
    "learned": 0.25375
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.24825,
    "learned": 0.24
   },
   {
    "kind": "random",
    "I": 2.504208744428033,
    "oracle": 0.80075,
    "learned": 0.6615
   },
   {
    "kind": "scrambled",
    "I": 0.22000055225413157,
    "oracle": 0.3895,
    "learned": 0.30625
   },
   {
    "kind": "random",
    "I": 1.8891651475372715,
    "oracle": 0.721,
    "learned": 0.65175
   },
   {
    "kind": "scrambled",
    "I": 0.07381663380167555,
    "oracle": 0.335,
    "learned": 0.31825
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.24275,
    "learned": 0.2465
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.25625,
    "learned": 0.244
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.2415,
    "learned": 0.23575
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.252,
    "learned": 0.2385
   },
   {
    "kind": "random",
    "I": 1.2835201492808048,
    "oracle": 0.5915,
    "learned": 0.58525
   },
   {
    "kind": "scrambled",
    "I": 0.1209103910316627,
    "oracle": 0.354,
    "learned": 0.3035
   },
   {
    "kind": "random",
    "I": 1.0040301399337508,
    "oracle": 0.53825,
    "learned": 0.5415
   },
   {
    "kind": "scrambled",
    "I": 0.012911381604483492,
    "oracle": 0.29,
    "learned": 0.26075
   },
   {
    "kind": "random",
    "I": 1.2447419038246985,
    "oracle": 0.61475,
    "learned": 0.62025
   },
   {
    "kind": "scrambled",
    "I": 0.1051872418450905,
    "oracle": 0.34075,
    "learned": 0.34
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.26175,
    "learned": 0.246
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.24275,
    "learned": 0.2295
   },
   {
    "kind": "random",
    "I": 1.1151162557651784,
    "oracle": 0.52825,
    "learned": 0.5375
   },
   {
    "kind": "scrambled",
    "I": 0.0366036892145333,
    "oracle": 0.28375,
    "learned": 0.275
   },
   {
    "kind": "random",
    "I": 0.9035477404121023,
    "oracle": 0.529,
    "learned": 0.5225
   },
   {
    "kind": "scrambled",
    "I": 0.012235141816472745,
    "oracle": 0.28575,
    "learned": 0.27025
   },
   {
    "kind": "random",
    "I": 2.2586122592056554,
    "oracle": 0.80525,
    "learned": 0.6545
   },
   {
    "kind": "scrambled",
    "I": 0.17179895076650148,
    "oracle": 0.3745,
    "learned": 0.28475
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.241,
    "learned": 0.25325
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.2495,
    "learned": 0.248
   },
   {
    "kind": "random",
    "I": 0.6944792626181644,
    "oracle": 0.425,
    "learned": 0.42125
   },
   {
    "kind": "scrambled",
    "I": 0.0030687040368434764,
    "oracle": 0.2715,
    "learned": 0.26925
   },
   {
    "kind": "random",
    "I": 1.3935936327237042,
    "oracle": 0.58975,
    "learned": 0.58325
   },
   {
    "kind": "scrambled",
    "I": 0.07854777405256932,
    "oracle": 0.30675,
    "learned": 0.29225
   },
   {
    "kind": "random",
    "I": 2.5961509364476667,
    "oracle": 0.8195,
    "learned": 0.7
   },
   {
    "kind": "scrambled",
    "I": 0.19767426262421722,
    "oracle": 0.3915,
    "learned": 0.293
   },
   {
    "kind": "random",
    "I": 0.9737699829947115,
    "oracle": 0.552,
    "learned": 0.54575
   },
   {
    "kind": "scrambled",
    "I": 0.06109728750479315,
    "oracle": 0.3105,
    "learned": 0.309
   },
   {
    "kind": "random",
    "I": 1.112174422587977,
    "oracle": 0.5585,
    "learned": 0.543
   },
   {
    "kind": "scrambled",
    "I": 0.12189280913519672,
    "oracle": 0.338,
    "learned": 0.332
   },
   {
    "kind": "random",
    "I": 3.4925956874759834,
    "oracle": 0.8975,
    "learned": 0.6955
   },
   {
    "kind": "scrambled",
    "I": 0.2883753421877745,
    "oracle": 0.398,
    "learned": 0.32125
   },
   {
    "kind": "random",
    "I": 1.2386588245578953,
    "oracle": 0.593,
    "learned": 0.5765
   },
   {
    "kind": "scrambled",
    "I": 0.07456376478044172,
    "oracle": 0.3195,
    "learned": 0.30075
   },
   {
    "kind": "random",
    "I": 0.6201775313673732,
    "oracle": 0.40875,
    "learned": 0.398
   },
   {
    "kind": "scrambled",
    "I": 0.009395354233593313,
    "oracle": 0.271,
    "learned": 0.274
   },
   {
    "kind": "random",
    "I": 1.951300896348959,
    "oracle": 0.71125,
    "learned": 0.6615
   },
   {
    "kind": "scrambled",
    "I": 0.34208009660065475,
    "oracle": 0.4055,
    "learned": 0.366
   },
   {
    "kind": "random",
    "I": 1.9591344172594638,
    "oracle": 0.713,
    "learned": 0.703
   },
   {
    "kind": "scrambled",
    "I": 0.4152419276961644,
    "oracle": 0.446,
    "learned": 0.40525
   },
   {
    "kind": "random",
    "I": 0.8840039414048473,
    "oracle": 0.45175,
    "learned": 0.45675
   },
   {
    "kind": "scrambled",
    "I": 0.030723382041311774,
    "oracle": 0.284,
    "learned": 0.296
   },
   {
    "kind": "random",
    "I": 0.5606253319137723,
    "oracle": 0.40825,
    "learned": 0.407
   },
   {
    "kind": "scrambled",
    "I": 0.008011529151247965,
    "oracle": 0.27775,
    "learned": 0.26925
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.25425,
    "learned": 0.24225
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.24825,
    "learned": 0.25075
   },
   {
    "kind": "random",
    "I": 2.075063815482302,
    "oracle": 0.746,
    "learned": 0.7215
   },
   {
    "kind": "scrambled",
    "I": 0.07346648953055732,
    "oracle": 0.321,
    "learned": 0.30525
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.24025,
    "learned": 0.24875
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.269,
    "learned": 0.256
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.2415,
    "learned": 0.2575
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.2555,
    "learned": 0.2505
   },
   {
    "kind": "random",
    "I": 3.0757099236118335,
    "oracle": 0.85425,
    "learned": 0.731
   },
   {
    "kind": "scrambled",
    "I": 0.2000067982215308,
    "oracle": 0.37725,
    "learned": 0.316
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.24,
    "learned": 0.26025
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.2515,
    "learned": 0.24475
   },
   {
    "kind": "random",
    "I": 1.3631411281992496,
    "oracle": 0.6245,
    "learned": 0.61975
   },
   {
    "kind": "scrambled",
    "I": 0.025579451091524513,
    "oracle": 0.298,
    "learned": 0.272
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.25,
    "learned": 0.251
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.24625,
    "learned": 0.24375
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.241,
    "learned": 0.2475
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.2495,
    "learned": 0.24075
   },
   {
    "kind": "random",
    "I": 0.8945802229662374,
    "oracle": 0.5045,
    "learned": 0.4845
   },
   {
    "kind": "scrambled",
    "I": 0.03376592435428316,
    "oracle": 0.30775,
    "learned": 0.2735
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.25075,
    "learned": 0.26175
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.2575,
    "learned": 0.23975
   },
   {
    "kind": "random",
    "I": 0.8775052193083165,
    "oracle": 0.451,
    "learned": 0.45425
   },
   {
    "kind": "scrambled",
    "I": 0.06588845126362639,
    "oracle": 0.3225,
    "learned": 0.313
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.24775,
    "learned": 0.2425
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.24775,
    "learned": 0.256
   },
   {
    "kind": "random",
    "I": 1.1129439803767258,
    "oracle": 0.56775,
    "learned": 0.55125
   },
   {
    "kind": "scrambled",
    "I": 0.06512143305124349,
    "oracle": 0.31725,
    "learned": 0.30125
   },
   {
    "kind": "random",
    "I": 3.6587272121327654,
    "oracle": 0.91775,
    "learned": 0.675
   },
   {
    "kind": "scrambled",
    "I": 0.38085792590921813,
    "oracle": 0.44825,
    "learned": 0.28175
   },
   {
    "kind": "random",
    "I": 2.0788391562759605,
    "oracle": 0.7565,
    "learned": 0.7075
   },
   {
    "kind": "scrambled",
    "I": 0.0905059865338532,
    "oracle": 0.34875,
    "learned": 0.302
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.23975,
    "learned": 0.2455
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.24325,
    "learned": 0.2515
   },
   {
    "kind": "random",
    "I": 0.7464808459531929,
    "oracle": 0.39825,
    "learned": 0.39875
   },
   {
    "kind": "scrambled",
    "I": 0.054527920361028784,
    "oracle": 0.2885,
    "learned": 0.29375
   },
   {
    "kind": "random",
    "I": 2.585520868397421,
    "oracle": 0.80875,
    "learned": 0.70825
   },
   {
    "kind": "scrambled",
    "I": 0.36383483526253346,
    "oracle": 0.41625,
    "learned": 0.3285
   },
   {
    "kind": "random",
    "I": 2.407968365989276,
    "oracle": 0.78775,
    "learned": 0.63675
   },
   {
    "kind": "scrambled",
    "I": 0.11201541611867169,
    "oracle": 0.3455,
    "learned": 0.245
   },
   {
    "kind": "random",
    "I": 2.4799063975649767,
    "oracle": 0.8,
    "learned": 0.737
   },
   {
    "kind": "scrambled",
    "I": 0.2199489674734891,
    "oracle": 0.37525,
    "learned": 0.36675
   },
   {
    "kind": "random",
    "I": 1.6781537770907509,
    "oracle": 0.667,
    "learned": 0.66075
   },
   {
    "kind": "scrambled",
    "I": 0.32007265649887146,
    "oracle": 0.406,
    "learned": 0.4
   },
   {
    "kind": "random",
    "I": 1.4575738211286289,
    "oracle": 0.60425,
    "learned": 0.5855
   },
   {
    "kind": "scrambled",
    "I": 0.11533597449413238,
    "oracle": 0.34075,
    "learned": 0.354
   },
   {
    "kind": "random",
    "I": 0.6848557019398213,
    "oracle": 0.4525,
    "learned": 0.45475
   },
   {
    "kind": "scrambled",
    "I": 0.01810156801311349,
    "oracle": 0.29825,
    "learned": 0.277
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.25975,
    "learned": 0.25775
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.2505,
    "learned": 0.251
   },
   {
    "kind": "random",
    "I": 0.7634022384206418,
    "oracle": 0.4245,
    "learned": 0.4385
   },
   {
    "kind": "scrambled",
    "I": 0.010525045732599346,
    "oracle": 0.27975,
    "learned": 0.258
   },
   {
    "kind": "random",
    "I": 1.0450809189591037,
    "oracle": 0.53875,
    "learned": 0.54325
   },
   {
    "kind": "scrambled",
    "I": 0.025765059434889763,
    "oracle": 0.30325,
    "learned": 0.27525
   },
   {
    "kind": "random",
    "I": 2.3707226344916092,
    "oracle": 0.79,
    "learned": 0.64425
   },
   {
    "kind": "scrambled",
    "I": 0.20821738066926582,
    "oracle": 0.38275,
    "learned": 0.305
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.253,
    "learned": 0.2445
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.235,
    "learned": 0.25475
   },
   {
    "kind": "random",
    "I": 1.5840267326256667,
    "oracle": 0.6395,
    "learned": 0.6535
   },
   {
    "kind": "scrambled",
    "I": 0.028848608822032728,
    "oracle": 0.301,
    "learned": 0.26025
   },
   {
    "kind": "random",
    "I": 3.172183186554448,
    "oracle": 0.875,
    "learned": 0.65625
   },
   {
    "kind": "scrambled",
    "I": 0.34540527369149593,
    "oracle": 0.4405,
    "learned": 0.2605
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.2435,
    "learned": 0.24075
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.26575,
    "learned": 0.2535
   },
   {
    "kind": "random",
    "I": 0.9063982221095637,
    "oracle": 0.507,
    "learned": 0.49725
   },
   {
    "kind": "scrambled",
    "I": 0.05812255667985947,
    "oracle": 0.30575,
    "learned": 0.29425
   },
   {
    "kind": "random",
    "I": 0.9701330060435489,
    "oracle": 0.54775,
    "learned": 0.54625
   },
   {
    "kind": "scrambled",
    "I": 0.055650131787654114,
    "oracle": 0.32225,
    "learned": 0.31025
   },
   {
    "kind": "random",
    "I": 1.5387629569240908,
    "oracle": 0.651,
    "learned": 0.605
   },
   {
    "kind": "scrambled",
    "I": 0.17055158084356903,
    "oracle": 0.368,
    "learned": 0.34425
   },
   {
    "kind": "random",
    "I": 1.1318745670774608,
    "oracle": 0.552,
    "learned": 0.524
   },
   {
    "kind": "scrambled",
    "I": 0.052655179138645924,
    "oracle": 0.3085,
    "learned": 0.29375
   },
   {
    "kind": "random",
    "I": 3.3188438638725044,
    "oracle": 0.88625,
    "learned": 0.7035
   },
   {
    "kind": "scrambled",
    "I": 0.24483195488380805,
    "oracle": 0.4135,
    "learned": 0.30775
   },
   {
    "kind": "random",
    "I": 0.0,
    "oracle": 0.25325,
    "learned": 0.245
   },
   {
    "kind": "scrambled",
    "I": 0.0,
    "oracle": 0.256,
    "learned": 0.246
   },
   {
    "kind": "random",
    "I": 2.8635116007998183,
    "oracle": 0.83725,
    "learned": 0.666
   },
   {
    "kind": "scrambled",
    "I": 0.12251400878552318,
    "oracle": 0.3685,
    "learned": 0.3005
   },
   {
    "kind": "identical",
    "I": 4.874113724530505,
    "oracle": 0.95825,
    "learned": 0.764
   },
   {
    "kind": "identical",
    "I": 3.488462674175609,
    "oracle": 0.8975,
    "learned": 0.63975
   },
   {
    "kind": "identical",
    "I": 1.019882756560472,
    "oracle": 0.5415,
    "learned": 0.5315
   },
   {
    "kind": "identical",
    "I": 0.6819116787287982,
    "oracle": 0.44225,
    "learned": 0.428
   },
   {
    "kind": "identical",
    "I": 3.5338072748560077,
    "oracle": 0.8935,
    "learned": 0.71425
   },
   {
    "kind": "identical",
    "I": 3.2196484218057946,
    "oracle": 0.861,
    "learned": 0.7735
   },
   {
    "kind": "identical",
    "I": 1.5238494976846693,
    "oracle": 0.678,
    "learned": 0.66625
   },
   {
    "kind": "identical",
    "I": 2.85079542504025,
    "oracle": 0.83025,
    "learned": 0.74125
   }
  ]
 },
 "controls": {
  "K1": {
   "n": 2000,
   "usable": true,
   "mean": -0.04754202051019964,
   "q95": -0.030320606075675823,
   "q05": -0.06456212393281867,
   "p_ge": 0.0004997501249375312,
   "compare": "analysis.c1.gain.mean",
   "description": "C1 estimator values permuted across test pairs"
  },
  "K2": {
   "n": 2000,
   "usable": false,
   "mean": 0.0,
   "q95": 0.0,
   "q05": 0.0,
   "p_ge": 1.0,
   "why_unusable": "null has no spread",
   "compare": "analysis.c2.rho_diff",
   "description": "C2 old and new bound swapped per series at random"
  },
  "K3": {
   "n": 2000,
   "usable": true,
   "mean": -1.08,
   "q95": 34.0,
   "q05": -36.0,
   "p_ge": 0.0004997501249375312,
   "compare": "analysis.c3.b_minus_c",
   "description": "C3 old and new U calls swapped per dataset at random"
  },
  "K4": {
   "n": 2000,
   "usable": true,
   "mean": -0.030388692869495085,
   "q95": -0.014066216326737083,
   "q05": -0.045660175801011914,
   "p_ge": 0.0004997501249375312,
   "compare": "analysis.c4.T.A.gain.mean",
   "description": "C4 T model fed overlaps permuted across test pairs"
  },
  "K4R": {
   "n": 2000,
   "usable": true,
   "mean": -0.03179333581695967,
   "q95": -0.014786659904421668,
   "q05": -0.04886425406836438,
   "p_ge": 0.0004997501249375312,
   "compare": "analysis.c4.R1.A.gain.mean",
   "description": "C4 revision, same permutation (only if triggered)"
  },
  "K5": {
   "n": 2000,
   "usable": true,
   "mean": -2.132,
   "q95": 212.0,
   "q05": -208.1,
   "p_ge": 0.0004997501249375312,
   "compare": "analysis.c5.b_minus_c_sub",
   "description": "C5 old and new any-flag swapped per replicate at random"
  }
 },
 "timing_s": {
  "analysis": 1783.052467100002,
  "K1": 0.0008519999973941594,
  "K2": 0.00038129999302327633,
  "K3": 0.0007677000248804688,
  "K4": 0.0005438000080175698,
  "K4R": 0.0005926000012550503,
  "K5": 0.0005523999861907214
 },
 "verdicts": {
  "C1": {
   "verdict": "PASS",
   "rules": [
    {
     "rule": "analysis.ok.C1 >= 1",
     "left": 1,
     "right": 1,
     "holds": true
    },
    {
     "rule": "analysis.c1.gain.mean > controls.K1.q95",
     "left": 0.12068616475765752,
     "right": -0.030320606075675823,
     "holds": true
    }
   ]
  },
  "C2": {
   "verdict": "FAIL",
   "rules": [
    {
     "rule": "analysis.ok.C2 >= 1",
     "left": 0,
     "right": 1,
     "holds": false
    },
    {
     "rule": "analysis.c2.rho_diff > controls.K2.q95",
     "left": 0.0,
     "right": null,
     "holds": null,
     "why": "control K2 unusable"
    }
   ]
  },
  "C3": {
   "verdict": "PASS",
   "rules": [
    {
     "rule": "analysis.ok.C3 >= 1",
     "left": 1,
     "right": 1,
     "holds": true
    },
    {
     "rule": "analysis.c3.b_minus_c > controls.K3.q95",
     "left": 442,
     "right": 34.0,
     "holds": true
    }
   ]
  },
  "C4": {
   "verdict": "FAIL",
   "rules": [
    {
     "rule": "analysis.ok.C4 >= 1",
     "left": 0,
     "right": 1,
     "holds": false
    },
    {
     "rule": "analysis.c4.T.A.gain.mean > controls.K4.q95",
     "left": 0.11335805786145643,
     "right": -0.014066216326737083,
     "holds": true
    }
   ]
  },
  "C4R": {
   "verdict": "FAIL",
   "rules": [
    {
     "rule": "analysis.ok.C4R >= 1",
     "left": 0,
     "right": 1,
     "holds": false
    },
    {
     "rule": "analysis.c4.R1.A.gain.mean > controls.K4R.q95",
     "left": 0.12966580273748565,
     "right": -0.014786659904421668,
     "holds": true
    }
   ]
  },
  "C5": {
   "verdict": "PASS",
   "rules": [
    {
     "rule": "analysis.ok.C5 >= 1",
     "left": 1,
     "right": 1,
     "holds": true
    },
    {
     "rule": "analysis.c5.b_minus_c_sub > controls.K5.q95",
     "left": 15580.0,
     "right": 212.0,
     "holds": true
    }
   ]
  },
  "D0_f79_replay": {
   "verdict": "PASS",
   "rules": [
    {
     "rule": "analysis.f79_reproduction.I <= 1e-12",
     "left": 0.0,
     "right": 1e-12,
     "holds": true
    },
    {
     "rule": "analysis.f79_reproduction.oracle <= 1e-12",
     "left": 0.0,
     "right": 1e-12,
     "holds": true
    },
    {
     "rule": "analysis.f79_reproduction.learned <= 1e-12",
     "left": 0.0,
     "right": 1e-12,
     "holds": true
    }
   ]
  }
 },
 "reproduce": null,
 "status": "scored"
}