{
  "scope": "Post-protocol native singleton-batch sensitivity control on the held-out questions. Does not replace primary results.",
  "seconds": 64.3668233329663,
  "scores": [
    [
      0.75,
      1.0,
      0.875,
      0.75,
      -0.75,
      -1.125,
      -0.875,
      1.0,
      0.75,
      1.125,
      0.5,
      -0.625,
      -0.875,
      -1.25,
      -1.125,
      -0.875
    ],
    [
      1.0,
      -1.375,
      0.875,
      1.0,
      1.25,
      1.0,
      0.875,
      0.375,
      1.125,
      0.375,
      0.75,
      0.625,
      0.75,
      0.625,
      -2.25,
      0.375
    ],
    [
      0.875,
      0.25,
      0.375,
      1.375,
      1.75,
      0.125,
      -0.75,
      1.0,
      1.0,
      1.375,
      1.125,
      0.75,
      1.25,
      0.625,
      0.375,
      0.75
    ],
    [
      0.625,
      0.625,
      1.0,
      0.75,
      1.0,
      1.0,
      0.625,
      0.375,
      0.375,
      0.875,
      1.0,
      0.625,
      -1.5,
      -1.5,
      0.375,
      0.875
    ],
    [
      -1.125,
      0.375,
      -0.125,
      0.25,
      -0.875,
      1.0,
      -1.625,
      0.75,
      -1.125,
      0.875,
      -0.5,
      0.875,
      -1.375,
      0.625,
      -1.0,
      0.5
    ],
    [
      1.375,
      0.75,
      0.5,
      0.625,
      -0.875,
      -1.125,
      1.125,
      1.0,
      -0.75,
      -0.5,
      0.5,
      -0.75,
      1.0,
      0.625,
      -0.875,
      0.75
    ],
    [
      0.75,
      -0.375,
      -0.625,
      -0.875,
      0.375,
      0.125,
      -0.625,
      0.875,
      0.375,
      0.875,
      0.375,
      -0.875,
      -0.875,
      1.0,
      0.375,
      -0.125
    ],
    [
      -4.375,
      -3.25,
      1.25,
      -1.375,
      0.75,
      0.25,
      1.0,
      -4.0,
      0.875,
      0.625,
      1.25,
      0.75,
      0.375,
      0.25,
      0.75,
      0.875
    ],
    [
      0.5,
      1.125,
      0.625,
      1.125,
      1.125,
      1.0,
      1.375,
      1.125,
      0.625,
      0.625,
      0.625,
      1.125,
      0.75,
      1.375,
      0.5,
      1.0
    ],
    [
      0.5,
      1.125,
      0.375,
      0.125,
      0.25,
      -0.125,
      0.625,
      1.0,
      -0.875,
      0.875,
      0.625,
      0.875,
      0.25,
      0.0,
      0.125,
      0.625
    ],
    [
      0.375,
      0.625,
      0.375,
      0.125,
      0.25,
      0.0,
      0.375,
      0.5,
      0.0,
      0.375,
      0.75,
      0.5,
      0.5,
      0.875,
      0.375,
      0.875
    ],
    [
      0.125,
      1.125,
      -0.25,
      -0.75,
      0.875,
      0.5,
      0.75,
      0.375,
      0.125,
      0.375,
      1.375,
      -0.875,
      0.875,
      -1.5,
      0.0,
      1.375
    ],
    [
      -0.25,
      0.25,
      -0.25,
      0.125,
      0.5,
      0.375,
      0.5,
      -0.125,
      -0.5,
      0.25,
      0.75,
      -0.125,
      0.125,
      0.625,
      -0.5,
      0.375
    ],
    [
      0.0,
      0.875,
      0.125,
      0.125,
      0.25,
      0.375,
      0.125,
      0.5,
      -1.5,
      0.125,
      0.125,
      0.25,
      0.625,
      0.5,
      0.375,
      1.125
    ],
    [
      0.875,
      0.875,
      0.625,
      0.875,
      0.75,
      -1.125,
      0.0,
      0.375,
      0.625,
      0.375,
      1.0,
      -0.25,
      -1.125,
      -0.125,
      0.5,
      0.25
    ],
    [
      0.0,
      -1.125,
      0.75,
      0.125,
      -1.625,
      0.625,
      -0.75,
      0.625,
      0.625,
      0.25,
      0.625,
      -1.625,
      0.375,
      0.375,
      0.75,
      0.875
    ],
    [
      -0.875,
      0.875,
      0.75,
      -0.25,
      1.75,
      -1.125,
      -0.5,
      -1.125,
      -0.375,
      -4.0,
      -1.0,
      -3.875,
      -4.25,
      1.25,
      -3.875,
      -1.25
    ],
    [
      -0.125,
      0.875,
      0.0,
      1.75,
      -0.125,
      0.875,
      0.625,
      1.5,
      1.25,
      1.0,
      0.75,
      1.625,
      1.0,
      0.625,
      1.25,
      1.75
    ],
    [
      0.0,
      0.375,
      0.75,
      -1.0,
      1.125,
      0.625,
      1.25,
      0.25,
      0.75,
      0.875,
      1.125,
      0.625,
      -1.75,
      0.75,
      0.125,
      -1.75
    ],
    [
      0.375,
      1.0,
      -0.875,
      0.625,
      0.25,
      0.375,
      0.875,
      0.75,
      -2.0,
      0.25,
      -0.875,
      0.875,
      0.625,
      1.25,
      -0.875,
      1.0
    ],
    [
      0.0,
      0.125,
      -0.25,
      0.125,
      0.0,
      -0.625,
      0.0,
      0.125,
      -0.125,
      -0.125,
      0.25,
      0.125,
      -0.125,
      -0.125,
      0.25,
      0.25
    ],
    [
      -0.375,
      0.0,
      0.0,
      0.25,
      0.5,
      -0.125,
      -0.125,
      0.0,
      -0.25,
      -0.25,
      -0.125,
      -0.125,
      0.125,
      0.375,
      -0.125,
      0.25
    ],
    [
      -0.25,
      -0.25,
      -0.125,
      1.625,
      -0.125,
      0.25,
      -0.25,
      0.125,
      0.0,
      0.875,
      -0.25,
      0.125,
      0.0,
      1.125,
      0.5,
      0.5
    ],
    [
      0.25,
      -0.25,
      0.75,
      -0.25,
      1.125,
      -0.125,
      -0.125,
      0.25,
      -0.25,
      -0.125,
      0.5,
      0.75,
      0.0,
      0.0,
      0.25,
      0.25
    ],
    [
      1.0,
      0.375,
      0.25,
      0.375,
      0.25,
      0.75,
      0.0,
      0.375,
      0.25,
      0.25,
      0.375,
      0.875,
      0.375,
      0.375,
      0.0,
      0.625
    ],
    [
      -0.25,
      0.0,
      0.0,
      -0.125,
      0.75,
      0.875,
      0.5,
      0.0,
      0.375,
      0.125,
      0.25,
      0.0,
      0.875,
      0.125,
      0.125,
      0.25
    ],
    [
      0.125,
      -0.375,
      0.375,
      0.25,
      -0.375,
      0.25,
      0.0,
      0.125,
      0.25,
      -0.5,
      -0.125,
      0.0,
      0.125,
      0.125,
      0.125,
      0.125
    ],
    [
      2.0,
      1.0,
      1.25,
      0.75,
      0.875,
      0.125,
      -0.375,
      -4.0,
      -3.0,
      1.0,
      0.625,
      -4.25,
      -2.0,
      0.125,
      0.125,
      -0.125
    ],
    [
      -0.5,
      -0.375,
      -0.75,
      0.5,
      -0.75,
      -0.625,
      -0.25,
      -0.125,
      -0.25,
      -0.375,
      -0.625,
      -0.25,
      0.0,
      -0.625,
      -0.125,
      0.0
    ],
    [
      -0.125,
      0.125,
      0.5,
      0.125,
      -0.5,
      0.0,
      -0.25,
      0.0,
      0.25,
      0.25,
      0.125,
      0.125,
      0.125,
      0.375,
      -0.25,
      -0.25
    ],
    [
      0.0,
      -0.125,
      -0.25,
      -0.25,
      0.25,
      0.125,
      0.75,
      0.125,
      0.125,
      0.25,
      0.0,
      0.25,
      0.25,
      -0.375,
      0.0,
      0.125
    ],
    [
      -0.75,
      0.125,
      0.375,
      0.0,
      0.0,
      0.0,
      -0.875,
      -0.625,
      0.0,
      0.125,
      0.0,
      0.0,
      0.0,
      0.0,
      -0.125,
      0.0
    ],
    [
      0.25,
      0.625,
      0.125,
      0.5,
      0.0,
      0.125,
      0.0,
      0.375,
      0.0,
      -0.25,
      0.25,
      0.5,
      0.125,
      0.5,
      -0.25,
      0.0
    ],
    [
      0.125,
      0.0,
      0.0,
      0.125,
      0.25,
      0.25,
      0.375,
      0.75,
      -0.125,
      -0.25,
      0.5,
      -1.5,
      0.5,
      0.125,
      0.125,
      0.0
    ],
    [
      -0.125,
      0.375,
      0.0,
      0.125,
      0.125,
      -0.375,
      0.875,
      0.0,
      0.5,
      -0.375,
      -0.375,
      0.0,
      -0.375,
      0.0,
      0.25,
      -0.125
    ],
    [
      0.5,
      0.125,
      0.625,
      0.125,
      0.125,
      0.5,
      0.125,
      0.125,
      0.25,
      0.375,
      -0.5,
      0.75,
      0.25,
      0.0,
      0.125,
      0.125
    ],
    [
      -0.125,
      -0.75,
      0.25,
      -0.25,
      -0.5,
      -0.375,
      0.125,
      -0.5,
      0.125,
      0.25,
      -0.375,
      0.5,
      -0.5,
      -0.25,
      -0.25,
      -0.375
    ],
    [
      -0.25,
      0.75,
      -0.125,
      0.5,
      0.25,
      0.25,
      -0.375,
      -0.25,
      0.125,
      0.125,
      0.625,
      -0.375,
      0.125,
      0.5,
      0.0,
      0.375
    ],
    [
      0.125,
      0.5,
      -0.25,
      -0.5,
      0.125,
      0.625,
      -0.375,
      -0.125,
      0.125,
      -0.25,
      0.25,
      -0.25,
      -0.125,
      0.375,
      0.125,
      0.125
    ],
    [
      0.0,
      0.375,
      0.375,
      0.25,
      0.125,
      0.75,
      0.875,
      0.75,
      0.25,
      0.375,
      1.25,
      0.75,
      0.125,
      -0.5,
      0.25,
      0.5
    ],
    [
      1.125,
      1.375,
      0.375,
      1.25,
      0.75,
      0.125,
      0.5,
      1.25,
      0.875,
      0.375,
      0.75,
      0.5,
      1.0,
      1.375,
      0.375,
      0.5
    ],
    [
      1.25,
      0.625,
      0.375,
      0.75,
      1.0,
      0.125,
      0.25,
      0.375,
      0.75,
      1.125,
      0.625,
      1.125,
      0.5,
      1.25,
      0.5,
      1.125
    ],
    [
      0.25,
      1.0,
      0.5,
      0.375,
      0.75,
      1.0,
      0.625,
      0.5,
      0.375,
      0.875,
      1.0,
      0.75,
      0.5,
      0.125,
      0.625,
      0.625
    ],
    [
      1.0,
      1.0,
      0.75,
      1.125,
      1.375,
      1.0,
      0.625,
      1.125,
      0.5,
      0.625,
      0.5,
      0.375,
      0.375,
      1.125,
      0.25,
      0.5
    ],
    [
      0.875,
      0.25,
      0.25,
      0.375,
      0.375,
      0.25,
      0.875,
      0.0,
      -0.25,
      -0.125,
      0.25,
      -0.25,
      -0.25,
      -0.25,
      0.25,
      0.375
    ],
    [
      0.625,
      0.5,
      0.25,
      1.75,
      0.625,
      1.0,
      1.25,
      0.625,
      1.5,
      0.75,
      0.625,
      1.0,
      0.75,
      1.375,
      1.25,
      1.25
    ],
    [
      1.375,
      0.875,
      0.75,
      1.0,
      0.375,
      1.125,
      0.875,
      0.75,
      0.625,
      0.75,
      0.125,
      0.75,
      0.125,
      0.625,
      0.5,
      0.875
    ],
    [
      1.0,
      0.625,
      0.625,
      0.75,
      0.875,
      1.375,
      -0.125,
      0.75,
      0.125,
      0.75,
      0.875,
      0.0,
      0.75,
      0.75,
      0.75,
      0.75
    ],
    [
      0.75,
      0.75,
      0.75,
      0.75,
      0.875,
      0.875,
      0.75,
      1.0,
      -0.375,
      0.5,
      1.0,
      1.0,
      0.5,
      0.75,
      0.625,
      0.875
    ],
    [
      0.375,
      0.125,
      0.625,
      0.125,
      0.375,
      0.625,
      0.25,
      0.875,
      0.25,
      0.625,
      0.0,
      0.25,
      0.875,
      0.125,
      0.5,
      0.375
    ],
    [
      0.625,
      1.5,
      0.5,
      0.75,
      1.125,
      1.5,
      1.25,
      0.375,
      0.75,
      1.625,
      0.375,
      1.25,
      1.375,
      0.75,
      1.25,
      1.25
    ],
    [
      0.875,
      -0.5,
      1.125,
      1.125,
      0.875,
      0.625,
      0.625,
      0.625,
      0.875,
      0.875,
      1.0,
      0.5,
      0.625,
      1.125,
      -0.75,
      0.5
    ],
    [
      0.0,
      0.0,
      0.0,
      0.5,
      1.0,
      -0.375,
      0.25,
      0.625,
      0.25,
      -0.625,
      0.375,
      -0.375,
      -0.5,
      0.5,
      0.25,
      0.125
    ],
    [
      0.0,
      -0.125,
      0.0,
      0.125,
      0.375,
      0.125,
      -0.25,
      -0.125,
      0.875,
      -0.5,
      0.5,
      -0.25,
      -0.125,
      -0.125,
      -0.25,
      0.125
    ],
    [
      1.25,
      1.375,
      1.125,
      1.125,
      1.125,
      1.375,
      0.875,
      1.0,
      1.125,
      1.625,
      0.625,
      0.625,
      0.875,
      1.375,
      1.0,
      0.625
    ],
    [
      0.375,
      0.375,
      -0.5,
      -0.125,
      -0.875,
      0.25,
      -0.25,
      0.375,
      0.5,
      0.75,
      -0.25,
      1.0,
      -0.125,
      -0.5,
      1.125,
      0.5
    ],
    [
      0.5,
      0.125,
      0.625,
      0.75,
      0.375,
      1.125,
      0.75,
      0.125,
      0.25,
      1.125,
      0.875,
      0.125,
      0.25,
      0.375,
      0.5,
      0.75
    ],
    [
      0.0,
      0.375,
      0.125,
      0.375,
      0.375,
      0.25,
      0.125,
      -0.125,
      0.5,
      -0.125,
      -0.125,
      0.375,
      -0.125,
      0.375,
      0.25,
      0.5
    ],
    [
      -0.625,
      0.0,
      0.375,
      0.0,
      0.125,
      0.375,
      0.25,
      -0.25,
      0.125,
      -0.125,
      -0.75,
      0.5,
      0.75,
      0.875,
      -0.5,
      0.25
    ],
    [
      0.5,
      0.875,
      -1.125,
      0.25,
      0.625,
      1.0,
      0.25,
      0.625,
      1.875,
      0.125,
      0.5,
      1.125,
      0.75,
      0.375,
      0.5,
      0.375
    ],
    [
      1.0,
      0.0,
      0.125,
      0.875,
      1.0,
      0.75,
      0.375,
      0.5,
      0.25,
      1.25,
      1.25,
      0.375,
      1.125,
      0.375,
      1.625,
      -0.125
    ],
    [
      1.625,
      2.0,
      1.625,
      1.75,
      1.0,
      1.75,
      1.375,
      1.625,
      1.625,
      1.5,
      1.5,
      1.5,
      1.75,
      1.625,
      1.625,
      1.125
    ],
    [
      1.25,
      1.125,
      1.25,
      0.5,
      1.375,
      0.875,
      1.25,
      0.75,
      1.5,
      1.375,
      1.25,
      1.625,
      0.875,
      1.0,
      1.75,
      0.5
    ],
    [
      -3.0,
      -1.0,
      -3.25,
      -0.375,
      -3.125,
      0.125,
      -0.375,
      0.875,
      0.75,
      -0.375,
      -3.0,
      -3.625,
      -3.125,
      -3.375,
      -1.125,
      -3.375
    ],
    [
      0.875,
      1.25,
      0.875,
      0.875,
      0.625,
      1.0,
      0.5,
      0.25,
      0.75,
      0.75,
      2.0,
      0.5,
      1.5,
      0.25,
      1.125,
      0.375
    ],
    [
      0.875,
      1.5,
      0.25,
      0.5,
      1.5,
      0.5,
      0.75,
      0.75,
      0.625,
      0.25,
      1.0,
      1.0,
      0.625,
      1.125,
      1.25,
      0.375
    ],
    [
      0.875,
      0.875,
      1.125,
      1.625,
      0.625,
      0.75,
      0.625,
      0.875,
      0.25,
      1.125,
      0.875,
      1.375,
      0.875,
      1.5,
      0.125,
      0.5
    ],
    [
      0.375,
      2.0,
      0.75,
      1.375,
      0.75,
      1.375,
      1.625,
      0.875,
      0.5,
      1.5,
      1.5,
      0.875,
      0.5,
      1.125,
      1.25,
      1.125
    ],
    [
      -3.0,
      0.5,
      -0.875,
      0.875,
      0.125,
      -3.25,
      -3.125,
      -3.0,
      -3.125,
      0.25,
      -2.75,
      -2.875,
      -3.375,
      -1.5,
      1.0,
      -1.125
    ],
    [
      -0.25,
      0.5,
      -0.375,
      0.125,
      0.625,
      1.0,
      0.75,
      1.25,
      0.25,
      0.5,
      0.875,
      0.875,
      0.0,
      0.875,
      0.5,
      0.5
    ],
    [
      1.0,
      0.375,
      1.625,
      0.5,
      0.125,
      1.125,
      0.875,
      2.0,
      0.75,
      0.375,
      0.625,
      0.5,
      0.0,
      0.0,
      0.625,
      0.75
    ],
    [
      0.875,
      0.5,
      0.625,
      -0.375,
      -0.25,
      0.875,
      0.5,
      0.25,
      1.25,
      0.375,
      0.625,
      0.125,
      0.5,
      0.375,
      0.5,
      0.375
    ],
    [
      1.375,
      0.5,
      1.25,
      1.5,
      1.75,
      1.25,
      0.375,
      1.125,
      1.75,
      1.25,
      0.125,
      0.375,
      1.125,
      1.0,
      0.875,
      1.125
    ],
    [
      0.125,
      0.375,
      1.25,
      0.5,
      0.75,
      0.625,
      0.625,
      0.625,
      0.375,
      0.875,
      1.625,
      0.875,
      0.25,
      0.75,
      0.25,
      0.875
    ],
    [
      0.375,
      0.25,
      0.25,
      0.75,
      1.0,
      0.25,
      1.0,
      0.25,
      0.125,
      0.625,
      0.375,
      0.125,
      0.875,
      1.125,
      0.125,
      1.125
    ],
    [
      1.25,
      1.375,
      0.75,
      1.125,
      0.875,
      1.375,
      0.625,
      0.875,
      -0.125,
      -0.125,
      1.0,
      0.75,
      0.625,
      0.875,
      1.375,
      0.375
    ],
    [
      0.5,
      1.125,
      1.375,
      1.125,
      0.625,
      0.25,
      -0.25,
      1.25,
      1.875,
      0.75,
      0.75,
      1.375,
      1.0,
      0.875,
      1.0,
      1.625
    ],
    [
      1.125,
      2.25,
      2.0,
      1.5,
      1.875,
      1.625,
      2.625,
      0.75,
      1.875,
      2.25,
      2.0,
      1.125,
      1.625,
      2.125,
      2.125,
      2.5
    ],
    [
      2.125,
      0.25,
      0.625,
      0.5,
      0.875,
      0.875,
      1.0,
      1.875,
      0.875,
      1.75,
      0.5,
      0.625,
      1.125,
      0.875,
      0.75,
      0.25
    ],
    [
      1.75,
      1.5,
      1.125,
      0.875,
      0.75,
      1.25,
      1.875,
      1.625,
      0.875,
      1.625,
      0.75,
      2.25,
      0.625,
      1.625,
      1.0,
      1.125
    ]
  ],
  "max_score_change": 0.375,
  "summary": [
    {
      "n": 1,
      "selection_disagreement": 0.0,
      "accuracy": 0.0375
    },
    {
      "n": 2,
      "selection_disagreement": 0.0875,
      "accuracy": 0.0375
    },
    {
      "n": 4,
      "selection_disagreement": 0.1125,
      "accuracy": 0.05
    },
    {
      "n": 8,
      "selection_disagreement": 0.2125,
      "accuracy": 0.0875
    },
    {
      "n": 16,
      "selection_disagreement": 0.2875,
      "accuracy": 0.1125
    }
  ]
}