{
  "config": {
    "variant": "10-class Colored MNIST variant (not IRM binary)",
    "palette": [
      [
        1.0,
        0.0,
        0.0
      ],
      [
        0.0,
        1.0,
        0.0
      ],
      [
        0.0,
        0.0,
        1.0
      ],
      [
        1.0,
        1.0,
        0.0
      ],
      [
        1.0,
        0.0,
        1.0
      ],
      [
        0.0,
        1.0,
        1.0
      ],
      [
        1.0,
        0.5,
        0.0
      ],
      [
        0.5,
        0.0,
        1.0
      ],
      [
        0.5,
        1.0,
        0.0
      ],
      [
        1.0,
        1.0,
        1.0
      ]
    ],
    "palette_names": [
      "red",
      "green",
      "blue",
      "yellow",
      "magenta",
      "cyan",
      "orange",
      "purple",
      "lime",
      "white"
    ],
    "train_domain": [
      "aligned",
      0.9
    ],
    "domains": {
      "matched": [
        "aligned",
        0.9
      ],
      "independent": [
        "aligned",
        0.0
      ],
      "misleading": [
        "shifted",
        0.9
      ]
    },
    "colour_seeds": {
      "train": 1000,
      "calibration": 1001,
      "validation": 1002,
      "test": 1003
    },
    "split_seed": 0,
    "colour_rule": "with probability p use the mapping colour, else a uniform colour from all 10; so the designated colour appears with probability p + (1 - p)/10",
    "effective_designated_colour_probability": {
      "matched": 0.91,
      "independent": 0.1,
      "misleading": 0.91
    },
    "rank_candidates_note": "rank 3 was added to the candidate set after inspecting the spectrum (three dominant directions); selection still uses validation only",
    "splits": {
      "train": 50000,
      "calibration": 5000,
      "validation": 5000,
      "test": 10000
    },
    "calibration_images_in_counterfactual_set": 2000,
    "model": "2352 -> 128 ReLU -> 10",
    "epochs": 8,
    "batch_size": 512,
    "learning_rate": 0.002,
    "training_seeds": [
      42,
      43,
      44
    ],
    "ranks": [
      1,
      2,
      3,
      4,
      8,
      16,
      32
    ],
    "strengths": [
      0.5,
      1.0
    ],
    "random_draws": 3,
    "selection": "validation only; objective = mean accuracy over the three domains",
    "dataset_sha256": "731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1",
    "torch": "2.7.1+cpu",
    "quick": false
  },
  "definitions": {
    "colour_subspace": "top right-singular vectors of within-image hidden differences over all 10 colours on calibration images; an ESTIMATED colour-associated subspace",
    "suppression": "A = I - strength * U U^T inserted between the frozen extractor and frozen head",
    "null_space_control": "A = I - W+W removes everything the head reads; logits become the bias",
    "row_space_identity": "A = W+W keeps everything the head reads; logits unchanged",
    "replacement": "h_patch = h + ((h_donor - h) U) U^T; donor is the SAME source image recolored. Uses the suppression-selected rank without further tuning. All ten donor colours, including the source colour, are evaluated equally. Aligned/shifted donors use known labels as diagnostic conditions, not deployable accuracy improvements. Random controls use the same rank, full replacement, and identical donors."
  },
  "aggregate_test": {
    "baseline_test": {
      "matched": {
        "mean": 0.9808666666666667,
        "sd": 0.0011614167593456512,
        "per_seed": [
          0.9799,
          0.9825,
          0.9802
        ]
      },
      "independent": {
        "mean": 0.8703,
        "sd": 0.0019815818596935503,
        "per_seed": [
          0.8676,
          0.871,
          0.8723
        ]
      },
      "misleading": {
        "mean": 0.8591333333333333,
        "sd": 0.01125650429258073,
        "per_seed": [
          0.8434,
          0.8691,
          0.8649
        ]
      }
    },
    "selected_test": {
      "matched": {
        "mean": 0.9756333333333332,
        "sd": 0.003086889840744046,
        "per_seed": [
          0.9727,
          0.9799,
          0.9743
        ]
      },
      "independent": {
        "mean": 0.8814333333333334,
        "sd": 0.003166842100403627,
        "per_seed": [
          0.8831,
          0.8842,
          0.877
        ]
      },
      "misleading": {
        "mean": 0.9034,
        "sd": 0.006219860662962403,
        "per_seed": [
          0.9007,
          0.8975,
          0.912
        ]
      }
    },
    "selected_test_full_strength": {
      "matched": {
        "mean": 0.9715666666666666,
        "sd": 0.002811089152307752,
        "per_seed": [
          0.9727,
          0.9677,
          0.9743
        ]
      },
      "independent": {
        "mean": 0.8758333333333334,
        "sd": 0.0064623697064020175,
        "per_seed": [
          0.8831,
          0.8674,
          0.877
        ]
      },
      "misleading": {
        "mean": 0.9059333333333334,
        "sd": 0.004650686926561409,
        "per_seed": [
          0.9007,
          0.9051,
          0.912
        ]
      }
    },
    "random_control_test": {
      "matched": {
        "mean": 0.9806444444444445,
        "sd": 0.0012178710426572566,
        "per_seed": [
          0.9798,
          0.9823666666666666,
          0.9797666666666668
        ]
      },
      "independent": {
        "mean": 0.8691555555555555,
        "sd": 0.0008949791362490454,
        "per_seed": [
          0.8683333333333333,
          0.8704000000000001,
          0.8687333333333332
        ]
      },
      "misleading": {
        "mean": 0.8588444444444444,
        "sd": 0.011949595788401102,
        "per_seed": [
          0.8422999999999999,
          0.8701,
          0.8641333333333333
        ]
      }
    },
    "null_space_control_test": {
      "matched": {
        "mean": 0.09826666666666667,
        "sd": 0.006411101482758031,
        "per_seed": [
          0.1028,
          0.0892,
          0.1028
        ]
      },
      "independent": {
        "mean": 0.09826666666666667,
        "sd": 0.006411101482758031,
        "per_seed": [
          0.1028,
          0.0892,
          0.1028
        ]
      },
      "misleading": {
        "mean": 0.09826666666666667,
        "sd": 0.006411101482758031,
        "per_seed": [
          0.1028,
          0.0892,
          0.1028
        ]
      }
    },
    "row_space_identity_test": {
      "matched": {
        "mean": 0.9808666666666667,
        "sd": 0.0011614167593456512,
        "per_seed": [
          0.9799,
          0.9825,
          0.9802
        ]
      },
      "independent": {
        "mean": 0.8703,
        "sd": 0.0019815818596935503,
        "per_seed": [
          0.8676,
          0.871,
          0.8723
        ]
      },
      "misleading": {
        "mean": 0.8591333333333333,
        "sd": 0.01125650429258073,
        "per_seed": [
          0.8434,
          0.8691,
          0.8649
        ]
      }
    }
  },
  "per_seed": [
    {
      "seed": 42,
      "baseline_validation": {
        "matched": 0.9804,
        "independent": 0.859,
        "misleading": 0.8362
      },
      "baseline_test": {
        "matched": {
          "accuracy": 0.9799,
          "correct": 9799,
          "total": 10000,
          "mean_true_class_probability": 0.9718909070855946,
          "per_digit_accuracy": {
            "0": 0.9928571428571429,
            "1": 0.9929515418502203,
            "2": 0.9777131782945736,
            "3": 0.9920792079207921,
            "4": 0.9816700610997964,
            "5": 0.9708520179372198,
            "6": 0.9812108559498957,
            "7": 0.97568093385214,
            "8": 0.9671457905544147,
            "9": 0.9643211100099108
          }
        },
        "independent": {
          "accuracy": 0.8676,
          "correct": 8676,
          "total": 10000,
          "mean_true_class_probability": 0.8315220538407555,
          "per_digit_accuracy": {
            "0": 0.9275510204081633,
            "1": 0.9726872246696036,
            "2": 0.8372093023255814,
            "3": 0.8940594059405941,
            "4": 0.8961303462321792,
            "5": 0.8408071748878924,
            "6": 0.8956158663883089,
            "7": 0.8501945525291829,
            "8": 0.7453798767967146,
            "9": 0.800792864222002
          }
        },
        "misleading": {
          "accuracy": 0.8434,
          "correct": 8434,
          "total": 10000,
          "mean_true_class_probability": 0.7973169746797446,
          "per_digit_accuracy": {
            "0": 0.926530612244898,
            "1": 0.9524229074889868,
            "2": 0.7819767441860465,
            "3": 0.9376237623762376,
            "4": 0.835030549898167,
            "5": 0.7825112107623319,
            "6": 0.8956158663883089,
            "7": 0.857976653696498,
            "8": 0.7453798767967146,
            "9": 0.7006937561942518
          }
        }
      },
      "spectrum_variance_fraction": [
        0.3318077027797699,
        0.21946364641189575,
        0.18265655636787415,
        0.040159232914447784,
        0.028129488229751587,
        0.02273988537490368,
        0.020931169390678406,
        0.016513021662831306,
        0.011560103856027126,
        0.010619101114571095,
        0.009323845617473125,
        0.007775085046887398,
        0.007490431424230337,
        0.006310070864856243,
        0.0057069952599704266,
        0.00532525684684515,
        0.004856762010604143,
        0.00409535551443696,
        0.003790098475292325,
        0.003662044182419777,
        0.003230119589716196,
        0.0031573947053402662,
        0.0026585832238197327,
        0.0025659287348389626,
        0.0022656049113720655,
        0.0020764353685081005,
        0.001964628230780363,
        0.0018793573835864663,
        0.0016919070621952415,
        0.001655144733376801,
        0.0014450059970840812,
        0.001319228787906468
      ],
      "validation_grid": [
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9802,
            "independent": 0.8658,
            "misleading": 0.8546
          },
          "objective": 0.9002
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.9776,
            "independent": 0.8656,
            "misleading": 0.8668
          },
          "objective": 0.9033333333333333
        },
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9794,
            "independent": 0.857,
            "misleading": 0.836
          },
          "objective": 0.8908
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.978,
            "independent": 0.8554,
            "misleading": 0.8322
          },
          "objective": 0.8885333333333335
        },
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.979,
            "independent": 0.856,
            "misleading": 0.8294
          },
          "objective": 0.8881333333333333
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9788,
            "independent": 0.85,
            "misleading": 0.8226
          },
          "objective": 0.8837999999999999
        },
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9808,
            "independent": 0.8612,
            "misleading": 0.839
          },
          "objective": 0.8936666666666667
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9804,
            "independent": 0.8608,
            "misleading": 0.8392
          },
          "objective": 0.8934666666666667
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.979,
            "independent": 0.873,
            "misleading": 0.8692
          },
          "objective": 0.9070666666666666
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.9704,
            "independent": 0.8768,
            "misleading": 0.8914
          },
          "objective": 0.9128666666666666
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9808,
            "independent": 0.8592,
            "misleading": 0.8388
          },
          "objective": 0.8929333333333332
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9796,
            "independent": 0.858,
            "misleading": 0.8374
          },
          "objective": 0.8916666666666667
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.981,
            "independent": 0.8592,
            "misleading": 0.8376
          },
          "objective": 0.8926
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.98,
            "independent": 0.8564,
            "misleading": 0.8372
          },
          "objective": 0.8912
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9814,
            "independent": 0.859,
            "misleading": 0.835
          },
          "objective": 0.8917999999999999
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9812,
            "independent": 0.8586,
            "misleading": 0.8272
          },
          "objective": 0.8889999999999999
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9596,
            "independent": 0.8416,
            "misleading": 0.853
          },
          "objective": 0.8847333333333335
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.863,
            "independent": 0.7382,
            "misleading": 0.7764
          },
          "objective": 0.7925333333333334
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.979,
            "independent": 0.853,
            "misleading": 0.8248
          },
          "objective": 0.8855999999999998
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9754,
            "independent": 0.8428,
            "misleading": 0.807
          },
          "objective": 0.8750666666666667
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9782,
            "independent": 0.8558,
            "misleading": 0.8264
          },
          "objective": 0.8868
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9762,
            "independent": 0.8496,
            "misleading": 0.814
          },
          "objective": 0.8799333333333333
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.98,
            "independent": 0.8598,
            "misleading": 0.8382
          },
          "objective": 0.8926666666666666
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.978,
            "independent": 0.8562,
            "misleading": 0.8402
          },
          "objective": 0.8914666666666666
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9598,
            "independent": 0.8434,
            "misleading": 0.8532
          },
          "objective": 0.8854666666666665
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.8468,
            "independent": 0.7462,
            "misleading": 0.7774
          },
          "objective": 0.7901333333333334
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9782,
            "independent": 0.8524,
            "misleading": 0.8252
          },
          "objective": 0.8852666666666668
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9752,
            "independent": 0.8414,
            "misleading": 0.81
          },
          "objective": 0.8755333333333333
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9774,
            "independent": 0.8526,
            "misleading": 0.8308
          },
          "objective": 0.8869333333333334
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.973,
            "independent": 0.8342,
            "misleading": 0.81
          },
          "objective": 0.8724
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9796,
            "independent": 0.858,
            "misleading": 0.842
          },
          "objective": 0.8932000000000001
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9786,
            "independent": 0.8468,
            "misleading": 0.8414
          },
          "objective": 0.8889333333333335
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9592,
            "independent": 0.8148,
            "misleading": 0.8186
          },
          "objective": 0.8642
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.609,
            "independent": 0.4906,
            "misleading": 0.4808
          },
          "objective": 0.5268
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9774,
            "independent": 0.8434,
            "misleading": 0.8166
          },
          "objective": 0.8791333333333334
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9738,
            "independent": 0.8158,
            "misleading": 0.782
          },
          "objective": 0.8572000000000001
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9784,
            "independent": 0.8582,
            "misleading": 0.8362
          },
          "objective": 0.8909333333333334
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9764,
            "independent": 0.849,
            "misleading": 0.825
          },
          "objective": 0.8834666666666667
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9808,
            "independent": 0.8584,
            "misleading": 0.8386
          },
          "objective": 0.8926
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9794,
            "independent": 0.855,
            "misleading": 0.8348
          },
          "objective": 0.8897333333333334
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9714,
            "independent": 0.8446,
            "misleading": 0.8344
          },
          "objective": 0.8834666666666667
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.5578,
            "independent": 0.4016,
            "misleading": 0.3836
          },
          "objective": 0.44766666666666666
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9762,
            "independent": 0.8456,
            "misleading": 0.825
          },
          "objective": 0.8822666666666666
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9726,
            "independent": 0.8154,
            "misleading": 0.7894
          },
          "objective": 0.8591333333333333
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9792,
            "independent": 0.8518,
            "misleading": 0.8264
          },
          "objective": 0.8858
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9748,
            "independent": 0.8342,
            "misleading": 0.8026
          },
          "objective": 0.8705333333333334
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9784,
            "independent": 0.859,
            "misleading": 0.84
          },
          "objective": 0.8924666666666666
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.971,
            "independent": 0.835,
            "misleading": 0.8306
          },
          "objective": 0.8788666666666667
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9772,
            "independent": 0.8508,
            "misleading": 0.8388
          },
          "objective": 0.8889333333333332
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.1452,
            "independent": 0.1368,
            "misleading": 0.16
          },
          "objective": 0.14733333333333334
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9794,
            "independent": 0.8506,
            "misleading": 0.832
          },
          "objective": 0.8873333333333333
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.963,
            "independent": 0.7836,
            "misleading": 0.7532
          },
          "objective": 0.8332666666666667
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9746,
            "independent": 0.8448,
            "misleading": 0.8086
          },
          "objective": 0.876
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9606,
            "independent": 0.794,
            "misleading": 0.7336
          },
          "objective": 0.8294
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9784,
            "independent": 0.8576,
            "misleading": 0.8244
          },
          "objective": 0.8868
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9536,
            "independent": 0.7682,
            "misleading": 0.6824
          },
          "objective": 0.8014
        }
      ],
      "selected": {
        "rank": 2,
        "strength": 1.0,
        "validation_objective": 0.9128666666666666
      },
      "selected_test": {
        "matched": {
          "accuracy": 0.9727,
          "correct": 9727,
          "total": 10000,
          "mean_true_class_probability": 0.9595134794346828,
          "per_digit_accuracy": {
            "0": 0.9887755102040816,
            "1": 0.9929515418502203,
            "2": 0.9515503875968992,
            "3": 0.9811881188118812,
            "4": 0.9704684317718941,
            "5": 0.9562780269058296,
            "6": 0.9770354906054279,
            "7": 0.9698443579766537,
            "8": 0.9691991786447639,
            "9": 0.9663032705649157
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 120,
            "wrong_to_correct": 48,
            "correct_both": 9679,
            "wrong_both": 153
          }
        },
        "independent": {
          "accuracy": 0.8831,
          "correct": 8831,
          "total": 10000,
          "mean_true_class_probability": 0.8460129613391422,
          "per_digit_accuracy": {
            "0": 0.9438775510204082,
            "1": 0.9718061674008811,
            "2": 0.8585271317829457,
            "3": 0.904950495049505,
            "4": 0.8940936863543788,
            "5": 0.8475336322869955,
            "6": 0.9321503131524008,
            "7": 0.8667315175097277,
            "8": 0.7813141683778234,
            "9": 0.8166501486620417
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 264,
            "wrong_to_correct": 419,
            "correct_both": 8412,
            "wrong_both": 905
          }
        },
        "misleading": {
          "accuracy": 0.9007,
          "correct": 9007,
          "total": 10000,
          "mean_true_class_probability": 0.861249781766884,
          "per_digit_accuracy": {
            "0": 0.9551020408163265,
            "1": 0.9524229074889868,
            "2": 0.8701550387596899,
            "3": 0.9485148514851485,
            "4": 0.9022403258655805,
            "5": 0.9114349775784754,
            "6": 0.9582463465553236,
            "7": 0.896887159533074,
            "8": 0.7597535934291582,
            "9": 0.8473736372646185
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 50,
            "wrong_to_correct": 623,
            "correct_both": 8384,
            "wrong_both": 943
          }
        }
      },
      "selected_test_full_strength": {
        "matched": {
          "accuracy": 0.9727,
          "correct": 9727,
          "total": 10000,
          "mean_true_class_probability": 0.9595134794346828,
          "per_digit_accuracy": {
            "0": 0.9887755102040816,
            "1": 0.9929515418502203,
            "2": 0.9515503875968992,
            "3": 0.9811881188118812,
            "4": 0.9704684317718941,
            "5": 0.9562780269058296,
            "6": 0.9770354906054279,
            "7": 0.9698443579766537,
            "8": 0.9691991786447639,
            "9": 0.9663032705649157
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 120,
            "wrong_to_correct": 48,
            "correct_both": 9679,
            "wrong_both": 153
          }
        },
        "independent": {
          "accuracy": 0.8831,
          "correct": 8831,
          "total": 10000,
          "mean_true_class_probability": 0.8460129613391422,
          "per_digit_accuracy": {
            "0": 0.9438775510204082,
            "1": 0.9718061674008811,
            "2": 0.8585271317829457,
            "3": 0.904950495049505,
            "4": 0.8940936863543788,
            "5": 0.8475336322869955,
            "6": 0.9321503131524008,
            "7": 0.8667315175097277,
            "8": 0.7813141683778234,
            "9": 0.8166501486620417
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 264,
            "wrong_to_correct": 419,
            "correct_both": 8412,
            "wrong_both": 905
          }
        },
        "misleading": {
          "accuracy": 0.9007,
          "correct": 9007,
          "total": 10000,
          "mean_true_class_probability": 0.861249781766884,
          "per_digit_accuracy": {
            "0": 0.9551020408163265,
            "1": 0.9524229074889868,
            "2": 0.8701550387596899,
            "3": 0.9485148514851485,
            "4": 0.9022403258655805,
            "5": 0.9114349775784754,
            "6": 0.9582463465553236,
            "7": 0.896887159533074,
            "8": 0.7597535934291582,
            "9": 0.8473736372646185
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 50,
            "wrong_to_correct": 623,
            "correct_both": 8384,
            "wrong_both": 943
          }
        }
      },
      "random_control_test": [
        {
          "matched": {
            "accuracy": 0.9802,
            "correct": 9802,
            "total": 10000,
            "mean_true_class_probability": 0.9718968256590926,
            "per_digit_accuracy": {
              "0": 0.9897959183673469,
              "1": 0.9911894273127754,
              "2": 0.9786821705426356,
              "3": 0.9910891089108911,
              "4": 0.9755600814663951,
              "5": 0.9719730941704036,
              "6": 0.9832985386221295,
              "7": 0.9776264591439688,
              "8": 0.9691991786447639,
              "9": 0.9712586719524281
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 15,
              "wrong_to_correct": 18,
              "correct_both": 9784,
              "wrong_both": 183
            }
          },
          "independent": {
            "accuracy": 0.8701,
            "correct": 8701,
            "total": 10000,
            "mean_true_class_probability": 0.8284824028364539,
            "per_digit_accuracy": {
              "0": 0.9255102040816326,
              "1": 0.9656387665198238,
              "2": 0.8372093023255814,
              "3": 0.8871287128712871,
              "4": 0.8655804480651731,
              "5": 0.8430493273542601,
              "6": 0.9050104384133612,
              "7": 0.8589494163424124,
              "8": 0.7628336755646817,
              "9": 0.8354806739345887
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 95,
              "wrong_to_correct": 120,
              "correct_both": 8581,
              "wrong_both": 1204
            }
          },
          "misleading": {
            "accuracy": 0.851,
            "correct": 8510,
            "total": 10000,
            "mean_true_class_probability": 0.7971940416852409,
            "per_digit_accuracy": {
              "0": 0.8989795918367347,
              "1": 0.9400881057268723,
              "2": 0.7877906976744186,
              "3": 0.9277227722772278,
              "4": 0.8238289205702648,
              "5": 0.7858744394618834,
              "6": 0.9269311064718163,
              "7": 0.8667315175097277,
              "8": 0.7566735112936345,
              "9": 0.7789890981169475
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 110,
              "wrong_to_correct": 186,
              "correct_both": 8324,
              "wrong_both": 1380
            }
          }
        },
        {
          "matched": {
            "accuracy": 0.9794,
            "correct": 9794,
            "total": 10000,
            "mean_true_class_probability": 0.9717948624518611,
            "per_digit_accuracy": {
              "0": 0.9918367346938776,
              "1": 0.9911894273127754,
              "2": 0.9777131782945736,
              "3": 0.9891089108910891,
              "4": 0.9816700610997964,
              "5": 0.9708520179372198,
              "6": 0.9801670146137788,
              "7": 0.97568093385214,
              "8": 0.9691991786447639,
              "9": 0.9643211100099108
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 9,
              "wrong_to_correct": 4,
              "correct_both": 9790,
              "wrong_both": 197
            }
          },
          "independent": {
            "accuracy": 0.8668,
            "correct": 8668,
            "total": 10000,
            "mean_true_class_probability": 0.8297947034987141,
            "per_digit_accuracy": {
              "0": 0.926530612244898,
              "1": 0.9674008810572687,
              "2": 0.8527131782945736,
              "3": 0.8851485148514852,
              "4": 0.8767820773930753,
              "5": 0.8396860986547086,
              "6": 0.8862212943632568,
              "7": 0.8385214007782101,
              "8": 0.7587268993839835,
              "9": 0.8206144697720515
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 69,
              "wrong_to_correct": 61,
              "correct_both": 8607,
              "wrong_both": 1263
            }
          },
          "misleading": {
            "accuracy": 0.8445,
            "correct": 8445,
            "total": 10000,
            "mean_true_class_probability": 0.7968983119327231,
            "per_digit_accuracy": {
              "0": 0.9173469387755102,
              "1": 0.9418502202643172,
              "2": 0.8071705426356589,
              "3": 0.9366336633663367,
              "4": 0.8167006109979633,
              "5": 0.7903587443946188,
              "6": 0.8977035490605428,
              "7": 0.8278210116731517,
              "8": 0.75564681724846,
              "9": 0.7373637264618434
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 88,
              "wrong_to_correct": 99,
              "correct_both": 8346,
              "wrong_both": 1467
            }
          }
        },
        {
          "matched": {
            "accuracy": 0.9798,
            "correct": 9798,
            "total": 10000,
            "mean_true_class_probability": 0.9707662093290123,
            "per_digit_accuracy": {
              "0": 0.9918367346938776,
              "1": 0.9876651982378855,
              "2": 0.9777131782945736,
              "3": 0.995049504950495,
              "4": 0.9837067209775967,
              "5": 0.9641255605381166,
              "6": 0.9832985386221295,
              "7": 0.97568093385214,
              "8": 0.9722792607802875,
              "9": 0.9643211100099108
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 19,
              "wrong_to_correct": 18,
              "correct_both": 9780,
              "wrong_both": 183
            }
          },
          "independent": {
            "accuracy": 0.8681,
            "correct": 8681,
            "total": 10000,
            "mean_true_class_probability": 0.8261050176146396,
            "per_digit_accuracy": {
              "0": 0.9357142857142857,
              "1": 0.9295154185022027,
              "2": 0.8430232558139535,
              "3": 0.8891089108910891,
              "4": 0.9042769857433809,
              "5": 0.8150224215246636,
              "6": 0.9081419624217119,
              "7": 0.8404669260700389,
              "8": 0.7905544147843943,
              "9": 0.8146679881070367
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 129,
              "wrong_to_correct": 134,
              "correct_both": 8547,
              "wrong_both": 1190
            }
          },
          "misleading": {
            "accuracy": 0.8314,
            "correct": 8314,
            "total": 10000,
            "mean_true_class_probability": 0.7716904974588055,
            "per_digit_accuracy": {
              "0": 0.939795918367347,
              "1": 0.8281938325991189,
              "2": 0.7790697674418605,
              "3": 0.9396039603960396,
              "4": 0.8482688391038696,
              "5": 0.7600896860986547,
              "6": 0.8956158663883089,
              "7": 0.8404669260700389,
              "8": 0.768993839835729,
              "9": 0.711595639246779
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 203,
              "wrong_to_correct": 83,
              "correct_both": 8231,
              "wrong_both": 1483
            }
          }
        }
      ],
      "null_space_control_test": {
        "matched": {
          "accuracy": 0.1028,
          "correct": 1028,
          "total": 10000,
          "mean_true_class_probability": 0.10006554160118103,
          "per_digit_accuracy": {
            "0": 0.0,
            "1": 0.0,
            "2": 0.0,
            "3": 0.0,
            "4": 0.0,
            "5": 0.0,
            "6": 0.0,
            "7": 1.0,
            "8": 0.0,
            "9": 0.0
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 8796,
            "wrong_to_correct": 25,
            "correct_both": 1003,
            "wrong_both": 176
          }
        },
        "independent": {
          "accuracy": 0.1028,
          "correct": 1028,
          "total": 10000,
          "mean_true_class_probability": 0.1000655527971685,
          "per_digit_accuracy": {
            "0": 0.0,
            "1": 0.0,
            "2": 0.0,
            "3": 0.0,
            "4": 0.0,
            "5": 0.0,
            "6": 0.0,
            "7": 1.0,
            "8": 0.0,
            "9": 0.0
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 7802,
            "wrong_to_correct": 154,
            "correct_both": 874,
            "wrong_both": 1170
          }
        },
        "misleading": {
          "accuracy": 0.1028,
          "correct": 1028,
          "total": 10000,
          "mean_true_class_probability": 0.10006550987362861,
          "per_digit_accuracy": {
            "0": 0.0,
            "1": 0.0,
            "2": 0.0,
            "3": 0.0,
            "4": 0.0,
            "5": 0.0,
            "6": 0.0,
            "7": 1.0,
            "8": 0.0,
            "9": 0.0
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 7552,
            "wrong_to_correct": 146,
            "correct_both": 882,
            "wrong_both": 1420
          }
        }
      },
      "row_space_identity_test": {
        "matched": {
          "accuracy": 0.9799,
          "correct": 9799,
          "total": 10000,
          "mean_true_class_probability": 0.9718909079286694,
          "per_digit_accuracy": {
            "0": 0.9928571428571429,
            "1": 0.9929515418502203,
            "2": 0.9777131782945736,
            "3": 0.9920792079207921,
            "4": 0.9816700610997964,
            "5": 0.9708520179372198,
            "6": 0.9812108559498957,
            "7": 0.97568093385214,
            "8": 0.9671457905544147,
            "9": 0.9643211100099108
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 0,
            "wrong_to_correct": 0,
            "correct_both": 9799,
            "wrong_both": 201
          }
        },
        "independent": {
          "accuracy": 0.8676,
          "correct": 8676,
          "total": 10000,
          "mean_true_class_probability": 0.8315220749780933,
          "per_digit_accuracy": {
            "0": 0.9275510204081633,
            "1": 0.9726872246696036,
            "2": 0.8372093023255814,
            "3": 0.8940594059405941,
            "4": 0.8961303462321792,
            "5": 0.8408071748878924,
            "6": 0.8956158663883089,
            "7": 0.8501945525291829,
            "8": 0.7453798767967146,
            "9": 0.800792864222002
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 0,
            "wrong_to_correct": 0,
            "correct_both": 8676,
            "wrong_both": 1324
          }
        },
        "misleading": {
          "accuracy": 0.8434,
          "correct": 8434,
          "total": 10000,
          "mean_true_class_probability": 0.7973170226072231,
          "per_digit_accuracy": {
            "0": 0.926530612244898,
            "1": 0.9524229074889868,
            "2": 0.7819767441860465,
            "3": 0.9376237623762376,
            "4": 0.835030549898167,
            "5": 0.7825112107623319,
            "6": 0.8956158663883089,
            "7": 0.857976653696498,
            "8": 0.7453798767967146,
            "9": 0.7006937561942518
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 0,
            "wrong_to_correct": 0,
            "correct_both": 8434,
            "wrong_both": 1566
          }
        }
      },
      "replacement": {
        "rank": 2,
        "test": {
          "matched": {
            "all_palette": {
              "accuracy": 0.95993,
              "correct": 95993,
              "total": 100000,
              "mean_true_class_probability": 0.9445927019338847,
              "per_digit_accuracy": {
                "0": 0.9870408163265306,
                "1": 0.9927753303964758,
                "2": 0.9173449612403101,
                "3": 0.9676237623762376,
                "4": 0.9655804480651731,
                "5": 0.9325112107623318,
                "6": 0.963152400835073,
                "7": 0.9623540856031129,
                "8": 0.9412731006160164,
                "9": 0.9637264618434093
              },
              "unique_images": 10000,
              "donors_per_image": 10,
              "changed_predictions": 3171,
              "transitions_vs_baseline": {
                "correct_to_wrong": 2472,
                "wrong_to_correct": 475,
                "correct_both": 95518,
                "wrong_both": 1535
              },
              "mean_donor_class_probability": 0.11081307203736535,
              "baseline_mean_donor_class_probability": 0.1000000003776464,
              "prediction_agreement_with_recoloring": 0.87927,
              "mean_abs_logit_difference_from_recoloring": 2.868203487041624
            },
            "aligned_donor": {
              "accuracy": 0.9866,
              "correct": 9866,
              "total": 10000,
              "mean_true_class_probability": 0.9790613968684638,
              "per_digit_accuracy": {
                "0": 0.9918367346938776,
                "1": 0.9920704845814978,
                "2": 0.9854651162790697,
                "3": 0.998019801980198,
                "4": 0.9918533604887984,
                "5": 0.9899103139013453,
                "6": 0.9864300626304802,
                "7": 0.9824902723735408,
                "8": 0.9794661190965093,
                "9": 0.9682854311199207
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 94,
              "transitions_vs_baseline": {
                "correct_to_wrong": 7,
                "wrong_to_correct": 74,
                "correct_both": 9792,
                "wrong_both": 127
              },
              "mean_donor_class_probability": 0.9790613968684638,
              "baseline_mean_donor_class_probability": 0.9718909070855946,
              "prediction_agreement_with_recoloring": 0.9927,
              "mean_abs_logit_difference_from_recoloring": 0.2659278904620558
            },
            "shifted_donor": {
              "accuracy": 0.9372,
              "correct": 9372,
              "total": 10000,
              "mean_true_class_probability": 0.9148476951263408,
              "per_digit_accuracy": {
                "0": 0.9826530612244898,
                "1": 0.9938325991189427,
                "2": 0.7761627906976745,
                "3": 0.9683168316831683,
                "4": 0.9633401221995926,
                "5": 0.8688340807174888,
                "6": 0.9446764091858038,
                "7": 0.9513618677042801,
                "8": 0.9671457905544147,
                "9": 0.9474727452923687
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 495,
              "transitions_vs_baseline": {
                "correct_to_wrong": 448,
                "wrong_to_correct": 21,
                "correct_both": 9351,
                "wrong_both": 180
              },
              "mean_donor_class_probability": 0.025896101637920926,
              "baseline_mean_donor_class_probability": 0.00268986526233837,
              "prediction_agreement_with_recoloring": 0.8538,
              "mean_abs_logit_difference_from_recoloring": 3.1341849917360207
            },
            "complement_patch": {
              "all_palette": {
                "accuracy": 0.91286,
                "correct": 91286,
                "total": 100000,
                "mean_true_class_probability": 0.885830132530171,
                "per_digit_accuracy": {
                  "0": 0.9386734693877551,
                  "1": 0.966431718061674,
                  "2": 0.8944767441860465,
                  "3": 0.9582178217821782,
                  "4": 0.959775967413442,
                  "5": 0.9478699551569507,
                  "6": 0.9551148225469729,
                  "7": 0.8925097276264592,
                  "8": 0.801129363449692,
                  "9": 0.8127849355797819
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 8145,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 7225,
                  "wrong_to_correct": 521,
                  "correct_both": 90765,
                  "wrong_both": 1489
                },
                "mean_donor_class_probability": 0.13121925061673417,
                "baseline_mean_donor_class_probability": 0.1000000003776464,
                "prediction_agreement_with_recoloring": 0.90903,
                "mean_abs_logit_difference_from_recoloring": 1.3544991413756087
              },
              "aligned_donor": {
                "accuracy": 0.9904,
                "correct": 9904,
                "total": 10000,
                "mean_true_class_probability": 0.9841249456915772,
                "per_digit_accuracy": {
                  "0": 0.996938775510204,
                  "1": 0.9973568281938326,
                  "2": 0.9893410852713178,
                  "3": 0.996039603960396,
                  "4": 0.9887983706720977,
                  "5": 0.9910313901345291,
                  "6": 0.9853862212943633,
                  "7": 0.9883268482490273,
                  "8": 0.9866529774127311,
                  "9": 0.9831516352824579
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 125,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 6,
                  "wrong_to_correct": 111,
                  "correct_both": 9793,
                  "wrong_both": 90
                },
                "mean_donor_class_probability": 0.9841249456915772,
                "baseline_mean_donor_class_probability": 0.9718909070855946,
                "prediction_agreement_with_recoloring": 0.9966,
                "mean_abs_logit_difference_from_recoloring": 0.13549948032647371
              },
              "shifted_donor": {
                "accuracy": 0.9211,
                "correct": 9211,
                "total": 10000,
                "mean_true_class_probability": 0.8876911811644884,
                "per_digit_accuracy": {
                  "0": 0.9408163265306122,
                  "1": 0.9436123348017621,
                  "2": 0.9069767441860465,
                  "3": 0.9663366336633663,
                  "4": 0.9633401221995926,
                  "5": 0.9607623318385651,
                  "6": 0.9686847599164927,
                  "7": 0.9241245136186771,
                  "8": 0.8039014373716632,
                  "9": 0.8344895936570862
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 741,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 644,
                  "wrong_to_correct": 56,
                  "correct_both": 9155,
                  "wrong_both": 145
                },
                "mean_donor_class_probability": 0.021165877820169703,
                "baseline_mean_donor_class_probability": 0.00268986526233837,
                "prediction_agreement_with_recoloring": 0.8896,
                "mean_abs_logit_difference_from_recoloring": 1.8472090273885802
              }
            },
            "recolored_reference": {
              "all_palette": {
                "accuracy": 0.86804,
                "correct": 86804,
                "total": 100000,
                "mean_true_class_probability": 0.8298507602755971,
                "per_digit_accuracy": {
                  "0": 0.9271428571428572,
                  "1": 0.967136563876652,
                  "2": 0.8362403100775194,
                  "3": 0.9030693069306931,
                  "4": 0.8959266802443991,
                  "5": 0.8410313901345291,
                  "6": 0.8972860125260961,
                  "7": 0.8552529182879377,
                  "8": 0.7337782340862423,
                  "9": 0.8082259663032706
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 12929,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 11834,
                  "wrong_to_correct": 648,
                  "correct_both": 86156,
                  "wrong_both": 1362
                },
                "mean_donor_class_probability": 0.17421834353305088,
                "baseline_mean_donor_class_probability": 0.1000000003776464,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "aligned_donor": {
                "accuracy": 0.9938,
                "correct": 9938,
                "total": 10000,
                "mean_true_class_probability": 0.9879955979234771,
                "per_digit_accuracy": {
                  "0": 0.9979591836734694,
                  "1": 0.9973568281938326,
                  "2": 0.9951550387596899,
                  "3": 0.999009900990099,
                  "4": 0.9938900203665988,
                  "5": 0.9955156950672646,
                  "6": 0.9895615866388309,
                  "7": 0.9912451361867705,
                  "8": 0.9948665297741273,
                  "9": 0.9831516352824579
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 140,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 0,
                  "wrong_to_correct": 139,
                  "correct_both": 9799,
                  "wrong_both": 62
                },
                "mean_donor_class_probability": 0.9879955979234771,
                "baseline_mean_donor_class_probability": 0.9718909070855946,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "shifted_donor": {
                "accuracy": 0.8401,
                "correct": 8401,
                "total": 10000,
                "mean_true_class_probability": 0.7925792598889998,
                "per_digit_accuracy": {
                  "0": 0.9275510204081633,
                  "1": 0.9515418502202643,
                  "2": 0.7693798449612403,
                  "3": 0.9405940594059405,
                  "4": 0.835030549898167,
                  "5": 0.7769058295964125,
                  "6": 0.8893528183716075,
                  "7": 0.8589494163424124,
                  "8": 0.7402464065708418,
                  "9": 0.6927651139742319
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1539,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 1446,
                  "wrong_to_correct": 48,
                  "correct_both": 8353,
                  "wrong_both": 153
                },
                "mean_donor_class_probability": 0.0641115700985687,
                "baseline_mean_donor_class_probability": 0.00268986526233837,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              }
            },
            "random_controls": [
              {
                "all_palette": {
                  "accuracy": 0.97947,
                  "correct": 97947,
                  "total": 100000,
                  "mean_true_class_probability": 0.9714739286717615,
                  "per_digit_accuracy": {
                    "0": 0.9892857142857143,
                    "1": 0.992863436123348,
                    "2": 0.9766472868217054,
                    "3": 0.9923762376237624,
                    "4": 0.9793279022403258,
                    "5": 0.9707399103139014,
                    "6": 0.9804801670146138,
                    "7": 0.9750972762645914,
                    "8": 0.9680698151950718,
                    "9": 0.967195242814668
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 339,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 165,
                    "wrong_to_correct": 122,
                    "correct_both": 97825,
                    "wrong_both": 1888
                  },
                  "mean_donor_class_probability": 0.09999800863514083,
                  "baseline_mean_donor_class_probability": 0.1000000003776464,
                  "prediction_agreement_with_recoloring": 0.87099,
                  "mean_abs_logit_difference_from_recoloring": 3.1924750240355766
                },
                "aligned_donor": {
                  "accuracy": 0.9796,
                  "correct": 9796,
                  "total": 10000,
                  "mean_true_class_probability": 0.9712064891152764,
                  "per_digit_accuracy": {
                    "0": 0.9836734693877551,
                    "1": 0.9911894273127754,
                    "2": 0.9767441860465116,
                    "3": 0.9930693069306931,
                    "4": 0.984725050916497,
                    "5": 0.9730941704035875,
                    "6": 0.9822546972860126,
                    "7": 0.9747081712062257,
                    "8": 0.9691991786447639,
                    "9": 0.9653121902874133
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 32,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 14,
                    "wrong_to_correct": 11,
                    "correct_both": 9785,
                    "wrong_both": 190
                  },
                  "mean_donor_class_probability": 0.9712064891152764,
                  "baseline_mean_donor_class_probability": 0.9718909070855946,
                  "prediction_agreement_with_recoloring": 0.9857,
                  "mean_abs_logit_difference_from_recoloring": 0.3024369600538909
                },
                "shifted_donor": {
                  "accuracy": 0.9791,
                  "correct": 9791,
                  "total": 10000,
                  "mean_true_class_probability": 0.9709637719817064,
                  "per_digit_accuracy": {
                    "0": 0.9928571428571429,
                    "1": 0.9929515418502203,
                    "2": 0.9767441860465116,
                    "3": 0.994059405940594,
                    "4": 0.9755600814663951,
                    "5": 0.9697309417040358,
                    "6": 0.9780793319415448,
                    "7": 0.97568093385214,
                    "8": 0.9671457905544147,
                    "9": 0.9653121902874133
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 27,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 15,
                    "wrong_to_correct": 7,
                    "correct_both": 9784,
                    "wrong_both": 194
                  },
                  "mean_donor_class_probability": 0.0027078344938898874,
                  "baseline_mean_donor_class_probability": 0.00268986526233837,
                  "prediction_agreement_with_recoloring": 0.8469,
                  "mean_abs_logit_difference_from_recoloring": 3.6638670245274154
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.9794,
                  "correct": 97940,
                  "total": 100000,
                  "mean_true_class_probability": 0.9716597936447867,
                  "per_digit_accuracy": {
                    "0": 0.9927551020408163,
                    "1": 0.9919823788546256,
                    "2": 0.9760658914728683,
                    "3": 0.9912871287128713,
                    "4": 0.9816700610997964,
                    "5": 0.9700672645739911,
                    "6": 0.981106471816284,
                    "7": 0.9760700389105058,
                    "8": 0.9661190965092402,
                    "9": 0.9644202180376611
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 123,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 74,
                    "wrong_to_correct": 24,
                    "correct_both": 97916,
                    "wrong_both": 1986
                  },
                  "mean_donor_class_probability": 0.10003786671307888,
                  "baseline_mean_donor_class_probability": 0.1000000003776464,
                  "prediction_agreement_with_recoloring": 0.87056,
                  "mean_abs_logit_difference_from_recoloring": 3.2434973624448573
                },
                "aligned_donor": {
                  "accuracy": 0.9795,
                  "correct": 9795,
                  "total": 10000,
                  "mean_true_class_probability": 0.9716147930537605,
                  "per_digit_accuracy": {
                    "0": 0.9928571428571429,
                    "1": 0.9911894273127754,
                    "2": 0.9777131782945736,
                    "3": 0.9930693069306931,
                    "4": 0.9816700610997964,
                    "5": 0.968609865470852,
                    "6": 0.9812108559498957,
                    "7": 0.97568093385214,
                    "8": 0.9650924024640657,
                    "9": 0.9653121902874133
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 13,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 6,
                    "wrong_to_correct": 2,
                    "correct_both": 9793,
                    "wrong_both": 199
                  },
                  "mean_donor_class_probability": 0.9716147930537605,
                  "baseline_mean_donor_class_probability": 0.9718909070855946,
                  "prediction_agreement_with_recoloring": 0.9856,
                  "mean_abs_logit_difference_from_recoloring": 0.30664995960976926
                },
                "shifted_donor": {
                  "accuracy": 0.979,
                  "correct": 9790,
                  "total": 10000,
                  "mean_true_class_probability": 0.9713309623789758,
                  "per_digit_accuracy": {
                    "0": 0.9928571428571429,
                    "1": 0.9920704845814978,
                    "2": 0.9728682170542635,
                    "3": 0.9910891089108911,
                    "4": 0.9816700610997964,
                    "5": 0.9697309417040358,
                    "6": 0.9812108559498957,
                    "7": 0.97568093385214,
                    "8": 0.9661190965092402,
                    "9": 0.9643211100099108
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 15,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 12,
                    "wrong_to_correct": 3,
                    "correct_both": 9787,
                    "wrong_both": 198
                  },
                  "mean_donor_class_probability": 0.003173852715980555,
                  "baseline_mean_donor_class_probability": 0.00268986526233837,
                  "prediction_agreement_with_recoloring": 0.8459,
                  "mean_abs_logit_difference_from_recoloring": 3.689805143836029
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.97972,
                  "correct": 97972,
                  "total": 100000,
                  "mean_true_class_probability": 0.9711541900181823,
                  "per_digit_accuracy": {
                    "0": 0.9912244897959184,
                    "1": 0.9931277533039647,
                    "2": 0.9767441860465116,
                    "3": 0.992970297029703,
                    "4": 0.9824847250509166,
                    "5": 0.969170403587444,
                    "6": 0.9817327766179541,
                    "7": 0.97568093385214,
                    "8": 0.9663244353182752,
                    "9": 0.9650148662041625
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 366,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 164,
                    "wrong_to_correct": 146,
                    "correct_both": 97826,
                    "wrong_both": 1864
                  },
                  "mean_donor_class_probability": 0.10056717100846403,
                  "baseline_mean_donor_class_probability": 0.1000000003776464,
                  "prediction_agreement_with_recoloring": 0.87158,
                  "mean_abs_logit_difference_from_recoloring": 3.224310502800084
                },
                "aligned_donor": {
                  "accuracy": 0.9819,
                  "correct": 9819,
                  "total": 10000,
                  "mean_true_class_probability": 0.9736922047380252,
                  "per_digit_accuracy": {
                    "0": 0.9938775510204082,
                    "1": 0.9929515418502203,
                    "2": 0.9777131782945736,
                    "3": 0.9930693069306931,
                    "4": 0.9826883910386965,
                    "5": 0.9809417040358744,
                    "6": 0.9822546972860126,
                    "7": 0.97568093385214,
                    "8": 0.9702258726899384,
                    "9": 0.9682854311199207
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 28,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 1,
                    "wrong_to_correct": 21,
                    "correct_both": 9798,
                    "wrong_both": 180
                  },
                  "mean_donor_class_probability": 0.9736922047380252,
                  "baseline_mean_donor_class_probability": 0.9718909070855946,
                  "prediction_agreement_with_recoloring": 0.988,
                  "mean_abs_logit_difference_from_recoloring": 0.30539085211753847
                },
                "shifted_donor": {
                  "accuracy": 0.9792,
                  "correct": 9792,
                  "total": 10000,
                  "mean_true_class_probability": 0.9702939103189752,
                  "per_digit_accuracy": {
                    "0": 0.9918367346938776,
                    "1": 0.9938325991189427,
                    "2": 0.9777131782945736,
                    "3": 0.9930693069306931,
                    "4": 0.9806517311608961,
                    "5": 0.9663677130044843,
                    "6": 0.9812108559498957,
                    "7": 0.97568093385214,
                    "8": 0.9609856262833676,
                    "9": 0.9672943508424182
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 34,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 17,
                    "wrong_to_correct": 10,
                    "correct_both": 9782,
                    "wrong_both": 191
                  },
                  "mean_donor_class_probability": 0.00274355699994352,
                  "baseline_mean_donor_class_probability": 0.00268986526233837,
                  "prediction_agreement_with_recoloring": 0.8464,
                  "mean_abs_logit_difference_from_recoloring": 3.663930208666697
                }
              }
            ]
          },
          "independent": {
            "all_palette": {
              "accuracy": 0.85038,
              "correct": 85038,
              "total": 100000,
              "mean_true_class_probability": 0.8165358199912754,
              "per_digit_accuracy": {
                "0": 0.9271428571428572,
                "1": 0.9696916299559472,
                "2": 0.8295542635658915,
                "3": 0.8686138613861386,
                "4": 0.8799389002036659,
                "5": 0.8187219730941704,
                "6": 0.8815240083507306,
                "7": 0.8380350194552529,
                "8": 0.7101642710472279,
                "9": 0.7622398414271556
              },
              "unique_images": 10000,
              "donors_per_image": 10,
              "changed_predictions": 10995,
              "transitions_vs_baseline": {
                "correct_to_wrong": 5501,
                "wrong_to_correct": 3779,
                "correct_both": 81259,
                "wrong_both": 9461
              },
              "mean_donor_class_probability": 0.12714107723907506,
              "baseline_mean_donor_class_probability": 0.10000000020868428,
              "prediction_agreement_with_recoloring": 0.86074,
              "mean_abs_logit_difference_from_recoloring": 2.9642327105181785
            },
            "aligned_donor": {
              "accuracy": 0.9218,
              "correct": 9218,
              "total": 10000,
              "mean_true_class_probability": 0.8955256666929919,
              "per_digit_accuracy": {
                "0": 0.95,
                "1": 0.9682819383259912,
                "2": 0.8972868217054264,
                "3": 0.9633663366336633,
                "4": 0.9674134419551935,
                "5": 0.9630044843049327,
                "6": 0.9739039665970772,
                "7": 0.9036964980544747,
                "8": 0.8172484599589322,
                "9": 0.8146679881070367
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 883,
              "transitions_vs_baseline": {
                "correct_to_wrong": 106,
                "wrong_to_correct": 648,
                "correct_both": 8570,
                "wrong_both": 676
              },
              "mean_donor_class_probability": 0.8955256666929919,
              "baseline_mean_donor_class_probability": 0.8315220538407555,
              "prediction_agreement_with_recoloring": 0.9257,
              "mean_abs_logit_difference_from_recoloring": 2.865309686962962
            },
            "shifted_donor": {
              "accuracy": 0.8026,
              "correct": 8026,
              "total": 10000,
              "mean_true_class_probability": 0.7612006512616256,
              "per_digit_accuracy": {
                "0": 0.9122448979591836,
                "1": 0.9718061674008811,
                "2": 0.7063953488372093,
                "3": 0.8772277227722772,
                "4": 0.8411405295315683,
                "5": 0.7073991031390134,
                "6": 0.7713987473903967,
                "7": 0.7986381322957199,
                "8": 0.7720739219712526,
                "9": 0.6392467789890981
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 1227,
              "transitions_vs_baseline": {
                "correct_to_wrong": 856,
                "wrong_to_correct": 206,
                "correct_both": 7820,
                "wrong_both": 1118
              },
              "mean_donor_class_probability": 0.032867776328202744,
              "baseline_mean_donor_class_probability": 0.015427719631169953,
              "prediction_agreement_with_recoloring": 0.8379,
              "mean_abs_logit_difference_from_recoloring": 2.955585696440134
            },
            "complement_patch": {
              "all_palette": {
                "accuracy": 0.84882,
                "correct": 84882,
                "total": 100000,
                "mean_true_class_probability": 0.8149315330495494,
                "per_digit_accuracy": {
                  "0": 0.9260204081632653,
                  "1": 0.9684581497797357,
                  "2": 0.830329457364341,
                  "3": 0.8656435643564356,
                  "4": 0.8756619144602852,
                  "5": 0.8152466367713005,
                  "6": 0.8805845511482254,
                  "7": 0.8381322957198444,
                  "8": 0.6992813141683778,
                  "9": 0.769970267591675
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 13790,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 6634,
                  "wrong_to_correct": 4756,
                  "correct_both": 80126,
                  "wrong_both": 8484
                },
                "mean_donor_class_probability": 0.14334848547255485,
                "baseline_mean_donor_class_probability": 0.10000000020868428,
                "prediction_agreement_with_recoloring": 0.88828,
                "mean_abs_logit_difference_from_recoloring": 1.3357951449851002
              },
              "aligned_donor": {
                "accuracy": 0.9711,
                "correct": 9711,
                "total": 10000,
                "mean_true_class_probability": 0.9576585406914763,
                "per_digit_accuracy": {
                  "0": 0.9948979591836735,
                  "1": 0.9964757709251101,
                  "2": 0.9312015503875969,
                  "3": 0.9673267326732673,
                  "4": 0.9714867617107943,
                  "5": 0.952914798206278,
                  "6": 0.9686847599164927,
                  "7": 0.9708171206225681,
                  "8": 0.9702258726899384,
                  "9": 0.9831516352824579
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1181,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 46,
                  "wrong_to_correct": 1081,
                  "correct_both": 8630,
                  "wrong_both": 243
                },
                "mean_donor_class_probability": 0.9576585406914763,
                "baseline_mean_donor_class_probability": 0.8315220538407555,
                "prediction_agreement_with_recoloring": 0.976,
                "mean_abs_logit_difference_from_recoloring": 1.3672252164696344
              },
              "shifted_donor": {
                "accuracy": 0.8753,
                "correct": 8753,
                "total": 10000,
                "mean_true_class_probability": 0.8336170237201617,
                "per_digit_accuracy": {
                  "0": 0.9336734693877551,
                  "1": 0.9444933920704845,
                  "2": 0.8614341085271318,
                  "3": 0.9227722772277228,
                  "4": 0.8879837067209776,
                  "5": 0.8845291479820628,
                  "6": 0.9331941544885177,
                  "7": 0.8793774319066148,
                  "8": 0.7125256673511293,
                  "9": 0.7849355797819624
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1079,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 382,
                  "wrong_to_correct": 459,
                  "correct_both": 8294,
                  "wrong_both": 865
                },
                "mean_donor_class_probability": 0.041939935453405325,
                "baseline_mean_donor_class_probability": 0.015427719631169953,
                "prediction_agreement_with_recoloring": 0.9069,
                "mean_abs_logit_difference_from_recoloring": 1.3352189597034827
              }
            },
            "recolored_reference": {
              "all_palette": {
                "accuracy": 0.86804,
                "correct": 86804,
                "total": 100000,
                "mean_true_class_probability": 0.8298507602755971,
                "per_digit_accuracy": {
                  "0": 0.9271428571428572,
                  "1": 0.967136563876652,
                  "2": 0.8362403100775194,
                  "3": 0.9030693069306931,
                  "4": 0.8959266802443991,
                  "5": 0.8410313901345291,
                  "6": 0.8972860125260961,
                  "7": 0.8552529182879377,
                  "8": 0.7337782340862423,
                  "9": 0.8082259663032706
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 14623,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 5861,
                  "wrong_to_correct": 5905,
                  "correct_both": 80899,
                  "wrong_both": 7335
                },
                "mean_donor_class_probability": 0.17421834353305088,
                "baseline_mean_donor_class_probability": 0.10000000020868428,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "aligned_donor": {
                "accuracy": 0.9938,
                "correct": 9938,
                "total": 10000,
                "mean_true_class_probability": 0.9879955979234771,
                "per_digit_accuracy": {
                  "0": 0.9979591836734694,
                  "1": 0.9973568281938326,
                  "2": 0.9951550387596899,
                  "3": 0.999009900990099,
                  "4": 0.9938900203665988,
                  "5": 0.9955156950672646,
                  "6": 0.9895615866388309,
                  "7": 0.9912451361867705,
                  "8": 0.9948665297741273,
                  "9": 0.9831516352824579
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1280,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 1,
                  "wrong_to_correct": 1263,
                  "correct_both": 8675,
                  "wrong_both": 61
                },
                "mean_donor_class_probability": 0.9879955979234771,
                "baseline_mean_donor_class_probability": 0.8315220538407555,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "shifted_donor": {
                "accuracy": 0.8401,
                "correct": 8401,
                "total": 10000,
                "mean_true_class_probability": 0.7925792598889998,
                "per_digit_accuracy": {
                  "0": 0.9275510204081633,
                  "1": 0.9515418502202643,
                  "2": 0.7693798449612403,
                  "3": 0.9405940594059405,
                  "4": 0.835030549898167,
                  "5": 0.7769058295964125,
                  "6": 0.8893528183716075,
                  "7": 0.8589494163424124,
                  "8": 0.7402464065708418,
                  "9": 0.6927651139742319
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1436,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 689,
                  "wrong_to_correct": 414,
                  "correct_both": 7987,
                  "wrong_both": 910
                },
                "mean_donor_class_probability": 0.0641115700985687,
                "baseline_mean_donor_class_probability": 0.015427719631169953,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              }
            },
            "random_controls": [
              {
                "all_palette": {
                  "accuracy": 0.86632,
                  "correct": 86632,
                  "total": 100000,
                  "mean_true_class_probability": 0.8294384059259018,
                  "per_digit_accuracy": {
                    "0": 0.9259183673469388,
                    "1": 0.9703964757709251,
                    "2": 0.836531007751938,
                    "3": 0.8940594059405941,
                    "4": 0.90010183299389,
                    "5": 0.8380044843049327,
                    "6": 0.8830897703549061,
                    "7": 0.8461089494163424,
                    "8": 0.749794661190965,
                    "9": 0.8033696729435085
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 2154,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 961,
                    "wrong_to_correct": 833,
                    "correct_both": 85799,
                    "wrong_both": 12407
                  },
                  "mean_donor_class_probability": 0.0998844850474597,
                  "baseline_mean_donor_class_probability": 0.10000000020868428,
                  "prediction_agreement_with_recoloring": 0.85266,
                  "mean_abs_logit_difference_from_recoloring": 3.2353171348398777
                },
                "aligned_donor": {
                  "accuracy": 0.8648,
                  "correct": 8648,
                  "total": 10000,
                  "mean_true_class_probability": 0.827063278846707,
                  "per_digit_accuracy": {
                    "0": 0.9132653061224489,
                    "1": 0.9647577092511013,
                    "2": 0.8333333333333334,
                    "3": 0.8960396039603961,
                    "4": 0.9317718940936863,
                    "5": 0.8295964125560538,
                    "6": 0.8872651356993737,
                    "7": 0.8414396887159533,
                    "8": 0.75564681724846,
                    "9": 0.77998017839445
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 235,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 114,
                    "wrong_to_correct": 86,
                    "correct_both": 8562,
                    "wrong_both": 1238
                  },
                  "mean_donor_class_probability": 0.827063278846707,
                  "baseline_mean_donor_class_probability": 0.8315220538407555,
                  "prediction_agreement_with_recoloring": 0.8692,
                  "mean_abs_logit_difference_from_recoloring": 3.1930622544648872
                },
                "shifted_donor": {
                  "accuracy": 0.8611,
                  "correct": 8611,
                  "total": 10000,
                  "mean_true_class_probability": 0.8253170313985575,
                  "per_digit_accuracy": {
                    "0": 0.9357142857142857,
                    "1": 0.9700440528634361,
                    "2": 0.8333333333333334,
                    "3": 0.897029702970297,
                    "4": 0.8727087576374746,
                    "5": 0.8374439461883408,
                    "6": 0.8622129436325678,
                    "7": 0.8521400778210116,
                    "8": 0.74435318275154,
                    "9": 0.7888999008919723
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 203,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 117,
                    "wrong_to_correct": 52,
                    "correct_both": 8559,
                    "wrong_both": 1272
                  },
                  "mean_donor_class_probability": 0.015407256320008065,
                  "baseline_mean_donor_class_probability": 0.015427719631169953,
                  "prediction_agreement_with_recoloring": 0.8542,
                  "mean_abs_logit_difference_from_recoloring": 3.258466065289229
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.86814,
                  "correct": 86814,
                  "total": 100000,
                  "mean_true_class_probability": 0.8315856537050712,
                  "per_digit_accuracy": {
                    "0": 0.9290816326530612,
                    "1": 0.9723348017621145,
                    "2": 0.8371124031007752,
                    "3": 0.8944554455445545,
                    "4": 0.8961303462321792,
                    "5": 0.8408071748878924,
                    "6": 0.8921711899791231,
                    "7": 0.8495136186770428,
                    "8": 0.750205338809035,
                    "9": 0.8040634291377602
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 912,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 366,
                    "wrong_to_correct": 420,
                    "correct_both": 86394,
                    "wrong_both": 12820
                  },
                  "mean_donor_class_probability": 0.09991090466514485,
                  "baseline_mean_donor_class_probability": 0.10000000020868428,
                  "prediction_agreement_with_recoloring": 0.85314,
                  "mean_abs_logit_difference_from_recoloring": 3.282676062422838
                },
                "aligned_donor": {
                  "accuracy": 0.8663,
                  "correct": 8663,
                  "total": 10000,
                  "mean_true_class_probability": 0.8295476756357284,
                  "per_digit_accuracy": {
                    "0": 0.9275510204081633,
                    "1": 0.9718061674008811,
                    "2": 0.8468992248062015,
                    "3": 0.899009900990099,
                    "4": 0.8971486761710794,
                    "5": 0.8374439461883408,
                    "6": 0.894572025052192,
                    "7": 0.8453307392996109,
                    "8": 0.7320328542094456,
                    "9": 0.7948463825569871
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 99,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 48,
                    "wrong_to_correct": 35,
                    "correct_both": 8628,
                    "wrong_both": 1289
                  },
                  "mean_donor_class_probability": 0.8295476756357284,
                  "baseline_mean_donor_class_probability": 0.8315220538407555,
                  "prediction_agreement_with_recoloring": 0.8708,
                  "mean_abs_logit_difference_from_recoloring": 3.24516764892742
                },
                "shifted_donor": {
                  "accuracy": 0.8687,
                  "correct": 8687,
                  "total": 10000,
                  "mean_true_class_probability": 0.8321869307243689,
                  "per_digit_accuracy": {
                    "0": 0.9295918367346939,
                    "1": 0.9744493392070485,
                    "2": 0.8304263565891473,
                    "3": 0.8930693069306931,
                    "4": 0.8991853360488798,
                    "5": 0.8419282511210763,
                    "6": 0.8903966597077244,
                    "7": 0.8501945525291829,
                    "8": 0.75564681724846,
                    "9": 0.8067393458870169
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 89,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 33,
                    "wrong_to_correct": 44,
                    "correct_both": 8643,
                    "wrong_both": 1280
                  },
                  "mean_donor_class_probability": 0.016137214085314072,
                  "baseline_mean_donor_class_probability": 0.015427719631169953,
                  "prediction_agreement_with_recoloring": 0.8566,
                  "mean_abs_logit_difference_from_recoloring": 3.2919874864626677
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.87065,
                  "correct": 87065,
                  "total": 100000,
                  "mean_true_class_probability": 0.8340495766025762,
                  "per_digit_accuracy": {
                    "0": 0.9341836734693878,
                    "1": 0.9719823788546256,
                    "2": 0.8359496124031007,
                    "3": 0.8931683168316832,
                    "4": 0.9015274949083503,
                    "5": 0.8507847533632287,
                    "6": 0.8972860125260961,
                    "7": 0.8502918287937743,
                    "8": 0.7485626283367557,
                    "9": 0.8087215064420218
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 2103,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 746,
                    "wrong_to_correct": 1051,
                    "correct_both": 86014,
                    "wrong_both": 12189
                  },
                  "mean_donor_class_probability": 0.1028754556930594,
                  "baseline_mean_donor_class_probability": 0.10000000020868428,
                  "prediction_agreement_with_recoloring": 0.85804,
                  "mean_abs_logit_difference_from_recoloring": 3.252779303600382
                },
                "aligned_donor": {
                  "accuracy": 0.8806,
                  "correct": 8806,
                  "total": 10000,
                  "mean_true_class_probability": 0.8456643067306421,
                  "per_digit_accuracy": {
                    "0": 0.9428571428571428,
                    "1": 0.973568281938326,
                    "2": 0.8352713178294574,
                    "3": 0.8930693069306931,
                    "4": 0.9103869653767821,
                    "5": 0.897982062780269,
                    "6": 0.8977035490605428,
                    "7": 0.8501945525291829,
                    "8": 0.768993839835729,
                    "9": 0.8275520317145689
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 226,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 32,
                    "wrong_to_correct": 162,
                    "correct_both": 8644,
                    "wrong_both": 1162
                  },
                  "mean_donor_class_probability": 0.8456643067306421,
                  "baseline_mean_donor_class_probability": 0.8315220538407555,
                  "prediction_agreement_with_recoloring": 0.8851,
                  "mean_abs_logit_difference_from_recoloring": 3.2250974885265156
                },
                "shifted_donor": {
                  "accuracy": 0.8668,
                  "correct": 8668,
                  "total": 10000,
                  "mean_true_class_probability": 0.8294363739756032,
                  "per_digit_accuracy": {
                    "0": 0.9316326530612244,
                    "1": 0.9762114537444934,
                    "2": 0.8410852713178295,
                    "3": 0.8930693069306931,
                    "4": 0.8788187372708758,
                    "5": 0.8352017937219731,
                    "6": 0.9008350730688935,
                    "7": 0.8511673151750972,
                    "8": 0.7330595482546202,
                    "9": 0.8097125867195243
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 194,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 84,
                    "wrong_to_correct": 76,
                    "correct_both": 8592,
                    "wrong_both": 1248
                  },
                  "mean_donor_class_probability": 0.016301219319359236,
                  "baseline_mean_donor_class_probability": 0.015427719631169953,
                  "prediction_agreement_with_recoloring": 0.8579,
                  "mean_abs_logit_difference_from_recoloring": 3.2692409400565734
                }
              }
            ]
          },
          "misleading": {
            "all_palette": {
              "accuracy": 0.87319,
              "correct": 87319,
              "total": 100000,
              "mean_true_class_probability": 0.8336244813861953,
              "per_digit_accuracy": {
                "0": 0.9285714285714286,
                "1": 0.9440528634361234,
                "2": 0.8542635658914729,
                "3": 0.9290099009900991,
                "4": 0.8926680244399186,
                "5": 0.8743273542600897,
                "6": 0.9310020876826722,
                "7": 0.8772373540856031,
                "8": 0.7207392197125256,
                "9": 0.7713577799801784
              },
              "unique_images": 10000,
              "donors_per_image": 10,
              "changed_predictions": 9270,
              "transitions_vs_baseline": {
                "correct_to_wrong": 2221,
                "wrong_to_correct": 5200,
                "correct_both": 82119,
                "wrong_both": 10460
              },
              "mean_donor_class_probability": 0.12267779188548551,
              "baseline_mean_donor_class_probability": 0.10000000018436021,
              "prediction_agreement_with_recoloring": 0.88859,
              "mean_abs_logit_difference_from_recoloring": 2.937375020098392
            },
            "aligned_donor": {
              "accuracy": 0.9247,
              "correct": 9247,
              "total": 10000,
              "mean_true_class_probability": 0.8939360016122871,
              "per_digit_accuracy": {
                "0": 0.9377551020408164,
                "1": 0.9462555066079296,
                "2": 0.9127906976744186,
                "3": 0.9683168316831683,
                "4": 0.9725050916496945,
                "5": 0.9719730941704036,
                "6": 0.9676409185803758,
                "7": 0.9202334630350194,
                "8": 0.8141683778234087,
                "9": 0.8384539147670962
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 1091,
              "transitions_vs_baseline": {
                "correct_to_wrong": 47,
                "wrong_to_correct": 860,
                "correct_both": 8387,
                "wrong_both": 706
              },
              "mean_donor_class_probability": 0.8939360016122871,
              "baseline_mean_donor_class_probability": 0.7973169746797446,
              "prediction_agreement_with_recoloring": 0.9285,
              "mean_abs_logit_difference_from_recoloring": 3.114310055396482
            },
            "shifted_donor": {
              "accuracy": 0.8376,
              "correct": 8376,
              "total": 10000,
              "mean_true_class_probability": 0.790609998055484,
              "per_digit_accuracy": {
                "0": 0.9255102040816326,
                "1": 0.9541850220264317,
                "2": 0.7635658914728682,
                "3": 0.9336633663366337,
                "4": 0.8319755600814664,
                "5": 0.7679372197309418,
                "6": 0.8872651356993737,
                "7": 0.853112840466926,
                "8": 0.7525667351129364,
                "9": 0.686818632309217
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 129,
              "transitions_vs_baseline": {
                "correct_to_wrong": 83,
                "wrong_to_correct": 25,
                "correct_both": 8351,
                "wrong_both": 1541
              },
              "mean_donor_class_probability": 0.05995777201935748,
              "baseline_mean_donor_class_probability": 0.05797885499426834,
              "prediction_agreement_with_recoloring": 0.982,
              "mean_abs_logit_difference_from_recoloring": 0.3205549363699555
            },
            "complement_patch": {
              "all_palette": {
                "accuracy": 0.80464,
                "correct": 80464,
                "total": 100000,
                "mean_true_class_probability": 0.7653605525154736,
                "per_digit_accuracy": {
                  "0": 0.9110204081632653,
                  "1": 0.9679295154185023,
                  "2": 0.7155038759689922,
                  "3": 0.8802970297029703,
                  "4": 0.8355397148676171,
                  "5": 0.6988789237668162,
                  "6": 0.7940501043841336,
                  "7": 0.8071011673151751,
                  "8": 0.7510266940451745,
                  "9": 0.6557978196233895
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 16363,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 8604,
                  "wrong_to_correct": 4728,
                  "correct_both": 75736,
                  "wrong_both": 10932
                },
                "mean_donor_class_probability": 0.15135744814479948,
                "baseline_mean_donor_class_probability": 0.10000000018436021,
                "prediction_agreement_with_recoloring": 0.87507,
                "mean_abs_logit_difference_from_recoloring": 1.3434122557605739
              },
              "aligned_donor": {
                "accuracy": 0.9529,
                "correct": 9529,
                "total": 10000,
                "mean_true_class_probability": 0.9335696927549623,
                "per_digit_accuracy": {
                  "0": 0.9897959183673469,
                  "1": 0.9973568281938326,
                  "2": 0.8042635658914729,
                  "3": 0.9772277227722772,
                  "4": 0.9745417515274949,
                  "5": 0.899103139013453,
                  "6": 0.9572025052192067,
                  "7": 0.9678988326848249,
                  "8": 0.9784394250513347,
                  "9": 0.977205153617443
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1421,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 128,
                  "wrong_to_correct": 1223,
                  "correct_both": 8306,
                  "wrong_both": 343
                },
                "mean_donor_class_probability": 0.9335696927549623,
                "baseline_mean_donor_class_probability": 0.7973169746797446,
                "prediction_agreement_with_recoloring": 0.9573,
                "mean_abs_logit_difference_from_recoloring": 1.8506782566493376
              },
              "shifted_donor": {
                "accuracy": 0.8446,
                "correct": 8446,
                "total": 10000,
                "mean_true_class_probability": 0.7969964754920433,
                "per_digit_accuracy": {
                  "0": 0.9326530612244898,
                  "1": 0.9506607929515418,
                  "2": 0.7800387596899225,
                  "3": 0.9386138613861386,
                  "4": 0.8391038696537678,
                  "5": 0.7847533632286996,
                  "6": 0.8966597077244259,
                  "7": 0.8647859922178989,
                  "8": 0.7392197125256673,
                  "9": 0.7016848364717542
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 138,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 45,
                  "wrong_to_correct": 57,
                  "correct_both": 8389,
                  "wrong_both": 1509
                },
                "mean_donor_class_probability": 0.06149667053762085,
                "baseline_mean_donor_class_probability": 0.05797885499426834,
                "prediction_agreement_with_recoloring": 0.9899,
                "mean_abs_logit_difference_from_recoloring": 0.1417402652595937
              }
            },
            "recolored_reference": {
              "all_palette": {
                "accuracy": 0.86804,
                "correct": 86804,
                "total": 100000,
                "mean_true_class_probability": 0.8298507602755971,
                "per_digit_accuracy": {
                  "0": 0.9271428571428572,
                  "1": 0.967136563876652,
                  "2": 0.8362403100775194,
                  "3": 0.9030693069306931,
                  "4": 0.8959266802443991,
                  "5": 0.8410313901345291,
                  "6": 0.8972860125260961,
                  "7": 0.8552529182879377,
                  "8": 0.7337782340862423,
                  "9": 0.8082259663032706
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 14552,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 4443,
                  "wrong_to_correct": 6907,
                  "correct_both": 79897,
                  "wrong_both": 8753
                },
                "mean_donor_class_probability": 0.17421834353305088,
                "baseline_mean_donor_class_probability": 0.10000000018436021,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "aligned_donor": {
                "accuracy": 0.9938,
                "correct": 9938,
                "total": 10000,
                "mean_true_class_probability": 0.9879955979234771,
                "per_digit_accuracy": {
                  "0": 0.9979591836734694,
                  "1": 0.9973568281938326,
                  "2": 0.9951550387596899,
                  "3": 0.999009900990099,
                  "4": 0.9938900203665988,
                  "5": 0.9955156950672646,
                  "6": 0.9895615866388309,
                  "7": 0.9912451361867705,
                  "8": 0.9948665297741273,
                  "9": 0.9831516352824579
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1521,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 1,
                  "wrong_to_correct": 1505,
                  "correct_both": 8433,
                  "wrong_both": 61
                },
                "mean_donor_class_probability": 0.9879955979234771,
                "baseline_mean_donor_class_probability": 0.7973169746797446,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "shifted_donor": {
                "accuracy": 0.8401,
                "correct": 8401,
                "total": 10000,
                "mean_true_class_probability": 0.7925792598889998,
                "per_digit_accuracy": {
                  "0": 0.9275510204081633,
                  "1": 0.9515418502202643,
                  "2": 0.7693798449612403,
                  "3": 0.9405940594059405,
                  "4": 0.835030549898167,
                  "5": 0.7769058295964125,
                  "6": 0.8893528183716075,
                  "7": 0.8589494163424124,
                  "8": 0.7402464065708418,
                  "9": 0.6927651139742319
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 172,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 81,
                  "wrong_to_correct": 48,
                  "correct_both": 8353,
                  "wrong_both": 1518
                },
                "mean_donor_class_probability": 0.0641115700985687,
                "baseline_mean_donor_class_probability": 0.05797885499426834,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              }
            },
            "random_controls": [
              {
                "all_palette": {
                  "accuracy": 0.84683,
                  "correct": 84683,
                  "total": 100000,
                  "mean_true_class_probability": 0.7990886522413247,
                  "per_digit_accuracy": {
                    "0": 0.8985714285714286,
                    "1": 0.9489867841409692,
                    "2": 0.7868217054263565,
                    "3": 0.934950495049505,
                    "4": 0.8669042769857433,
                    "5": 0.7812780269058296,
                    "6": 0.907098121085595,
                    "7": 0.8520428015564202,
                    "8": 0.7498973305954826,
                    "9": 0.7242814667988107
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 2694,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 956,
                    "wrong_to_correct": 1299,
                    "correct_both": 83384,
                    "wrong_both": 14361
                  },
                  "mean_donor_class_probability": 0.09935892117810839,
                  "baseline_mean_donor_class_probability": 0.10000000018436021,
                  "prediction_agreement_with_recoloring": 0.85696,
                  "mean_abs_logit_difference_from_recoloring": 3.241081436718162
                },
                "aligned_donor": {
                  "accuracy": 0.8393,
                  "correct": 8393,
                  "total": 10000,
                  "mean_true_class_probability": 0.7874375682245872,
                  "per_digit_accuracy": {
                    "0": 0.8530612244897959,
                    "1": 0.9374449339207048,
                    "2": 0.7810077519379846,
                    "3": 0.9386138613861386,
                    "4": 0.905295315682281,
                    "5": 0.7701793721973094,
                    "6": 0.9144050104384134,
                    "7": 0.8433852140077821,
                    "8": 0.7484599589322382,
                    "9": 0.6848364717542121
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 336,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 168,
                    "wrong_to_correct": 127,
                    "correct_both": 8266,
                    "wrong_both": 1439
                  },
                  "mean_donor_class_probability": 0.7874375682245872,
                  "baseline_mean_donor_class_probability": 0.7973169746797446,
                  "prediction_agreement_with_recoloring": 0.8438,
                  "mean_abs_logit_difference_from_recoloring": 3.6434032748277856
                },
                "shifted_donor": {
                  "accuracy": 0.8435,
                  "correct": 8435,
                  "total": 10000,
                  "mean_true_class_probability": 0.796800101585703,
                  "per_digit_accuracy": {
                    "0": 0.9306122448979591,
                    "1": 0.9524229074889868,
                    "2": 0.7800387596899225,
                    "3": 0.9386138613861386,
                    "4": 0.8340122199592668,
                    "5": 0.781390134529148,
                    "6": 0.8935281837160751,
                    "7": 0.8599221789883269,
                    "8": 0.7453798767967146,
                    "9": 0.7006937561942518
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 25,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 10,
                    "wrong_to_correct": 11,
                    "correct_both": 8424,
                    "wrong_both": 1555
                  },
                  "mean_donor_class_probability": 0.057906379886135756,
                  "baseline_mean_donor_class_probability": 0.05797885499426834,
                  "prediction_agreement_with_recoloring": 0.9819,
                  "mean_abs_logit_difference_from_recoloring": 0.352671114609763
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.84255,
                  "correct": 84255,
                  "total": 100000,
                  "mean_true_class_probability": 0.7973399392185968,
                  "per_digit_accuracy": {
                    "0": 0.9256122448979592,
                    "1": 0.9505726872246696,
                    "2": 0.7942829457364341,
                    "3": 0.9379207920792079,
                    "4": 0.8341140529531568,
                    "5": 0.7795964125560538,
                    "6": 0.8940501043841336,
                    "7": 0.8521400778210116,
                    "8": 0.7451745379876796,
                    "9": 0.6934588701684836
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1001,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 442,
                    "wrong_to_correct": 357,
                    "correct_both": 83898,
                    "wrong_both": 15303
                  },
                  "mean_donor_class_probability": 0.09980317033245276,
                  "baseline_mean_donor_class_probability": 0.10000000018436021,
                  "prediction_agreement_with_recoloring": 0.85412,
                  "mean_abs_logit_difference_from_recoloring": 3.2762311953286214
                },
                "aligned_donor": {
                  "accuracy": 0.8382,
                  "correct": 8382,
                  "total": 10000,
                  "mean_true_class_probability": 0.7939321142608201,
                  "per_digit_accuracy": {
                    "0": 0.9214285714285714,
                    "1": 0.9506607929515418,
                    "2": 0.8042635658914729,
                    "3": 0.9396039603960396,
                    "4": 0.835030549898167,
                    "5": 0.7724215246636771,
                    "6": 0.8914405010438413,
                    "7": 0.8424124513618677,
                    "8": 0.7340862422997947,
                    "9": 0.6709613478691774
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 156,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 89,
                    "wrong_to_correct": 37,
                    "correct_both": 8345,
                    "wrong_both": 1529
                  },
                  "mean_donor_class_probability": 0.7939321142608201,
                  "baseline_mean_donor_class_probability": 0.7973169746797446,
                  "prediction_agreement_with_recoloring": 0.8429,
                  "mean_abs_logit_difference_from_recoloring": 3.669970234230906
                },
                "shifted_donor": {
                  "accuracy": 0.8438,
                  "correct": 8438,
                  "total": 10000,
                  "mean_true_class_probability": 0.7974215618342169,
                  "per_digit_accuracy": {
                    "0": 0.9285714285714286,
                    "1": 0.9524229074889868,
                    "2": 0.7810077519379846,
                    "3": 0.9366336633663367,
                    "4": 0.8360488798370672,
                    "5": 0.7836322869955157,
                    "6": 0.8956158663883089,
                    "7": 0.8589494163424124,
                    "8": 0.7464065708418891,
                    "9": 0.7006937561942518
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 12,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 3,
                    "wrong_to_correct": 7,
                    "correct_both": 8431,
                    "wrong_both": 1559
                  },
                  "mean_donor_class_probability": 0.05813248730910475,
                  "baseline_mean_donor_class_probability": 0.05797885499426834,
                  "prediction_agreement_with_recoloring": 0.9832,
                  "mean_abs_logit_difference_from_recoloring": 0.3556940364424884
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.8465,
                  "correct": 84650,
                  "total": 100000,
                  "mean_true_class_probability": 0.7995406920990145,
                  "per_digit_accuracy": {
                    "0": 0.9302040816326531,
                    "1": 0.9451982378854625,
                    "2": 0.774515503875969,
                    "3": 0.9395049504950495,
                    "4": 0.8476578411405296,
                    "5": 0.8019058295964125,
                    "6": 0.8980167014613779,
                    "7": 0.8554474708171206,
                    "8": 0.7573921971252566,
                    "9": 0.7009910802775025
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1949,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 611,
                    "wrong_to_correct": 921,
                    "correct_both": 83729,
                    "wrong_both": 14739
                  },
                  "mean_donor_class_probability": 0.10295648776250793,
                  "baseline_mean_donor_class_probability": 0.10000000018436021,
                  "prediction_agreement_with_recoloring": 0.85829,
                  "mean_abs_logit_difference_from_recoloring": 3.252296119105883
                },
                "aligned_donor": {
                  "accuracy": 0.8565,
                  "correct": 8565,
                  "total": 10000,
                  "mean_true_class_probability": 0.8104571585849609,
                  "per_digit_accuracy": {
                    "0": 0.9489795918367347,
                    "1": 0.9480176211453745,
                    "2": 0.7790697674418605,
                    "3": 0.9376237623762376,
                    "4": 0.8533604887983707,
                    "5": 0.8598654708520179,
                    "6": 0.8966597077244259,
                    "7": 0.8560311284046692,
                    "8": 0.7659137577002053,
                    "9": 0.711595639246779
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 229,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 28,
                    "wrong_to_correct": 159,
                    "correct_both": 8406,
                    "wrong_both": 1407
                  },
                  "mean_donor_class_probability": 0.8104571585849609,
                  "baseline_mean_donor_class_probability": 0.7973169746797446,
                  "prediction_agreement_with_recoloring": 0.8612,
                  "mean_abs_logit_difference_from_recoloring": 3.6454920517293736
                },
                "shifted_donor": {
                  "accuracy": 0.8433,
                  "correct": 8433,
                  "total": 10000,
                  "mean_true_class_probability": 0.7970295718028377,
                  "per_digit_accuracy": {
                    "0": 0.9285714285714286,
                    "1": 0.9524229074889868,
                    "2": 0.7800387596899225,
                    "3": 0.9386138613861386,
                    "4": 0.835030549898167,
                    "5": 0.7780269058295964,
                    "6": 0.8966597077244259,
                    "7": 0.8560311284046692,
                    "8": 0.7474332648870636,
                    "9": 0.7016848364717542
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 20,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 9,
                    "wrong_to_correct": 8,
                    "correct_both": 8425,
                    "wrong_both": 1558
                  },
                  "mean_donor_class_probability": 0.05810073197143125,
                  "baseline_mean_donor_class_probability": 0.05797885499426834,
                  "prediction_agreement_with_recoloring": 0.9833,
                  "mean_abs_logit_difference_from_recoloring": 0.3525357707528025
                }
              }
            ]
          }
        },
        "checks": {
          "self_patch_logits_max_abs_diff": 0.0,
          "selected_coordinates_max_abs_diff": 7.62939453125e-05,
          "complementary_logit_effects_max_abs_diff": 3.910064697265625e-05,
          "perpendicular_component_max_abs_diff": 2.6702880859375e-05,
          "full_patch_vs_donor_logits_max_abs_diff": 7.62939453125e-06
        }
      },
      "checks": {
        "learned_tensors_identical": true,
        "identity_at_zero_strength_matches_baseline": true,
        "U_orthonormal": true,
        "UUT_idempotent": true,
        "wrapped_vs_direct_logits_max_abs_diff": 8.58306884765625e-06,
        "null_projection_logits_minus_bias_max_abs": 2.466142177581787e-05,
        "row_projection_logits_max_abs_diff": 3.4332275390625e-05,
        "head_rank": 10,
        "row_space_dim": 10,
        "null_space_dim": 118
      }
    },
    {
      "seed": 43,
      "baseline_validation": {
        "matched": 0.9784,
        "independent": 0.8618,
        "misleading": 0.865
      },
      "baseline_test": {
        "matched": {
          "accuracy": 0.9825,
          "correct": 9825,
          "total": 10000,
          "mean_true_class_probability": 0.974861143469167,
          "per_digit_accuracy": {
            "0": 0.9928571428571429,
            "1": 0.9938325991189427,
            "2": 0.9786821705426356,
            "3": 0.9841584158415841,
            "4": 0.9837067209775967,
            "5": 0.968609865470852,
            "6": 0.9791231732776617,
            "7": 0.9815175097276264,
            "8": 0.9804928131416838,
            "9": 0.979187314172448
          }
        },
        "independent": {
          "accuracy": 0.871,
          "correct": 8710,
          "total": 10000,
          "mean_true_class_probability": 0.8369359211550298,
          "per_digit_accuracy": {
            "0": 0.9469387755102041,
            "1": 0.9788546255506608,
            "2": 0.8381782945736435,
            "3": 0.8564356435643564,
            "4": 0.8778004073319755,
            "5": 0.8183856502242153,
            "6": 0.8778705636743215,
            "7": 0.8764591439688716,
            "8": 0.8080082135523614,
            "9": 0.8126858275520317
          }
        },
        "misleading": {
          "accuracy": 0.8691,
          "correct": 8691,
          "total": 10000,
          "mean_true_class_probability": 0.8311145776122599,
          "per_digit_accuracy": {
            "0": 0.9316326530612244,
            "1": 0.9744493392070485,
            "2": 0.7994186046511628,
            "3": 0.9128712871287129,
            "4": 0.8594704684317719,
            "5": 0.7746636771300448,
            "6": 0.9269311064718163,
            "7": 0.8891050583657587,
            "8": 0.7874743326488707,
            "9": 0.8136769078295342
          }
        }
      },
      "spectrum_variance_fraction": [
        0.3114084303379059,
        0.21842584013938904,
        0.18313096463680267,
        0.042941562831401825,
        0.03319890797138214,
        0.025757577270269394,
        0.021582456305623055,
        0.016683707013726234,
        0.012947586365044117,
        0.010855739936232567,
        0.009676582179963589,
        0.009007067419588566,
        0.007335279136896133,
        0.006861134432256222,
        0.005589641164988279,
        0.005405930802226067,
        0.0049781580455601215,
        0.004927230533212423,
        0.004096871707588434,
        0.0039026010781526566,
        0.0035715759731829166,
        0.0034384794998914003,
        0.0031693160999566317,
        0.0030878486577421427,
        0.002618813421577215,
        0.0022549002896994352,
        0.0021801097318530083,
        0.0020240256562829018,
        0.0018970705568790436,
        0.0017707052174955606,
        0.001590514904819429,
        0.0015135693829506636
      ],
      "validation_grid": [
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.978,
            "independent": 0.8682,
            "misleading": 0.8782
          },
          "objective": 0.9081333333333333
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.9766,
            "independent": 0.8716,
            "misleading": 0.887
          },
          "objective": 0.9117333333333333
        },
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9784,
            "independent": 0.8582,
            "misleading": 0.8622
          },
          "objective": 0.8996
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9778,
            "independent": 0.8512,
            "misleading": 0.857
          },
          "objective": 0.8953333333333333
        },
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9784,
            "independent": 0.8622,
            "misleading": 0.8654
          },
          "objective": 0.902
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9784,
            "independent": 0.862,
            "misleading": 0.8648
          },
          "objective": 0.9017333333333334
        },
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9782,
            "independent": 0.8588,
            "misleading": 0.8644
          },
          "objective": 0.9004666666666666
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.979,
            "independent": 0.8564,
            "misleading": 0.8602
          },
          "objective": 0.8985333333333333
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9772,
            "independent": 0.8798,
            "misleading": 0.8906
          },
          "objective": 0.9158666666666666
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.9626,
            "independent": 0.8678,
            "misleading": 0.8974
          },
          "objective": 0.9092666666666668
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9782,
            "independent": 0.8594,
            "misleading": 0.8684
          },
          "objective": 0.902
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9776,
            "independent": 0.8536,
            "misleading": 0.8686
          },
          "objective": 0.8999333333333333
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9784,
            "independent": 0.86,
            "misleading": 0.8624
          },
          "objective": 0.9002666666666667
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9784,
            "independent": 0.8568,
            "misleading": 0.8576
          },
          "objective": 0.8976000000000001
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9792,
            "independent": 0.859,
            "misleading": 0.8652
          },
          "objective": 0.9011333333333335
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9788,
            "independent": 0.8582,
            "misleading": 0.863
          },
          "objective": 0.9
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9598,
            "independent": 0.8368,
            "misleading": 0.8578
          },
          "objective": 0.8847999999999999
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.84,
            "independent": 0.7256,
            "misleading": 0.7558
          },
          "objective": 0.7737999999999999
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9778,
            "independent": 0.858,
            "misleading": 0.8646
          },
          "objective": 0.9001333333333333
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.978,
            "independent": 0.8574,
            "misleading": 0.8662
          },
          "objective": 0.9005333333333333
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.977,
            "independent": 0.86,
            "misleading": 0.8614
          },
          "objective": 0.8994666666666666
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9774,
            "independent": 0.8536,
            "misleading": 0.8554
          },
          "objective": 0.8954666666666666
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9788,
            "independent": 0.8572,
            "misleading": 0.8644
          },
          "objective": 0.9001333333333332
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9788,
            "independent": 0.8538,
            "misleading": 0.8612
          },
          "objective": 0.8979333333333334
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.958,
            "independent": 0.8494,
            "misleading": 0.8638
          },
          "objective": 0.8904
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.843,
            "independent": 0.7434,
            "misleading": 0.7808
          },
          "objective": 0.7890666666666667
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.979,
            "independent": 0.8588,
            "misleading": 0.857
          },
          "objective": 0.8982666666666667
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9782,
            "independent": 0.8518,
            "misleading": 0.8464
          },
          "objective": 0.8921333333333333
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9782,
            "independent": 0.8598,
            "misleading": 0.862
          },
          "objective": 0.9
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9776,
            "independent": 0.8556,
            "misleading": 0.8578
          },
          "objective": 0.8970000000000001
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.978,
            "independent": 0.8588,
            "misleading": 0.8614
          },
          "objective": 0.8994
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9782,
            "independent": 0.854,
            "misleading": 0.8572
          },
          "objective": 0.8964666666666666
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9672,
            "independent": 0.839,
            "misleading": 0.8652
          },
          "objective": 0.8904666666666667
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.674,
            "independent": 0.5282,
            "misleading": 0.5562
          },
          "objective": 0.5861333333333333
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.978,
            "independent": 0.8548,
            "misleading": 0.8512
          },
          "objective": 0.8946666666666667
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.975,
            "independent": 0.842,
            "misleading": 0.829
          },
          "objective": 0.882
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9786,
            "independent": 0.8588,
            "misleading": 0.8608
          },
          "objective": 0.8994
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9794,
            "independent": 0.8486,
            "misleading": 0.8524
          },
          "objective": 0.8934666666666667
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9768,
            "independent": 0.8518,
            "misleading": 0.8606
          },
          "objective": 0.8964
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9736,
            "independent": 0.841,
            "misleading": 0.8556
          },
          "objective": 0.8900666666666667
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.965,
            "independent": 0.8154,
            "misleading": 0.8438
          },
          "objective": 0.8747333333333334
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.339,
            "independent": 0.3018,
            "misleading": 0.2576
          },
          "objective": 0.2994666666666667
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9726,
            "independent": 0.8494,
            "misleading": 0.8506
          },
          "objective": 0.8908666666666667
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.952,
            "independent": 0.7922,
            "misleading": 0.7908
          },
          "objective": 0.8450000000000001
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9788,
            "independent": 0.8562,
            "misleading": 0.8566
          },
          "objective": 0.8972000000000001
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.973,
            "independent": 0.8328,
            "misleading": 0.8294
          },
          "objective": 0.8784000000000001
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.977,
            "independent": 0.8586,
            "misleading": 0.8558
          },
          "objective": 0.8971333333333332
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9766,
            "independent": 0.8524,
            "misleading": 0.8404
          },
          "objective": 0.8898000000000001
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9764,
            "independent": 0.861,
            "misleading": 0.8694
          },
          "objective": 0.9022666666666668
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.3646,
            "independent": 0.3018,
            "misleading": 0.292
          },
          "objective": 0.3194666666666666
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9778,
            "independent": 0.8458,
            "misleading": 0.8468
          },
          "objective": 0.8901333333333333
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9638,
            "independent": 0.778,
            "misleading": 0.777
          },
          "objective": 0.8396
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9738,
            "independent": 0.849,
            "misleading": 0.836
          },
          "objective": 0.8862666666666666
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.948,
            "independent": 0.7818,
            "misleading": 0.7466
          },
          "objective": 0.8254666666666667
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9764,
            "independent": 0.844,
            "misleading": 0.8484
          },
          "objective": 0.8896000000000001
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9632,
            "independent": 0.7632,
            "misleading": 0.7514
          },
          "objective": 0.8259333333333333
        }
      ],
      "selected": {
        "rank": 2,
        "strength": 0.5,
        "validation_objective": 0.9158666666666666
      },
      "selected_test": {
        "matched": {
          "accuracy": 0.9799,
          "correct": 9799,
          "total": 10000,
          "mean_true_class_probability": 0.9710837870704053,
          "per_digit_accuracy": {
            "0": 0.9948979591836735,
            "1": 0.9955947136563876,
            "2": 0.9777131782945736,
            "3": 0.9702970297029703,
            "4": 0.9867617107942973,
            "5": 0.9708520179372198,
            "6": 0.9770354906054279,
            "7": 0.9815175097276264,
            "8": 0.9661190965092402,
            "9": 0.9752229930624381
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 55,
            "wrong_to_correct": 29,
            "correct_both": 9770,
            "wrong_both": 146
          }
        },
        "independent": {
          "accuracy": 0.8842,
          "correct": 8842,
          "total": 10000,
          "mean_true_class_probability": 0.8489126363420159,
          "per_digit_accuracy": {
            "0": 0.963265306122449,
            "1": 0.986784140969163,
            "2": 0.8701550387596899,
            "3": 0.8415841584158416,
            "4": 0.9175152749490835,
            "5": 0.82847533632287,
            "6": 0.894572025052192,
            "7": 0.9036964980544747,
            "8": 0.8090349075975359,
            "9": 0.8087215064420218
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 135,
            "wrong_to_correct": 267,
            "correct_both": 8575,
            "wrong_both": 1023
          }
        },
        "misleading": {
          "accuracy": 0.8975,
          "correct": 8975,
          "total": 10000,
          "mean_true_class_probability": 0.8625637369687487,
          "per_digit_accuracy": {
            "0": 0.9581632653061225,
            "1": 0.9753303964757709,
            "2": 0.8866279069767442,
            "3": 0.9148514851485149,
            "4": 0.9185336048879837,
            "5": 0.8452914798206278,
            "6": 0.9331941544885177,
            "7": 0.9260700389105059,
            "8": 0.768993839835729,
            "9": 0.8315163528245788
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 48,
            "wrong_to_correct": 332,
            "correct_both": 8643,
            "wrong_both": 977
          }
        }
      },
      "selected_test_full_strength": {
        "matched": {
          "accuracy": 0.9677,
          "correct": 9677,
          "total": 10000,
          "mean_true_class_probability": 0.9570521259462014,
          "per_digit_accuracy": {
            "0": 0.996938775510204,
            "1": 0.9955947136563876,
            "2": 0.9709302325581395,
            "3": 0.9108910891089109,
            "4": 0.9877800407331976,
            "5": 0.9585201793721974,
            "6": 0.9749478079331941,
            "7": 0.9834630350194552,
            "8": 0.9322381930184805,
            "9": 0.9613478691774033
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 193,
            "wrong_to_correct": 45,
            "correct_both": 9632,
            "wrong_both": 130
          }
        },
        "independent": {
          "accuracy": 0.8674,
          "correct": 8674,
          "total": 10000,
          "mean_true_class_probability": 0.8359261286301244,
          "per_digit_accuracy": {
            "0": 0.9673469387755103,
            "1": 0.9841409691629956,
            "2": 0.8788759689922481,
            "3": 0.7851485148514852,
            "4": 0.9307535641547862,
            "5": 0.820627802690583,
            "6": 0.906054279749478,
            "7": 0.9105058365758755,
            "8": 0.7474332648870636,
            "9": 0.7244796828543112
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 429,
            "wrong_to_correct": 393,
            "correct_both": 8281,
            "wrong_both": 897
          }
        },
        "misleading": {
          "accuracy": 0.9051,
          "correct": 9051,
          "total": 10000,
          "mean_true_class_probability": 0.8728997110910144,
          "per_digit_accuracy": {
            "0": 0.9663265306122449,
            "1": 0.973568281938326,
            "2": 0.9147286821705426,
            "3": 0.904950495049505,
            "4": 0.9470468431771895,
            "5": 0.8811659192825112,
            "6": 0.941544885177453,
            "7": 0.9396887159533074,
            "8": 0.7351129363449692,
            "9": 0.8334985133795837
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 126,
            "wrong_to_correct": 486,
            "correct_both": 8565,
            "wrong_both": 823
          }
        }
      },
      "random_control_test": [
        {
          "matched": {
            "accuracy": 0.9822,
            "correct": 9822,
            "total": 10000,
            "mean_true_class_probability": 0.9745683307761676,
            "per_digit_accuracy": {
              "0": 0.9908163265306122,
              "1": 0.9929515418502203,
              "2": 0.9786821705426356,
              "3": 0.9831683168316832,
              "4": 0.984725050916497,
              "5": 0.968609865470852,
              "6": 0.9780793319415448,
              "7": 0.9815175097276264,
              "8": 0.9815195071868583,
              "9": 0.979187314172448
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 8,
              "wrong_to_correct": 5,
              "correct_both": 9817,
              "wrong_both": 170
            }
          },
          "independent": {
            "accuracy": 0.87,
            "correct": 8700,
            "total": 10000,
            "mean_true_class_probability": 0.8336870805249242,
            "per_digit_accuracy": {
              "0": 0.9469387755102041,
              "1": 0.9788546255506608,
              "2": 0.8362403100775194,
              "3": 0.8514851485148515,
              "4": 0.8839103869653768,
              "5": 0.8105381165919282,
              "6": 0.8716075156576201,
              "7": 0.877431906614786,
              "8": 0.8100616016427105,
              "9": 0.8136769078295342
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 45,
              "wrong_to_correct": 35,
              "correct_both": 8665,
              "wrong_both": 1255
            }
          },
          "misleading": {
            "accuracy": 0.8725,
            "correct": 8725,
            "total": 10000,
            "mean_true_class_probability": 0.8328151843223546,
            "per_digit_accuracy": {
              "0": 0.9326530612244898,
              "1": 0.9744493392070485,
              "2": 0.8052325581395349,
              "3": 0.903960396039604,
              "4": 0.8788187372708758,
              "5": 0.7679372197309418,
              "6": 0.9206680584551148,
              "7": 0.8900778210116731,
              "8": 0.7926078028747433,
              "9": 0.8364717542120912
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 31,
              "wrong_to_correct": 65,
              "correct_both": 8660,
              "wrong_both": 1244
            }
          }
        },
        {
          "matched": {
            "accuracy": 0.9828,
            "correct": 9828,
            "total": 10000,
            "mean_true_class_probability": 0.9747671801870894,
            "per_digit_accuracy": {
              "0": 0.9938775510204082,
              "1": 0.9938325991189427,
              "2": 0.9786821705426356,
              "3": 0.9831683168316832,
              "4": 0.9816700610997964,
              "5": 0.968609865470852,
              "6": 0.9801670146137788,
              "7": 0.9815175097276264,
              "8": 0.9835728952772074,
              "9": 0.9801783944499505
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 5,
              "wrong_to_correct": 8,
              "correct_both": 9820,
              "wrong_both": 167
            }
          },
          "independent": {
            "accuracy": 0.8702,
            "correct": 8702,
            "total": 10000,
            "mean_true_class_probability": 0.8347612341496006,
            "per_digit_accuracy": {
              "0": 0.95,
              "1": 0.9797356828193833,
              "2": 0.8313953488372093,
              "3": 0.8574257425742574,
              "4": 0.8594704684317719,
              "5": 0.7993273542600897,
              "6": 0.8830897703549061,
              "7": 0.8842412451361867,
              "8": 0.8275154004106776,
              "9": 0.8097125867195243
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 59,
              "wrong_to_correct": 51,
              "correct_both": 8651,
              "wrong_both": 1239
            }
          },
          "misleading": {
            "accuracy": 0.8679,
            "correct": 8679,
            "total": 10000,
            "mean_true_class_probability": 0.8290768365612221,
            "per_digit_accuracy": {
              "0": 0.9346938775510204,
              "1": 0.9744493392070485,
              "2": 0.8013565891472868,
              "3": 0.9108910891089109,
              "4": 0.8360488798370672,
              "5": 0.7600896860986547,
              "6": 0.9279749478079332,
              "7": 0.8978599221789884,
              "8": 0.7987679671457906,
              "9": 0.8136769078295342
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 53,
              "wrong_to_correct": 41,
              "correct_both": 8638,
              "wrong_both": 1268
            }
          }
        },
        {
          "matched": {
            "accuracy": 0.9821,
            "correct": 9821,
            "total": 10000,
            "mean_true_class_probability": 0.9744259530150766,
            "per_digit_accuracy": {
              "0": 0.9908163265306122,
              "1": 0.9929515418502203,
              "2": 0.9806201550387597,
              "3": 0.9831683168316832,
              "4": 0.9826883910386965,
              "5": 0.968609865470852,
              "6": 0.9770354906054279,
              "7": 0.9815175097276264,
              "8": 0.9804928131416838,
              "9": 0.9801783944499505
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 8,
              "wrong_to_correct": 4,
              "correct_both": 9817,
              "wrong_both": 171
            }
          },
          "independent": {
            "accuracy": 0.871,
            "correct": 8710,
            "total": 10000,
            "mean_true_class_probability": 0.8354209865039997,
            "per_digit_accuracy": {
              "0": 0.9418367346938775,
              "1": 0.9762114537444934,
              "2": 0.8401162790697675,
              "3": 0.8534653465346534,
              "4": 0.8737270875763747,
              "5": 0.8161434977578476,
              "6": 0.8716075156576201,
              "7": 0.877431906614786,
              "8": 0.8182751540041068,
              "9": 0.8225966303270565
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 39,
              "wrong_to_correct": 39,
              "correct_both": 8671,
              "wrong_both": 1251
            }
          },
          "misleading": {
            "accuracy": 0.8699,
            "correct": 8699,
            "total": 10000,
            "mean_true_class_probability": 0.8292840063335097,
            "per_digit_accuracy": {
              "0": 0.9275510204081633,
              "1": 0.9718061674008811,
              "2": 0.8178294573643411,
              "3": 0.907920792079208,
              "4": 0.8564154786150713,
              "5": 0.7780269058295964,
              "6": 0.9206680584551148,
              "7": 0.8900778210116731,
              "8": 0.7823408624229979,
              "9": 0.8245787908820614
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 39,
              "wrong_to_correct": 47,
              "correct_both": 8652,
              "wrong_both": 1262
            }
          }
        }
      ],
      "null_space_control_test": {
        "matched": {
          "accuracy": 0.0892,
          "correct": 892,
          "total": 10000,
          "mean_true_class_probability": 0.09995662999004126,
          "per_digit_accuracy": {
            "0": 0.0,
            "1": 0.0,
            "2": 0.0,
            "3": 0.0,
            "4": 0.0,
            "5": 1.0,
            "6": 0.0,
            "7": 0.0,
            "8": 0.0,
            "9": 0.0
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 8961,
            "wrong_to_correct": 28,
            "correct_both": 864,
            "wrong_both": 147
          }
        },
        "independent": {
          "accuracy": 0.0892,
          "correct": 892,
          "total": 10000,
          "mean_true_class_probability": 0.0999566571816802,
          "per_digit_accuracy": {
            "0": 0.0,
            "1": 0.0,
            "2": 0.0,
            "3": 0.0,
            "4": 0.0,
            "5": 1.0,
            "6": 0.0,
            "7": 0.0,
            "8": 0.0,
            "9": 0.0
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 7980,
            "wrong_to_correct": 162,
            "correct_both": 730,
            "wrong_both": 1128
          }
        },
        "misleading": {
          "accuracy": 0.0892,
          "correct": 892,
          "total": 10000,
          "mean_true_class_probability": 0.09995666927099228,
          "per_digit_accuracy": {
            "0": 0.0,
            "1": 0.0,
            "2": 0.0,
            "3": 0.0,
            "4": 0.0,
            "5": 1.0,
            "6": 0.0,
            "7": 0.0,
            "8": 0.0,
            "9": 0.0
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 8000,
            "wrong_to_correct": 201,
            "correct_both": 691,
            "wrong_both": 1108
          }
        }
      },
      "row_space_identity_test": {
        "matched": {
          "accuracy": 0.9825,
          "correct": 9825,
          "total": 10000,
          "mean_true_class_probability": 0.9748611431401534,
          "per_digit_accuracy": {
            "0": 0.9928571428571429,
            "1": 0.9938325991189427,
            "2": 0.9786821705426356,
            "3": 0.9841584158415841,
            "4": 0.9837067209775967,
            "5": 0.968609865470852,
            "6": 0.9791231732776617,
            "7": 0.9815175097276264,
            "8": 0.9804928131416838,
            "9": 0.979187314172448
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 0,
            "wrong_to_correct": 0,
            "correct_both": 9825,
            "wrong_both": 175
          }
        },
        "independent": {
          "accuracy": 0.871,
          "correct": 8710,
          "total": 10000,
          "mean_true_class_probability": 0.8369359190951975,
          "per_digit_accuracy": {
            "0": 0.9469387755102041,
            "1": 0.9788546255506608,
            "2": 0.8381782945736435,
            "3": 0.8564356435643564,
            "4": 0.8778004073319755,
            "5": 0.8183856502242153,
            "6": 0.8778705636743215,
            "7": 0.8764591439688716,
            "8": 0.8080082135523614,
            "9": 0.8126858275520317
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 0,
            "wrong_to_correct": 0,
            "correct_both": 8710,
            "wrong_both": 1290
          }
        },
        "misleading": {
          "accuracy": 0.8691,
          "correct": 8691,
          "total": 10000,
          "mean_true_class_probability": 0.8311145790761225,
          "per_digit_accuracy": {
            "0": 0.9316326530612244,
            "1": 0.9744493392070485,
            "2": 0.7994186046511628,
            "3": 0.9128712871287129,
            "4": 0.8594704684317719,
            "5": 0.7746636771300448,
            "6": 0.9269311064718163,
            "7": 0.8891050583657587,
            "8": 0.7874743326488707,
            "9": 0.8136769078295342
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 0,
            "wrong_to_correct": 0,
            "correct_both": 8691,
            "wrong_both": 1309
          }
        }
      },
      "replacement": {
        "rank": 2,
        "test": {
          "matched": {
            "all_palette": {
              "accuracy": 0.9533,
              "correct": 95330,
              "total": 100000,
              "mean_true_class_probability": 0.938599737435474,
              "per_digit_accuracy": {
                "0": 0.9870408163265306,
                "1": 0.9929515418502203,
                "2": 0.858139534883721,
                "3": 0.9321782178217822,
                "4": 0.965071283095723,
                "5": 0.9344170403587444,
                "6": 0.975991649269311,
                "7": 0.9678988326848249,
                "8": 0.9482546201232033,
                "9": 0.9680872150644202
              },
              "unique_images": 10000,
              "donors_per_image": 10,
              "changed_predictions": 3882,
              "transitions_vs_baseline": {
                "correct_to_wrong": 3310,
                "wrong_to_correct": 390,
                "correct_both": 94940,
                "wrong_both": 1360
              },
              "mean_donor_class_probability": 0.11332179247984295,
              "baseline_mean_donor_class_probability": 0.10000000039033324,
              "prediction_agreement_with_recoloring": 0.87738,
              "mean_abs_logit_difference_from_recoloring": 2.900597902024604
            },
            "aligned_donor": {
              "accuracy": 0.9885,
              "correct": 9885,
              "total": 10000,
              "mean_true_class_probability": 0.9823396244628999,
              "per_digit_accuracy": {
                "0": 0.996938775510204,
                "1": 0.9911894273127754,
                "2": 0.9874031007751938,
                "3": 0.9900990099009901,
                "4": 0.9908350305498982,
                "5": 0.992152466367713,
                "6": 0.9822546972860126,
                "7": 0.9863813229571985,
                "8": 0.9876796714579056,
                "9": 0.9801783944499505
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 91,
              "transitions_vs_baseline": {
                "correct_to_wrong": 10,
                "wrong_to_correct": 70,
                "correct_both": 9815,
                "wrong_both": 105
              },
              "mean_donor_class_probability": 0.9823396244628999,
              "baseline_mean_donor_class_probability": 0.974861143469167,
              "prediction_agreement_with_recoloring": 0.9939,
              "mean_abs_logit_difference_from_recoloring": 0.2688265423336625
            },
            "shifted_donor": {
              "accuracy": 0.9082,
              "correct": 9082,
              "total": 10000,
              "mean_true_class_probability": 0.8930155146050767,
              "per_digit_accuracy": {
                "0": 0.9795918367346939,
                "1": 0.9973568281938326,
                "2": 0.4951550387596899,
                "3": 0.9099009900990099,
                "4": 0.9582484725050916,
                "5": 0.8878923766816144,
                "6": 0.9728601252609603,
                "7": 0.9513618677042801,
                "8": 0.9815195071868583,
                "9": 0.9524281466798811
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 805,
              "transitions_vs_baseline": {
                "correct_to_wrong": 762,
                "wrong_to_correct": 19,
                "correct_both": 9063,
                "wrong_both": 156
              },
              "mean_donor_class_probability": 0.04662198002741645,
              "baseline_mean_donor_class_probability": 0.0019923183986627993,
              "prediction_agreement_with_recoloring": 0.8551,
              "mean_abs_logit_difference_from_recoloring": 3.230274905288853
            },
            "complement_patch": {
              "all_palette": {
                "accuracy": 0.9216,
                "correct": 92160,
                "total": 100000,
                "mean_true_class_probability": 0.8947665268400041,
                "per_digit_accuracy": {
                  "0": 0.9645918367346938,
                  "1": 0.9572687224669604,
                  "2": 0.8987403100775194,
                  "3": 0.9465346534653465,
                  "4": 0.9394093686354379,
                  "5": 0.9258968609865471,
                  "6": 0.931419624217119,
                  "7": 0.9174124513618677,
                  "8": 0.8575975359342916,
                  "9": 0.8737363726461843
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 7254,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 6505,
                  "wrong_to_correct": 415,
                  "correct_both": 91745,
                  "wrong_both": 1335
                },
                "mean_donor_class_probability": 0.1302867573848212,
                "baseline_mean_donor_class_probability": 0.10000000039033324,
                "prediction_agreement_with_recoloring": 0.90816,
                "mean_abs_logit_difference_from_recoloring": 1.349948490098562
              },
              "aligned_donor": {
                "accuracy": 0.9904,
                "correct": 9904,
                "total": 10000,
                "mean_true_class_probability": 0.9853182948334899,
                "per_digit_accuracy": {
                  "0": 0.996938775510204,
                  "1": 0.9964757709251101,
                  "2": 0.9767441860465116,
                  "3": 0.9851485148514851,
                  "4": 0.9887983706720977,
                  "5": 0.9943946188340808,
                  "6": 0.9864300626304802,
                  "7": 0.9902723735408561,
                  "8": 0.9948665297741273,
                  "9": 0.9940535183349851
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 118,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 14,
                  "wrong_to_correct": 93,
                  "correct_both": 9811,
                  "wrong_both": 82
                },
                "mean_donor_class_probability": 0.9853182948334899,
                "baseline_mean_donor_class_probability": 0.974861143469167,
                "prediction_agreement_with_recoloring": 0.9957,
                "mean_abs_logit_difference_from_recoloring": 0.13765286857716738
              },
              "shifted_donor": {
                "accuracy": 0.9261,
                "correct": 9261,
                "total": 10000,
                "mean_true_class_probability": 0.8926281531848512,
                "per_digit_accuracy": {
                  "0": 0.9673469387755103,
                  "1": 0.8933920704845815,
                  "2": 0.9312015503875969,
                  "3": 0.9653465346534653,
                  "4": 0.960285132382892,
                  "5": 0.9405829596412556,
                  "6": 0.9478079331941545,
                  "7": 0.943579766536965,
                  "8": 0.7987679671457906,
                  "9": 0.9167492566897919
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 691,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 610,
                  "wrong_to_correct": 46,
                  "correct_both": 9215,
                  "wrong_both": 129
                },
                "mean_donor_class_probability": 0.025603597186628326,
                "baseline_mean_donor_class_probability": 0.0019923183986627993,
                "prediction_agreement_with_recoloring": 0.9039,
                "mean_abs_logit_difference_from_recoloring": 1.827932888941774
              }
            },
            "recolored_reference": {
              "all_palette": {
                "accuracy": 0.87212,
                "correct": 87212,
                "total": 100000,
                "mean_true_class_probability": 0.8365256038477711,
                "per_digit_accuracy": {
                  "0": 0.9380612244897959,
                  "1": 0.974273127753304,
                  "2": 0.8486434108527132,
                  "3": 0.8669306930693069,
                  "4": 0.8793279022403259,
                  "5": 0.8089686098654708,
                  "6": 0.8872651356993737,
                  "7": 0.8836575875486381,
                  "8": 0.7941478439425051,
                  "9": 0.8203171456888008
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 12484,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 11574,
                  "wrong_to_correct": 536,
                  "correct_both": 86676,
                  "wrong_both": 1214
                },
                "mean_donor_class_probability": 0.17803537685845774,
                "baseline_mean_donor_class_probability": 0.10000000039033324,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "aligned_donor": {
                "accuracy": 0.9944,
                "correct": 9944,
                "total": 10000,
                "mean_true_class_probability": 0.9900084610020509,
                "per_digit_accuracy": {
                  "0": 0.9979591836734694,
                  "1": 0.9964757709251101,
                  "2": 0.9961240310077519,
                  "3": 0.9900990099009901,
                  "4": 0.9938900203665988,
                  "5": 0.9966367713004485,
                  "6": 0.9864300626304802,
                  "7": 0.9931906614785992,
                  "8": 0.9989733059548255,
                  "9": 0.9940535183349851
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 121,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 0,
                  "wrong_to_correct": 119,
                  "correct_both": 9825,
                  "wrong_both": 56
                },
                "mean_donor_class_probability": 0.9900084610020509,
                "baseline_mean_donor_class_probability": 0.974861143469167,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "shifted_donor": {
                "accuracy": 0.8694,
                "correct": 8694,
                "total": 10000,
                "mean_true_class_probability": 0.8304767185413066,
                "per_digit_accuracy": {
                  "0": 0.9346938775510204,
                  "1": 0.9744493392070485,
                  "2": 0.7926356589147286,
                  "3": 0.9198019801980198,
                  "4": 0.860488798370672,
                  "5": 0.773542600896861,
                  "6": 0.9269311064718163,
                  "7": 0.8861867704280155,
                  "8": 0.7885010266940452,
                  "9": 0.8156590683845392
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1258,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 1172,
                  "wrong_to_correct": 41,
                  "correct_both": 8653,
                  "wrong_both": 134
                },
                "mean_donor_class_probability": 0.05870254770011943,
                "baseline_mean_donor_class_probability": 0.0019923183986627993,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              }
            },
            "random_controls": [
              {
                "all_palette": {
                  "accuracy": 0.98185,
                  "correct": 98185,
                  "total": 100000,
                  "mean_true_class_probability": 0.9742084075655217,
                  "per_digit_accuracy": {
                    "0": 0.991734693877551,
                    "1": 0.9934801762114538,
                    "2": 0.9797480620155039,
                    "3": 0.9822772277227723,
                    "4": 0.9832993890020366,
                    "5": 0.9692825112107624,
                    "6": 0.9793319415448852,
                    "7": 0.9807392996108949,
                    "8": 0.9791581108829569,
                    "9": 0.9767096134786918
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 285,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 146,
                    "wrong_to_correct": 81,
                    "correct_both": 98104,
                    "wrong_both": 1669
                  },
                  "mean_donor_class_probability": 0.10010714000882207,
                  "baseline_mean_donor_class_probability": 0.10000000039033324,
                  "prediction_agreement_with_recoloring": 0.87507,
                  "mean_abs_logit_difference_from_recoloring": 3.2273172205817087
                },
                "aligned_donor": {
                  "accuracy": 0.9822,
                  "correct": 9822,
                  "total": 10000,
                  "mean_true_class_probability": 0.9749895233692052,
                  "per_digit_accuracy": {
                    "0": 0.9908163265306122,
                    "1": 0.9929515418502203,
                    "2": 0.9796511627906976,
                    "3": 0.9851485148514851,
                    "4": 0.9837067209775967,
                    "5": 0.9663677130044843,
                    "6": 0.9791231732776617,
                    "7": 0.9815175097276264,
                    "8": 0.9804928131416838,
                    "9": 0.979187314172448
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 19,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 8,
                    "wrong_to_correct": 5,
                    "correct_both": 9817,
                    "wrong_both": 170
                  },
                  "mean_donor_class_probability": 0.9749895233692052,
                  "baseline_mean_donor_class_probability": 0.974861143469167,
                  "prediction_agreement_with_recoloring": 0.9876,
                  "mean_abs_logit_difference_from_recoloring": 0.3056706716192886
                },
                "shifted_donor": {
                  "accuracy": 0.9815,
                  "correct": 9815,
                  "total": 10000,
                  "mean_true_class_probability": 0.9734579686480187,
                  "per_digit_accuracy": {
                    "0": 0.9908163265306122,
                    "1": 0.9929515418502203,
                    "2": 0.9806201550387597,
                    "3": 0.9861386138613861,
                    "4": 0.9816700610997964,
                    "5": 0.9697309417040358,
                    "6": 0.9791231732776617,
                    "7": 0.9815175097276264,
                    "8": 0.9815195071868583,
                    "9": 0.9682854311199207
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 28,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 17,
                    "wrong_to_correct": 7,
                    "correct_both": 9808,
                    "wrong_both": 168
                  },
                  "mean_donor_class_probability": 0.0020790557641256905,
                  "baseline_mean_donor_class_probability": 0.0019923183986627993,
                  "prediction_agreement_with_recoloring": 0.8742,
                  "mean_abs_logit_difference_from_recoloring": 3.6165005929597838
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.98217,
                  "correct": 98217,
                  "total": 100000,
                  "mean_true_class_probability": 0.9744236370358348,
                  "per_digit_accuracy": {
                    "0": 0.9915306122448979,
                    "1": 0.993568281938326,
                    "2": 0.9800387596899225,
                    "3": 0.9844554455445544,
                    "4": 0.9828920570264765,
                    "5": 0.9710762331838565,
                    "6": 0.9788100208768267,
                    "7": 0.9797665369649805,
                    "8": 0.9795687885010267,
                    "9": 0.9774033696729435
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 219,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 122,
                    "wrong_to_correct": 89,
                    "correct_both": 98128,
                    "wrong_both": 1661
                  },
                  "mean_donor_class_probability": 0.1001571880378946,
                  "baseline_mean_donor_class_probability": 0.10000000039033324,
                  "prediction_agreement_with_recoloring": 0.87578,
                  "mean_abs_logit_difference_from_recoloring": 3.1821099436170495
                },
                "aligned_donor": {
                  "accuracy": 0.9828,
                  "correct": 9828,
                  "total": 10000,
                  "mean_true_class_probability": 0.9752315462723229,
                  "per_digit_accuracy": {
                    "0": 0.9908163265306122,
                    "1": 0.9938325991189427,
                    "2": 0.9786821705426356,
                    "3": 0.9841584158415841,
                    "4": 0.9826883910386965,
                    "5": 0.9742152466367713,
                    "6": 0.9801670146137788,
                    "7": 0.9815175097276264,
                    "8": 0.9825462012320328,
                    "9": 0.977205153617443
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 20,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 7,
                    "wrong_to_correct": 10,
                    "correct_both": 9818,
                    "wrong_both": 165
                  },
                  "mean_donor_class_probability": 0.9752315462723229,
                  "baseline_mean_donor_class_probability": 0.974861143469167,
                  "prediction_agreement_with_recoloring": 0.9882,
                  "mean_abs_logit_difference_from_recoloring": 0.30105485289283096
                },
                "shifted_donor": {
                  "accuracy": 0.9816,
                  "correct": 9816,
                  "total": 10000,
                  "mean_true_class_probability": 0.974085313434687,
                  "per_digit_accuracy": {
                    "0": 0.9928571428571429,
                    "1": 0.9947136563876652,
                    "2": 0.9806201550387597,
                    "3": 0.9851485148514851,
                    "4": 0.9837067209775967,
                    "5": 0.9697309417040358,
                    "6": 0.9780793319415448,
                    "7": 0.9795719844357976,
                    "8": 0.973305954825462,
                    "9": 0.9752229930624381
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 23,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 16,
                    "wrong_to_correct": 7,
                    "correct_both": 9809,
                    "wrong_both": 168
                  },
                  "mean_donor_class_probability": 0.002191576946001161,
                  "baseline_mean_donor_class_probability": 0.0019923183986627993,
                  "prediction_agreement_with_recoloring": 0.875,
                  "mean_abs_logit_difference_from_recoloring": 3.5625182345160096
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.98182,
                  "correct": 98182,
                  "total": 100000,
                  "mean_true_class_probability": 0.974101674400114,
                  "per_digit_accuracy": {
                    "0": 0.9912244897959184,
                    "1": 0.9941850220264317,
                    "2": 0.9780038759689923,
                    "3": 0.9825742574257426,
                    "4": 0.9832993890020366,
                    "5": 0.968609865470852,
                    "6": 0.9793319415448852,
                    "7": 0.9797665369649805,
                    "8": 0.9797741273100616,
                    "9": 0.9785926660059465
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 237,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 138,
                    "wrong_to_correct": 70,
                    "correct_both": 98112,
                    "wrong_both": 1680
                  },
                  "mean_donor_class_probability": 0.10022936359891339,
                  "baseline_mean_donor_class_probability": 0.10000000039033324,
                  "prediction_agreement_with_recoloring": 0.87531,
                  "mean_abs_logit_difference_from_recoloring": 3.1948526330694462
                },
                "aligned_donor": {
                  "accuracy": 0.9827,
                  "correct": 9827,
                  "total": 10000,
                  "mean_true_class_probability": 0.9752887408589187,
                  "per_digit_accuracy": {
                    "0": 0.9897959183673469,
                    "1": 0.9947136563876652,
                    "2": 0.9815891472868217,
                    "3": 0.9841584158415841,
                    "4": 0.9837067209775967,
                    "5": 0.9697309417040358,
                    "6": 0.9812108559498957,
                    "7": 0.980544747081712,
                    "8": 0.9825462012320328,
                    "9": 0.9762140733399405
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 19,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 7,
                    "wrong_to_correct": 9,
                    "correct_both": 9818,
                    "wrong_both": 166
                  },
                  "mean_donor_class_probability": 0.9752887408589187,
                  "baseline_mean_donor_class_probability": 0.974861143469167,
                  "prediction_agreement_with_recoloring": 0.9881,
                  "mean_abs_logit_difference_from_recoloring": 0.30227487245231865
                },
                "shifted_donor": {
                  "accuracy": 0.9814,
                  "correct": 9814,
                  "total": 10000,
                  "mean_true_class_probability": 0.9734943875237361,
                  "per_digit_accuracy": {
                    "0": 0.9918367346938776,
                    "1": 0.9938325991189427,
                    "2": 0.9757751937984496,
                    "3": 0.9841584158415841,
                    "4": 0.9826883910386965,
                    "5": 0.9674887892376681,
                    "6": 0.9801670146137788,
                    "7": 0.980544747081712,
                    "8": 0.9753593429158111,
                    "9": 0.979187314172448
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 26,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 17,
                    "wrong_to_correct": 6,
                    "correct_both": 9808,
                    "wrong_both": 169
                  },
                  "mean_donor_class_probability": 0.002450200650205341,
                  "baseline_mean_donor_class_probability": 0.0019923183986627993,
                  "prediction_agreement_with_recoloring": 0.8747,
                  "mean_abs_logit_difference_from_recoloring": 3.579584260859713
                }
              }
            ]
          },
          "independent": {
            "all_palette": {
              "accuracy": 0.84688,
              "correct": 84688,
              "total": 100000,
              "mean_true_class_probability": 0.8144528189548257,
              "per_digit_accuracy": {
                "0": 0.9324489795918367,
                "1": 0.9636123348017621,
                "2": 0.8114341085271318,
                "3": 0.8253465346534653,
                "4": 0.8284114052953157,
                "5": 0.7762331838565022,
                "6": 0.898329853862213,
                "7": 0.8553501945525291,
                "8": 0.7808008213552361,
                "9": 0.7770069375619425
              },
              "unique_images": 10000,
              "donors_per_image": 10,
              "changed_predictions": 11454,
              "transitions_vs_baseline": {
                "correct_to_wrong": 6060,
                "wrong_to_correct": 3648,
                "correct_both": 81040,
                "wrong_both": 9252
              },
              "mean_donor_class_probability": 0.12805450257414425,
              "baseline_mean_donor_class_probability": 0.10000000023387637,
              "prediction_agreement_with_recoloring": 0.85241,
              "mean_abs_logit_difference_from_recoloring": 2.965173329845924
            },
            "aligned_donor": {
              "accuracy": 0.9289,
              "correct": 9289,
              "total": 10000,
              "mean_true_class_probability": 0.9037902567658881,
              "per_digit_accuracy": {
                "0": 0.9683673469387755,
                "1": 0.9577092511013215,
                "2": 0.9060077519379846,
                "3": 0.9554455445544554,
                "4": 0.9592668024439919,
                "5": 0.9383408071748879,
                "6": 0.9373695198329853,
                "7": 0.9202334630350194,
                "8": 0.86652977412731,
                "9": 0.8780971258671952
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 900,
              "transitions_vs_baseline": {
                "correct_to_wrong": 97,
                "wrong_to_correct": 676,
                "correct_both": 8613,
                "wrong_both": 614
              },
              "mean_donor_class_probability": 0.9037902567658881,
              "baseline_mean_donor_class_probability": 0.8369359211550298,
              "prediction_agreement_with_recoloring": 0.9323,
              "mean_abs_logit_difference_from_recoloring": 2.9103312909607215
            },
            "shifted_donor": {
              "accuracy": 0.7921,
              "correct": 7921,
              "total": 10000,
              "mean_true_class_probability": 0.7595009511365972,
              "per_digit_accuracy": {
                "0": 0.9112244897959184,
                "1": 0.9841409691629956,
                "2": 0.6056201550387597,
                "3": 0.7702970297029703,
                "4": 0.7942973523421588,
                "5": 0.6737668161434978,
                "6": 0.8716075156576201,
                "7": 0.7996108949416343,
                "8": 0.8572895277207392,
                "9": 0.6293359762140733
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 1364,
              "transitions_vs_baseline": {
                "correct_to_wrong": 980,
                "wrong_to_correct": 191,
                "correct_both": 7730,
                "wrong_both": 1099
              },
              "mean_donor_class_probability": 0.03915389114784166,
              "baseline_mean_donor_class_probability": 0.013621444965288066,
              "prediction_agreement_with_recoloring": 0.8248,
              "mean_abs_logit_difference_from_recoloring": 2.950358677873481
            },
            "complement_patch": {
              "all_palette": {
                "accuracy": 0.8486,
                "correct": 84860,
                "total": 100000,
                "mean_true_class_probability": 0.8150625482241308,
                "per_digit_accuracy": {
                  "0": 0.9330612244897959,
                  "1": 0.9631718061674008,
                  "2": 0.8122093023255814,
                  "3": 0.8301980198019802,
                  "4": 0.8309572301425662,
                  "5": 0.7790358744394619,
                  "6": 0.8968684759916493,
                  "7": 0.8636186770428016,
                  "8": 0.7806981519507187,
                  "9": 0.776412289395441
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 14751,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 7217,
                  "wrong_to_correct": 4977,
                  "correct_both": 79883,
                  "wrong_both": 7923
                },
                "mean_donor_class_probability": 0.14402613308082204,
                "baseline_mean_donor_class_probability": 0.10000000023387637,
                "prediction_agreement_with_recoloring": 0.88476,
                "mean_abs_logit_difference_from_recoloring": 1.3328304918068703
              },
              "aligned_donor": {
                "accuracy": 0.9661,
                "correct": 9661,
                "total": 10000,
                "mean_true_class_probability": 0.9509754241916926,
                "per_digit_accuracy": {
                  "0": 0.9948979591836735,
                  "1": 0.9964757709251101,
                  "2": 0.8662790697674418,
                  "3": 0.9386138613861386,
                  "4": 0.9765784114052953,
                  "5": 0.9641255605381166,
                  "6": 0.9791231732776617,
                  "7": 0.980544747081712,
                  "8": 0.973305954825462,
                  "9": 0.9910802775024777
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1215,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 97,
                  "wrong_to_correct": 1048,
                  "correct_both": 8613,
                  "wrong_both": 242
                },
                "mean_donor_class_probability": 0.9509754241916926,
                "baseline_mean_donor_class_probability": 0.8369359211550298,
                "prediction_agreement_with_recoloring": 0.9708,
                "mean_abs_logit_difference_from_recoloring": 1.3631346655426546
              },
              "shifted_donor": {
                "accuracy": 0.8864,
                "correct": 8864,
                "total": 10000,
                "mean_true_class_probability": 0.8463174121804034,
                "per_digit_accuracy": {
                  "0": 0.9510204081632653,
                  "1": 0.8986784140969163,
                  "2": 0.8953488372093024,
                  "3": 0.9217821782178218,
                  "4": 0.8890020366598778,
                  "5": 0.8452914798206278,
                  "6": 0.9290187891440501,
                  "7": 0.9114785992217899,
                  "8": 0.7515400410677618,
                  "9": 0.8632309217046581
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1181,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 398,
                  "wrong_to_correct": 552,
                  "correct_both": 8312,
                  "wrong_both": 738
                },
                "mean_donor_class_probability": 0.03953012654512257,
                "baseline_mean_donor_class_probability": 0.013621444965288066,
                "prediction_agreement_with_recoloring": 0.9211,
                "mean_abs_logit_difference_from_recoloring": 1.3436397909144684
              }
            },
            "recolored_reference": {
              "all_palette": {
                "accuracy": 0.87212,
                "correct": 87212,
                "total": 100000,
                "mean_true_class_probability": 0.8365256038477711,
                "per_digit_accuracy": {
                  "0": 0.9380612244897959,
                  "1": 0.974273127753304,
                  "2": 0.8486434108527132,
                  "3": 0.8669306930693069,
                  "4": 0.8793279022403259,
                  "5": 0.8089686098654708,
                  "6": 0.8872651356993737,
                  "7": 0.8836575875486381,
                  "8": 0.7941478439425051,
                  "9": 0.8203171456888008
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 14693,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 5897,
                  "wrong_to_correct": 6009,
                  "correct_both": 81203,
                  "wrong_both": 6891
                },
                "mean_donor_class_probability": 0.17803537685845774,
                "baseline_mean_donor_class_probability": 0.10000000023387637,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "aligned_donor": {
                "accuracy": 0.9944,
                "correct": 9944,
                "total": 10000,
                "mean_true_class_probability": 0.9900084610020509,
                "per_digit_accuracy": {
                  "0": 0.9979591836734694,
                  "1": 0.9964757709251101,
                  "2": 0.9961240310077519,
                  "3": 0.9900990099009901,
                  "4": 0.9938900203665988,
                  "5": 0.9966367713004485,
                  "6": 0.9864300626304802,
                  "7": 0.9931906614785992,
                  "8": 0.9989733059548255,
                  "9": 0.9940535183349851
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1248,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 1,
                  "wrong_to_correct": 1235,
                  "correct_both": 8709,
                  "wrong_both": 55
                },
                "mean_donor_class_probability": 0.9900084610020509,
                "baseline_mean_donor_class_probability": 0.8369359211550298,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "shifted_donor": {
                "accuracy": 0.8694,
                "correct": 8694,
                "total": 10000,
                "mean_true_class_probability": 0.8304767185413066,
                "per_digit_accuracy": {
                  "0": 0.9346938775510204,
                  "1": 0.9744493392070485,
                  "2": 0.7926356589147286,
                  "3": 0.9198019801980198,
                  "4": 0.860488798370672,
                  "5": 0.773542600896861,
                  "6": 0.9269311064718163,
                  "7": 0.8861867704280155,
                  "8": 0.7885010266940452,
                  "9": 0.8156590683845392
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1315,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 512,
                  "wrong_to_correct": 496,
                  "correct_both": 8198,
                  "wrong_both": 794
                },
                "mean_donor_class_probability": 0.05870254770011943,
                "baseline_mean_donor_class_probability": 0.013621444965288066,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              }
            },
            "random_controls": [
              {
                "all_palette": {
                  "accuracy": 0.87027,
                  "correct": 87027,
                  "total": 100000,
                  "mean_true_class_probability": 0.8356793887070719,
                  "per_digit_accuracy": {
                    "0": 0.9418367346938775,
                    "1": 0.9811453744493392,
                    "2": 0.8375968992248062,
                    "3": 0.8612871287128713,
                    "4": 0.8755600814663951,
                    "5": 0.8159192825112107,
                    "6": 0.8732776617954071,
                    "7": 0.8779182879377432,
                    "8": 0.7953798767967146,
                    "9": 0.8229930624380575
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 2149,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 965,
                    "wrong_to_correct": 892,
                    "correct_both": 86135,
                    "wrong_both": 12008
                  },
                  "mean_donor_class_probability": 0.10094548261880663,
                  "baseline_mean_donor_class_probability": 0.10000000023387637,
                  "prediction_agreement_with_recoloring": 0.85288,
                  "mean_abs_logit_difference_from_recoloring": 3.264918075688377
                },
                "aligned_donor": {
                  "accuracy": 0.8759,
                  "correct": 8759,
                  "total": 10000,
                  "mean_true_class_probability": 0.841748849012486,
                  "per_digit_accuracy": {
                    "0": 0.9438775510204082,
                    "1": 0.9797356828193833,
                    "2": 0.8420542635658915,
                    "3": 0.8712871287128713,
                    "4": 0.8879837067209776,
                    "5": 0.7802690582959642,
                    "6": 0.8684759916492694,
                    "7": 0.8832684824902723,
                    "8": 0.8069815195071869,
                    "9": 0.8711595639246779
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 262,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 88,
                    "wrong_to_correct": 137,
                    "correct_both": 8622,
                    "wrong_both": 1153
                  },
                  "mean_donor_class_probability": 0.841748849012486,
                  "baseline_mean_donor_class_probability": 0.8369359211550298,
                  "prediction_agreement_with_recoloring": 0.88,
                  "mean_abs_logit_difference_from_recoloring": 3.2365041534047947
                },
                "shifted_donor": {
                  "accuracy": 0.8621,
                  "correct": 8621,
                  "total": 10000,
                  "mean_true_class_probability": 0.826285104886954,
                  "per_digit_accuracy": {
                    "0": 0.9408163265306122,
                    "1": 0.9762114537444934,
                    "2": 0.8362403100775194,
                    "3": 0.8841584158415842,
                    "4": 0.8329938900203666,
                    "5": 0.8239910313901345,
                    "6": 0.8716075156576201,
                    "7": 0.8745136186770428,
                    "8": 0.8018480492813141,
                    "9": 0.7601585728444004
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 242,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 151,
                    "wrong_to_correct": 62,
                    "correct_both": 8559,
                    "wrong_both": 1228
                  },
                  "mean_donor_class_probability": 0.01422388089690533,
                  "baseline_mean_donor_class_probability": 0.013621444965288066,
                  "prediction_agreement_with_recoloring": 0.865,
                  "mean_abs_logit_difference_from_recoloring": 3.2335971591023727
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.87126,
                  "correct": 87126,
                  "total": 100000,
                  "mean_true_class_probability": 0.8374675516623591,
                  "per_digit_accuracy": {
                    "0": 0.9440816326530612,
                    "1": 0.9783259911894273,
                    "2": 0.8398255813953488,
                    "3": 0.8610891089108911,
                    "4": 0.8755600814663951,
                    "5": 0.8192825112107623,
                    "6": 0.8783924843423799,
                    "7": 0.8760700389105058,
                    "8": 0.8095482546201233,
                    "9": 0.8120911793855302
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1355,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 579,
                    "wrong_to_correct": 605,
                    "correct_both": 86521,
                    "wrong_both": 12295
                  },
                  "mean_donor_class_probability": 0.10087858975825494,
                  "baseline_mean_donor_class_probability": 0.10000000023387637,
                  "prediction_agreement_with_recoloring": 0.85343,
                  "mean_abs_logit_difference_from_recoloring": 3.2239253364023694
                },
                "aligned_donor": {
                  "accuracy": 0.8753,
                  "correct": 8753,
                  "total": 10000,
                  "mean_true_class_probability": 0.8420382843676946,
                  "per_digit_accuracy": {
                    "0": 0.9438775510204082,
                    "1": 0.9797356828193833,
                    "2": 0.8362403100775194,
                    "3": 0.8643564356435643,
                    "4": 0.8788187372708758,
                    "5": 0.8295964125560538,
                    "6": 0.8778705636743215,
                    "7": 0.8754863813229572,
                    "8": 0.8275154004106776,
                    "9": 0.8225966303270565
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 161,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 49,
                    "wrong_to_correct": 92,
                    "correct_both": 8661,
                    "wrong_both": 1198
                  },
                  "mean_donor_class_probability": 0.8420382843676946,
                  "baseline_mean_donor_class_probability": 0.8369359211550298,
                  "prediction_agreement_with_recoloring": 0.8794,
                  "mean_abs_logit_difference_from_recoloring": 3.189020750168115
                },
                "shifted_donor": {
                  "accuracy": 0.8693,
                  "correct": 8693,
                  "total": 10000,
                  "mean_true_class_probability": 0.8361150175608747,
                  "per_digit_accuracy": {
                    "0": 0.9530612244897959,
                    "1": 0.9806167400881057,
                    "2": 0.8410852713178295,
                    "3": 0.8633663366336634,
                    "4": 0.8879837067209776,
                    "5": 0.8183856502242153,
                    "6": 0.8789144050104384,
                    "7": 0.877431906614786,
                    "8": 0.7720739219712526,
                    "9": 0.800792864222002
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 144,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 73,
                    "wrong_to_correct": 56,
                    "correct_both": 8637,
                    "wrong_both": 1234
                  },
                  "mean_donor_class_probability": 0.013987110191301009,
                  "baseline_mean_donor_class_probability": 0.013621444965288066,
                  "prediction_agreement_with_recoloring": 0.8674,
                  "mean_abs_logit_difference_from_recoloring": 3.2104359844944814
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.87232,
                  "correct": 87232,
                  "total": 100000,
                  "mean_true_class_probability": 0.8384736595517611,
                  "per_digit_accuracy": {
                    "0": 0.9416326530612245,
                    "1": 0.9832599118942731,
                    "2": 0.8436046511627907,
                    "3": 0.8648514851485148,
                    "4": 0.8780040733197556,
                    "5": 0.8154708520179372,
                    "6": 0.8850730688935282,
                    "7": 0.8767509727626459,
                    "8": 0.8043121149897331,
                    "9": 0.810802775024777
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1694,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 633,
                    "wrong_to_correct": 765,
                    "correct_both": 86467,
                    "wrong_both": 12135
                  },
                  "mean_donor_class_probability": 0.10099305584152236,
                  "baseline_mean_donor_class_probability": 0.10000000023387637,
                  "prediction_agreement_with_recoloring": 0.8548,
                  "mean_abs_logit_difference_from_recoloring": 3.2211298829259647
                },
                "aligned_donor": {
                  "accuracy": 0.8762,
                  "correct": 8762,
                  "total": 10000,
                  "mean_true_class_probability": 0.8418835630610587,
                  "per_digit_accuracy": {
                    "0": 0.9306122448979591,
                    "1": 0.9859030837004406,
                    "2": 0.8662790697674418,
                    "3": 0.8831683168316832,
                    "4": 0.8767820773930753,
                    "5": 0.8139013452914798,
                    "6": 0.9039665970772442,
                    "7": 0.877431906614786,
                    "8": 0.811088295687885,
                    "9": 0.7928642220019821
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 215,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 66,
                    "wrong_to_correct": 118,
                    "correct_both": 8644,
                    "wrong_both": 1172
                  },
                  "mean_donor_class_probability": 0.8418835630610587,
                  "baseline_mean_donor_class_probability": 0.8369359211550298,
                  "prediction_agreement_with_recoloring": 0.8801,
                  "mean_abs_logit_difference_from_recoloring": 3.204215446637161
                },
                "shifted_donor": {
                  "accuracy": 0.8702,
                  "correct": 8702,
                  "total": 10000,
                  "mean_true_class_probability": 0.8360681151290384,
                  "per_digit_accuracy": {
                    "0": 0.9530612244897959,
                    "1": 0.9753303964757709,
                    "2": 0.8333333333333334,
                    "3": 0.8782178217821782,
                    "4": 0.8737270875763747,
                    "5": 0.8183856502242153,
                    "6": 0.8830897703549061,
                    "7": 0.8784046692607004,
                    "8": 0.7761806981519507,
                    "9": 0.8136769078295342
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 175,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 74,
                    "wrong_to_correct": 66,
                    "correct_both": 8636,
                    "wrong_both": 1224
                  },
                  "mean_donor_class_probability": 0.0145828408647706,
                  "baseline_mean_donor_class_probability": 0.013621444965288066,
                  "prediction_agreement_with_recoloring": 0.8686,
                  "mean_abs_logit_difference_from_recoloring": 3.20059480623791
                }
              }
            ]
          },
          "misleading": {
            "all_palette": {
              "accuracy": 0.88088,
              "correct": 88088,
              "total": 100000,
              "mean_true_class_probability": 0.8436732530755718,
              "per_digit_accuracy": {
                "0": 0.9408163265306122,
                "1": 0.8985022026431718,
                "2": 0.8813953488372093,
                "3": 0.9194059405940594,
                "4": 0.8822810590631365,
                "5": 0.8375560538116592,
                "6": 0.927035490605428,
                "7": 0.9031128404669261,
                "8": 0.7618069815195072,
                "9": 0.8491575817641229
              },
              "unique_images": 10000,
              "donors_per_image": 10,
              "changed_predictions": 8452,
              "transitions_vs_baseline": {
                "correct_to_wrong": 2833,
                "wrong_to_correct": 4011,
                "correct_both": 84077,
                "wrong_both": 9079
              },
              "mean_donor_class_probability": 0.12163397220882106,
              "baseline_mean_donor_class_probability": 0.1000000001551866,
              "prediction_agreement_with_recoloring": 0.88093,
              "mean_abs_logit_difference_from_recoloring": 2.9389959092601705
            },
            "aligned_donor": {
              "accuracy": 0.931,
              "correct": 9310,
              "total": 10000,
              "mean_true_class_probability": 0.8995197152110795,
              "per_digit_accuracy": {
                "0": 0.9673469387755103,
                "1": 0.9048458149779736,
                "2": 0.9312015503875969,
                "3": 0.9693069306930693,
                "4": 0.9663951120162932,
                "5": 0.9562780269058296,
                "6": 0.9446764091858038,
                "7": 0.9464980544747081,
                "8": 0.813141683778234,
                "9": 0.9147670961347869
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 983,
              "transitions_vs_baseline": {
                "correct_to_wrong": 109,
                "wrong_to_correct": 728,
                "correct_both": 8582,
                "wrong_both": 581
              },
              "mean_donor_class_probability": 0.8995197152110795,
              "baseline_mean_donor_class_probability": 0.8311145776122599,
              "prediction_agreement_with_recoloring": 0.9344,
              "mean_abs_logit_difference_from_recoloring": 3.2118620517123118
            },
            "shifted_donor": {
              "accuracy": 0.8616,
              "correct": 8616,
              "total": 10000,
              "mean_true_class_probability": 0.8238193958355176,
              "per_digit_accuracy": {
                "0": 0.9275510204081633,
                "1": 0.9753303964757709,
                "2": 0.7819767441860465,
                "3": 0.901980198019802,
                "4": 0.8523421588594705,
                "5": 0.7600896860986547,
                "6": 0.9269311064718163,
                "7": 0.8745136186770428,
                "8": 0.7997946611909651,
                "9": 0.7938553022794846
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 153,
              "transitions_vs_baseline": {
                "correct_to_wrong": 101,
                "wrong_to_correct": 26,
                "correct_both": 8590,
                "wrong_both": 1283
              },
              "mean_donor_class_probability": 0.05544166726065273,
              "baseline_mean_donor_class_probability": 0.052920170606241934,
              "prediction_agreement_with_recoloring": 0.98,
              "mean_abs_logit_difference_from_recoloring": 0.3198295195020735
            },
            "complement_patch": {
              "all_palette": {
                "accuracy": 0.79998,
                "correct": 79998,
                "total": 100000,
                "mean_true_class_probability": 0.765404136350644,
                "per_digit_accuracy": {
                  "0": 0.9136734693877551,
                  "1": 0.9813215859030837,
                  "2": 0.6134689922480621,
                  "3": 0.7907920792079208,
                  "4": 0.8004073319755601,
                  "5": 0.6823991031390134,
                  "6": 0.8747390396659708,
                  "7": 0.8198443579766537,
                  "8": 0.8476386036960986,
                  "9": 0.6518334985133796
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 17265,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 10635,
                  "wrong_to_correct": 3723,
                  "correct_both": 76275,
                  "wrong_both": 9367
                },
                "mean_donor_class_probability": 0.14754375791367488,
                "baseline_mean_donor_class_probability": 0.1000000001551866,
                "prediction_agreement_with_recoloring": 0.8624,
                "mean_abs_logit_difference_from_recoloring": 1.346151452479327
              },
              "aligned_donor": {
                "accuracy": 0.9233,
                "correct": 9233,
                "total": 10000,
                "mean_true_class_probability": 0.9101017426000136,
                "per_digit_accuracy": {
                  "0": 0.9846938775510204,
                  "1": 0.9973568281938326,
                  "2": 0.5164728682170543,
                  "3": 0.9227722772277228,
                  "4": 0.9745417515274949,
                  "5": 0.9260089686098655,
                  "6": 0.9791231732776617,
                  "7": 0.9688715953307393,
                  "8": 0.9897330595482546,
                  "9": 0.981169474727453
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1433,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 393,
                  "wrong_to_correct": 935,
                  "correct_both": 8298,
                  "wrong_both": 374
                },
                "mean_donor_class_probability": 0.9101017426000136,
                "baseline_mean_donor_class_probability": 0.8311145776122599,
                "prediction_agreement_with_recoloring": 0.9275,
                "mean_abs_logit_difference_from_recoloring": 1.8323426652615518
              },
              "shifted_donor": {
                "accuracy": 0.8706,
                "correct": 8706,
                "total": 10000,
                "mean_true_class_probability": 0.8315232426776312,
                "per_digit_accuracy": {
                  "0": 0.9346938775510204,
                  "1": 0.9621145374449339,
                  "2": 0.8071705426356589,
                  "3": 0.9178217821782179,
                  "4": 0.8655804480651731,
                  "5": 0.7802690582959642,
                  "6": 0.9279749478079332,
                  "7": 0.892023346303502,
                  "8": 0.7792607802874744,
                  "9": 0.819623389494549
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 160,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 54,
                  "wrong_to_correct": 69,
                  "correct_both": 8637,
                  "wrong_both": 1240
                },
                "mean_donor_class_probability": 0.05641310895019553,
                "baseline_mean_donor_class_probability": 0.052920170606241934,
                "prediction_agreement_with_recoloring": 0.9891,
                "mean_abs_logit_difference_from_recoloring": 0.14381200752466916
              }
            },
            "recolored_reference": {
              "all_palette": {
                "accuracy": 0.87212,
                "correct": 87212,
                "total": 100000,
                "mean_true_class_probability": 0.8365256038477711,
                "per_digit_accuracy": {
                  "0": 0.9380612244897959,
                  "1": 0.974273127753304,
                  "2": 0.8486434108527132,
                  "3": 0.8669306930693069,
                  "4": 0.8793279022403259,
                  "5": 0.8089686098654708,
                  "6": 0.8872651356993737,
                  "7": 0.8836575875486381,
                  "8": 0.7941478439425051,
                  "9": 0.8203171456888008
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 13520,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 5088,
                  "wrong_to_correct": 5390,
                  "correct_both": 81822,
                  "wrong_both": 7700
                },
                "mean_donor_class_probability": 0.17803537685845774,
                "baseline_mean_donor_class_probability": 0.1000000001551866,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "aligned_donor": {
                "accuracy": 0.9944,
                "correct": 9944,
                "total": 10000,
                "mean_true_class_probability": 0.9900084610020509,
                "per_digit_accuracy": {
                  "0": 0.9979591836734694,
                  "1": 0.9964757709251101,
                  "2": 0.9961240310077519,
                  "3": 0.9900990099009901,
                  "4": 0.9938900203665988,
                  "5": 0.9966367713004485,
                  "6": 0.9864300626304802,
                  "7": 0.9931906614785992,
                  "8": 0.9989733059548255,
                  "9": 0.9940535183349851
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1270,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 0,
                  "wrong_to_correct": 1253,
                  "correct_both": 8691,
                  "wrong_both": 56
                },
                "mean_donor_class_probability": 0.9900084610020509,
                "baseline_mean_donor_class_probability": 0.8311145776122599,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "shifted_donor": {
                "accuracy": 0.8694,
                "correct": 8694,
                "total": 10000,
                "mean_true_class_probability": 0.8304767185413066,
                "per_digit_accuracy": {
                  "0": 0.9346938775510204,
                  "1": 0.9744493392070485,
                  "2": 0.7926356589147286,
                  "3": 0.9198019801980198,
                  "4": 0.860488798370672,
                  "5": 0.773542600896861,
                  "6": 0.9269311064718163,
                  "7": 0.8861867704280155,
                  "8": 0.7885010266940452,
                  "9": 0.8156590683845392
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 168,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 60,
                  "wrong_to_correct": 63,
                  "correct_both": 8631,
                  "wrong_both": 1246
                },
                "mean_donor_class_probability": 0.05870254770011943,
                "baseline_mean_donor_class_probability": 0.052920170606241934,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              }
            },
            "random_controls": [
              {
                "all_palette": {
                  "accuracy": 0.87543,
                  "correct": 87543,
                  "total": 100000,
                  "mean_true_class_probability": 0.8403673618397832,
                  "per_digit_accuracy": {
                    "0": 0.9314285714285714,
                    "1": 0.9776211453744493,
                    "2": 0.8044573643410853,
                    "3": 0.9013861386138614,
                    "4": 0.8769857433808554,
                    "5": 0.7711883408071749,
                    "6": 0.9250521920668059,
                    "7": 0.8899805447470817,
                    "8": 0.7938398357289528,
                    "9": 0.8601585728444004
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 2107,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 525,
                    "wrong_to_correct": 1158,
                    "correct_both": 86385,
                    "wrong_both": 11932
                  },
                  "mean_donor_class_probability": 0.10094055143746705,
                  "baseline_mean_donor_class_probability": 0.1000000001551866,
                  "prediction_agreement_with_recoloring": 0.86813,
                  "mean_abs_logit_difference_from_recoloring": 3.220443365824595
                },
                "aligned_donor": {
                  "accuracy": 0.8783,
                  "correct": 8783,
                  "total": 10000,
                  "mean_true_class_probability": 0.8442543475316637,
                  "per_digit_accuracy": {
                    "0": 0.9306122448979591,
                    "1": 0.9753303964757709,
                    "2": 0.814922480620155,
                    "3": 0.9099009900990099,
                    "4": 0.8818737270875764,
                    "5": 0.7365470852017937,
                    "6": 0.9217118997912317,
                    "7": 0.8959143968871596,
                    "8": 0.797741273100616,
                    "9": 0.8919722497522299
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 285,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 66,
                    "wrong_to_correct": 158,
                    "correct_both": 8625,
                    "wrong_both": 1151
                  },
                  "mean_donor_class_probability": 0.8442543475316637,
                  "baseline_mean_donor_class_probability": 0.8311145776122599,
                  "prediction_agreement_with_recoloring": 0.8823,
                  "mean_abs_logit_difference_from_recoloring": 3.6037073729563134
                },
                "shifted_donor": {
                  "accuracy": 0.8686,
                  "correct": 8686,
                  "total": 10000,
                  "mean_true_class_probability": 0.8299317050755053,
                  "per_digit_accuracy": {
                    "0": 0.9295918367346939,
                    "1": 0.9744493392070485,
                    "2": 0.8003875968992248,
                    "3": 0.9158415841584159,
                    "4": 0.8574338085539714,
                    "5": 0.7757847533632287,
                    "6": 0.9248434237995825,
                    "7": 0.8881322957198443,
                    "8": 0.7895277207392197,
                    "9": 0.8087215064420218
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 23,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 13,
                    "wrong_to_correct": 8,
                    "correct_both": 8678,
                    "wrong_both": 1301
                  },
                  "mean_donor_class_probability": 0.053087171849589485,
                  "baseline_mean_donor_class_probability": 0.052920170606241934,
                  "prediction_agreement_with_recoloring": 0.9828,
                  "mean_abs_logit_difference_from_recoloring": 0.34939068830251696
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.86805,
                  "correct": 86805,
                  "total": 100000,
                  "mean_true_class_probability": 0.8278333565910078,
                  "per_digit_accuracy": {
                    "0": 0.9151020408163265,
                    "1": 0.9709251101321585,
                    "2": 0.806298449612403,
                    "3": 0.9123762376237624,
                    "4": 0.84989816700611,
                    "5": 0.7797085201793722,
                    "6": 0.9272442588726514,
                    "7": 0.8859922178988326,
                    "8": 0.8003080082135524,
                    "9": 0.8120911793855302
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1575,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 704,
                    "wrong_to_correct": 599,
                    "correct_both": 86206,
                    "wrong_both": 12491
                  },
                  "mean_donor_class_probability": 0.10081743211484943,
                  "baseline_mean_donor_class_probability": 0.1000000001551866,
                  "prediction_agreement_with_recoloring": 0.86411,
                  "mean_abs_logit_difference_from_recoloring": 3.193961534773373
                },
                "aligned_donor": {
                  "accuracy": 0.8698,
                  "correct": 8698,
                  "total": 10000,
                  "mean_true_class_probability": 0.8311290353686054,
                  "per_digit_accuracy": {
                    "0": 0.9122448979591836,
                    "1": 0.9744493392070485,
                    "2": 0.8158914728682171,
                    "3": 0.9128712871287129,
                    "4": 0.8513238289205702,
                    "5": 0.7881165919282511,
                    "6": 0.9290187891440501,
                    "7": 0.8891050583657587,
                    "8": 0.8121149897330595,
                    "9": 0.7928642220019821
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 214,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 87,
                    "wrong_to_correct": 94,
                    "correct_both": 8604,
                    "wrong_both": 1215
                  },
                  "mean_donor_class_probability": 0.8311290353686054,
                  "baseline_mean_donor_class_probability": 0.8311145776122599,
                  "prediction_agreement_with_recoloring": 0.8736,
                  "mean_abs_logit_difference_from_recoloring": 3.5479963135582953
                },
                "shifted_donor": {
                  "accuracy": 0.869,
                  "correct": 8690,
                  "total": 10000,
                  "mean_true_class_probability": 0.8309870360344818,
                  "per_digit_accuracy": {
                    "0": 0.9346938775510204,
                    "1": 0.9744493392070485,
                    "2": 0.7994186046511628,
                    "3": 0.9128712871287129,
                    "4": 0.8594704684317719,
                    "5": 0.7746636771300448,
                    "6": 0.9269311064718163,
                    "7": 0.8891050583657587,
                    "8": 0.7833675564681725,
                    "9": 0.8136769078295342
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 12,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 5,
                    "wrong_to_correct": 4,
                    "correct_both": 8686,
                    "wrong_both": 1305
                  },
                  "mean_donor_class_probability": 0.052990029542856414,
                  "baseline_mean_donor_class_probability": 0.052920170606241934,
                  "prediction_agreement_with_recoloring": 0.9835,
                  "mean_abs_logit_difference_from_recoloring": 0.3466972352211922
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.87007,
                  "correct": 87007,
                  "total": 100000,
                  "mean_true_class_probability": 0.8331460023060837,
                  "per_digit_accuracy": {
                    "0": 0.9195918367346939,
                    "1": 0.9802643171806168,
                    "2": 0.8184108527131783,
                    "3": 0.9104950495049505,
                    "4": 0.8625254582484725,
                    "5": 0.7771300448430494,
                    "6": 0.9275574112734865,
                    "7": 0.8859922178988326,
                    "8": 0.7847022587268994,
                    "9": 0.8114965312190288
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1625,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 578,
                    "wrong_to_correct": 675,
                    "correct_both": 86332,
                    "wrong_both": 12415
                  },
                  "mean_donor_class_probability": 0.10064950796616075,
                  "baseline_mean_donor_class_probability": 0.1000000001551866,
                  "prediction_agreement_with_recoloring": 0.86539,
                  "mean_abs_logit_difference_from_recoloring": 3.1842936426099446
                },
                "aligned_donor": {
                  "accuracy": 0.8722,
                  "correct": 8722,
                  "total": 10000,
                  "mean_true_class_probability": 0.8344126276167848,
                  "per_digit_accuracy": {
                    "0": 0.9091836734693878,
                    "1": 0.9823788546255506,
                    "2": 0.8430232558139535,
                    "3": 0.9207920792079208,
                    "4": 0.8625254582484725,
                    "5": 0.7769058295964125,
                    "6": 0.94258872651357,
                    "7": 0.8871595330739299,
                    "8": 0.7823408624229979,
                    "9": 0.7918731417244796
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 228,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 73,
                    "wrong_to_correct": 104,
                    "correct_both": 8618,
                    "wrong_both": 1205
                  },
                  "mean_donor_class_probability": 0.8344126276167848,
                  "baseline_mean_donor_class_probability": 0.8311145776122599,
                  "prediction_agreement_with_recoloring": 0.8762,
                  "mean_abs_logit_difference_from_recoloring": 3.5667896652939173
                },
                "shifted_donor": {
                  "accuracy": 0.8694,
                  "correct": 8694,
                  "total": 10000,
                  "mean_true_class_probability": 0.8309094885391741,
                  "per_digit_accuracy": {
                    "0": 0.9316326530612244,
                    "1": 0.9726872246696036,
                    "2": 0.7994186046511628,
                    "3": 0.9148514851485149,
                    "4": 0.860488798370672,
                    "5": 0.7757847533632287,
                    "6": 0.9269311064718163,
                    "7": 0.8891050583657587,
                    "8": 0.7885010266940452,
                    "9": 0.8136769078295342
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 13,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 4,
                    "wrong_to_correct": 7,
                    "correct_both": 8687,
                    "wrong_both": 1302
                  },
                  "mean_donor_class_probability": 0.053060919428217196,
                  "baseline_mean_donor_class_probability": 0.052920170606241934,
                  "prediction_agreement_with_recoloring": 0.9837,
                  "mean_abs_logit_difference_from_recoloring": 0.34580067202102394
                }
              }
            ]
          }
        },
        "checks": {
          "self_patch_logits_max_abs_diff": 0.0,
          "selected_coordinates_max_abs_diff": 4.1961669921875e-05,
          "complementary_logit_effects_max_abs_diff": 4.57763671875e-05,
          "perpendicular_component_max_abs_diff": 1.52587890625e-05,
          "full_patch_vs_donor_logits_max_abs_diff": 3.814697265625e-06
        }
      },
      "checks": {
        "learned_tensors_identical": true,
        "identity_at_zero_strength_matches_baseline": true,
        "U_orthonormal": true,
        "UUT_idempotent": true,
        "wrapped_vs_direct_logits_max_abs_diff": 9.5367431640625e-06,
        "null_projection_logits_minus_bias_max_abs": 3.0174851417541504e-05,
        "row_projection_logits_max_abs_diff": 4.1961669921875e-05,
        "head_rank": 10,
        "row_space_dim": 10,
        "null_space_dim": 118
      }
    },
    {
      "seed": 44,
      "baseline_validation": {
        "matched": 0.9768,
        "independent": 0.8616,
        "misleading": 0.8604
      },
      "baseline_test": {
        "matched": {
          "accuracy": 0.9802,
          "correct": 9802,
          "total": 10000,
          "mean_true_class_probability": 0.9730798745489537,
          "per_digit_accuracy": {
            "0": 0.9908163265306122,
            "1": 0.9938325991189427,
            "2": 0.9777131782945736,
            "3": 0.9881188118811881,
            "4": 0.9694501018329938,
            "5": 0.9719730941704036,
            "6": 0.9832985386221295,
            "7": 0.9776264591439688,
            "8": 0.9702258726899384,
            "9": 0.9762140733399405
          }
        },
        "independent": {
          "accuracy": 0.8723,
          "correct": 8723,
          "total": 10000,
          "mean_true_class_probability": 0.836104625771648,
          "per_digit_accuracy": {
            "0": 0.9510204081632653,
            "1": 0.9770925110132158,
            "2": 0.8362403100775194,
            "3": 0.8514851485148515,
            "4": 0.8126272912423625,
            "5": 0.8183856502242153,
            "6": 0.9217118997912317,
            "7": 0.867704280155642,
            "8": 0.8049281314168378,
            "9": 0.8642220019821606
          }
        },
        "misleading": {
          "accuracy": 0.8649,
          "correct": 8649,
          "total": 10000,
          "mean_true_class_probability": 0.8283011444443807,
          "per_digit_accuracy": {
            "0": 0.936734693877551,
            "1": 0.9541850220264317,
            "2": 0.7587209302325582,
            "3": 0.9188118811881189,
            "4": 0.7647657841140529,
            "5": 0.7892376681614349,
            "6": 0.9405010438413361,
            "7": 0.8900778210116731,
            "8": 0.8100616016427105,
            "9": 0.869177403369673
          }
        }
      },
      "spectrum_variance_fraction": [
        0.3236301839351654,
        0.20880386233329773,
        0.18533866107463837,
        0.043627094477415085,
        0.03025248274207115,
        0.0233417060226202,
        0.021094081923365593,
        0.015902908518910408,
        0.012906600721180439,
        0.010419477708637714,
        0.009241445921361446,
        0.00879752729088068,
        0.007889905013144016,
        0.007070882711559534,
        0.005786007735878229,
        0.005642694886773825,
        0.004968133755028248,
        0.004606721922755241,
        0.004516467452049255,
        0.003877977142110467,
        0.0037819822318851948,
        0.0034334463998675346,
        0.003054908476769924,
        0.002838705200701952,
        0.0024676185566931963,
        0.0024096742272377014,
        0.0023639430291950703,
        0.0020985763985663652,
        0.001881699892692268,
        0.001682780683040619,
        0.0015509045915678144,
        0.0015108392108231783
      ],
      "validation_grid": [
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9766,
            "independent": 0.865,
            "misleading": 0.8714
          },
          "objective": 0.9043333333333333
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.9756,
            "independent": 0.8644,
            "misleading": 0.8808
          },
          "objective": 0.9069333333333333
        },
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9766,
            "independent": 0.8602,
            "misleading": 0.8608
          },
          "objective": 0.8992
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9768,
            "independent": 0.8598,
            "misleading": 0.8608
          },
          "objective": 0.8991333333333333
        },
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9772,
            "independent": 0.8588,
            "misleading": 0.8584
          },
          "objective": 0.8981333333333333
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9764,
            "independent": 0.8568,
            "misleading": 0.855
          },
          "objective": 0.8960666666666667
        },
        {
          "rank": 1,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9768,
            "independent": 0.8584,
            "misleading": 0.8568
          },
          "objective": 0.8973333333333334
        },
        {
          "rank": 1,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.977,
            "independent": 0.857,
            "misleading": 0.8542
          },
          "objective": 0.8960666666666667
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9768,
            "independent": 0.8716,
            "misleading": 0.8848
          },
          "objective": 0.9110666666666667
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.9704,
            "independent": 0.87,
            "misleading": 0.9044
          },
          "objective": 0.9149333333333334
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9772,
            "independent": 0.8584,
            "misleading": 0.8592
          },
          "objective": 0.8982666666666667
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.977,
            "independent": 0.8582,
            "misleading": 0.8576
          },
          "objective": 0.8976000000000001
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9768,
            "independent": 0.8594,
            "misleading": 0.8598
          },
          "objective": 0.8986666666666667
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9764,
            "independent": 0.8582,
            "misleading": 0.8548
          },
          "objective": 0.8964666666666666
        },
        {
          "rank": 2,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9766,
            "independent": 0.8582,
            "misleading": 0.859
          },
          "objective": 0.8979333333333334
        },
        {
          "rank": 2,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9774,
            "independent": 0.8548,
            "misleading": 0.8554
          },
          "objective": 0.8958666666666667
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9662,
            "independent": 0.8468,
            "misleading": 0.8786
          },
          "objective": 0.8972000000000001
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.8852,
            "independent": 0.7512,
            "misleading": 0.8158
          },
          "objective": 0.8174
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9768,
            "independent": 0.859,
            "misleading": 0.859
          },
          "objective": 0.8982666666666667
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9766,
            "independent": 0.852,
            "misleading": 0.8532
          },
          "objective": 0.8939333333333334
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9764,
            "independent": 0.8582,
            "misleading": 0.856
          },
          "objective": 0.8968666666666666
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9762,
            "independent": 0.8552,
            "misleading": 0.8522
          },
          "objective": 0.8945333333333333
        },
        {
          "rank": 3,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9774,
            "independent": 0.8628,
            "misleading": 0.8606
          },
          "objective": 0.9002666666666667
        },
        {
          "rank": 3,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9774,
            "independent": 0.8568,
            "misleading": 0.8574
          },
          "objective": 0.8972000000000001
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.964,
            "independent": 0.843,
            "misleading": 0.8666
          },
          "objective": 0.8912
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.8622,
            "independent": 0.7526,
            "misleading": 0.7828
          },
          "objective": 0.7992
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9762,
            "independent": 0.8582,
            "misleading": 0.855
          },
          "objective": 0.8964666666666666
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.976,
            "independent": 0.8424,
            "misleading": 0.8482
          },
          "objective": 0.8888666666666666
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9772,
            "independent": 0.8602,
            "misleading": 0.8532
          },
          "objective": 0.8968666666666666
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9778,
            "independent": 0.8576,
            "misleading": 0.8468
          },
          "objective": 0.8940666666666667
        },
        {
          "rank": 4,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9756,
            "independent": 0.8442,
            "misleading": 0.8484
          },
          "objective": 0.8893999999999999
        },
        {
          "rank": 4,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9682,
            "independent": 0.8074,
            "misleading": 0.8142
          },
          "objective": 0.8632666666666666
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.968,
            "independent": 0.8344,
            "misleading": 0.8568
          },
          "objective": 0.8864000000000001
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.7822,
            "independent": 0.6352,
            "misleading": 0.6694
          },
          "objective": 0.6956000000000001
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9774,
            "independent": 0.8608,
            "misleading": 0.85
          },
          "objective": 0.8960666666666667
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9738,
            "independent": 0.8466,
            "misleading": 0.8264
          },
          "objective": 0.8822666666666666
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9758,
            "independent": 0.861,
            "misleading": 0.8612
          },
          "objective": 0.8993333333333333
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9764,
            "independent": 0.8556,
            "misleading": 0.8542
          },
          "objective": 0.8954
        },
        {
          "rank": 8,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.976,
            "independent": 0.8618,
            "misleading": 0.8586
          },
          "objective": 0.8988
        },
        {
          "rank": 8,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9756,
            "independent": 0.8566,
            "misleading": 0.8514
          },
          "objective": 0.8945333333333334
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9648,
            "independent": 0.8194,
            "misleading": 0.8394
          },
          "objective": 0.8745333333333334
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.3628,
            "independent": 0.345,
            "misleading": 0.3446
          },
          "objective": 0.3508
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9752,
            "independent": 0.8558,
            "misleading": 0.8568
          },
          "objective": 0.8959333333333334
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.97,
            "independent": 0.8362,
            "misleading": 0.848
          },
          "objective": 0.8847333333333333
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9746,
            "independent": 0.8558,
            "misleading": 0.8642
          },
          "objective": 0.8982
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.968,
            "independent": 0.8268,
            "misleading": 0.8462
          },
          "objective": 0.8803333333333333
        },
        {
          "rank": 16,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9762,
            "independent": 0.8482,
            "misleading": 0.856
          },
          "objective": 0.8934666666666665
        },
        {
          "rank": 16,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.961,
            "independent": 0.8056,
            "misleading": 0.8328
          },
          "objective": 0.8664666666666667
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "colour",
          "validation": {
            "matched": 0.9758,
            "independent": 0.848,
            "misleading": 0.853
          },
          "objective": 0.8922666666666667
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "colour",
          "validation": {
            "matched": 0.2152,
            "independent": 0.1702,
            "misleading": 0.164
          },
          "objective": 0.18313333333333334
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "random_0",
          "validation": {
            "matched": 0.9712,
            "independent": 0.8326,
            "misleading": 0.844
          },
          "objective": 0.8825999999999999
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "random_0",
          "validation": {
            "matched": 0.9094,
            "independent": 0.7106,
            "misleading": 0.727
          },
          "objective": 0.7823333333333333
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "random_1",
          "validation": {
            "matched": 0.9776,
            "independent": 0.8514,
            "misleading": 0.843
          },
          "objective": 0.8906666666666667
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "random_1",
          "validation": {
            "matched": 0.9654,
            "independent": 0.7846,
            "misleading": 0.7734
          },
          "objective": 0.8411333333333334
        },
        {
          "rank": 32,
          "strength": 0.5,
          "kind": "random_2",
          "validation": {
            "matched": 0.9744,
            "independent": 0.8342,
            "misleading": 0.8234
          },
          "objective": 0.8773333333333334
        },
        {
          "rank": 32,
          "strength": 1.0,
          "kind": "random_2",
          "validation": {
            "matched": 0.9512,
            "independent": 0.7754,
            "misleading": 0.7464
          },
          "objective": 0.8243333333333333
        }
      ],
      "selected": {
        "rank": 2,
        "strength": 1.0,
        "validation_objective": 0.9149333333333334
      },
      "selected_test": {
        "matched": {
          "accuracy": 0.9743,
          "correct": 9743,
          "total": 10000,
          "mean_true_class_probability": 0.9608689131410134,
          "per_digit_accuracy": {
            "0": 0.9918367346938776,
            "1": 0.9947136563876652,
            "2": 0.9670542635658915,
            "3": 0.9603960396039604,
            "4": 0.9562118126272913,
            "5": 0.9540358744394619,
            "6": 0.9812108559498957,
            "7": 0.9815175097276264,
            "8": 0.971252566735113,
            "9": 0.9801783944499505
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 97,
            "wrong_to_correct": 38,
            "correct_both": 9705,
            "wrong_both": 160
          }
        },
        "independent": {
          "accuracy": 0.877,
          "correct": 8770,
          "total": 10000,
          "mean_true_class_probability": 0.8435368923291843,
          "per_digit_accuracy": {
            "0": 0.9510204081632653,
            "1": 0.9797356828193833,
            "2": 0.8691860465116279,
            "3": 0.8267326732673267,
            "4": 0.8289205702647657,
            "5": 0.8150224215246636,
            "6": 0.9352818371607515,
            "7": 0.8832684824902723,
            "8": 0.8182751540041068,
            "9": 0.844400396432111
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 288,
            "wrong_to_correct": 335,
            "correct_both": 8435,
            "wrong_both": 942
          }
        },
        "misleading": {
          "accuracy": 0.912,
          "correct": 9120,
          "total": 10000,
          "mean_true_class_probability": 0.883480634181061,
          "per_digit_accuracy": {
            "0": 0.9571428571428572,
            "1": 0.9665198237885463,
            "2": 0.8943798449612403,
            "3": 0.9386138613861386,
            "4": 0.8727087576374746,
            "5": 0.899103139013453,
            "6": 0.964509394572025,
            "7": 0.8998054474708171,
            "8": 0.8039014373716632,
            "9": 0.9147670961347869
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 48,
            "wrong_to_correct": 519,
            "correct_both": 8601,
            "wrong_both": 832
          }
        }
      },
      "selected_test_full_strength": {
        "matched": {
          "accuracy": 0.9743,
          "correct": 9743,
          "total": 10000,
          "mean_true_class_probability": 0.9608689131410134,
          "per_digit_accuracy": {
            "0": 0.9918367346938776,
            "1": 0.9947136563876652,
            "2": 0.9670542635658915,
            "3": 0.9603960396039604,
            "4": 0.9562118126272913,
            "5": 0.9540358744394619,
            "6": 0.9812108559498957,
            "7": 0.9815175097276264,
            "8": 0.971252566735113,
            "9": 0.9801783944499505
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 97,
            "wrong_to_correct": 38,
            "correct_both": 9705,
            "wrong_both": 160
          }
        },
        "independent": {
          "accuracy": 0.877,
          "correct": 8770,
          "total": 10000,
          "mean_true_class_probability": 0.8435368923291843,
          "per_digit_accuracy": {
            "0": 0.9510204081632653,
            "1": 0.9797356828193833,
            "2": 0.8691860465116279,
            "3": 0.8267326732673267,
            "4": 0.8289205702647657,
            "5": 0.8150224215246636,
            "6": 0.9352818371607515,
            "7": 0.8832684824902723,
            "8": 0.8182751540041068,
            "9": 0.844400396432111
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 288,
            "wrong_to_correct": 335,
            "correct_both": 8435,
            "wrong_both": 942
          }
        },
        "misleading": {
          "accuracy": 0.912,
          "correct": 9120,
          "total": 10000,
          "mean_true_class_probability": 0.883480634181061,
          "per_digit_accuracy": {
            "0": 0.9571428571428572,
            "1": 0.9665198237885463,
            "2": 0.8943798449612403,
            "3": 0.9386138613861386,
            "4": 0.8727087576374746,
            "5": 0.899103139013453,
            "6": 0.964509394572025,
            "7": 0.8998054474708171,
            "8": 0.8039014373716632,
            "9": 0.9147670961347869
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 48,
            "wrong_to_correct": 519,
            "correct_both": 8601,
            "wrong_both": 832
          }
        }
      },
      "random_control_test": [
        {
          "matched": {
            "accuracy": 0.9803,
            "correct": 9803,
            "total": 10000,
            "mean_true_class_probability": 0.9720936041375233,
            "per_digit_accuracy": {
              "0": 0.9908163265306122,
              "1": 0.9955947136563876,
              "2": 0.9767441860465116,
              "3": 0.9881188118811881,
              "4": 0.9725050916496945,
              "5": 0.9742152466367713,
              "6": 0.9780793319415448,
              "7": 0.980544747081712,
              "8": 0.9702258726899384,
              "9": 0.9732408325074331
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 14,
              "wrong_to_correct": 15,
              "correct_both": 9788,
              "wrong_both": 183
            }
          },
          "independent": {
            "accuracy": 0.8679,
            "correct": 8679,
            "total": 10000,
            "mean_true_class_probability": 0.8307233216806468,
            "per_digit_accuracy": {
              "0": 0.9357142857142857,
              "1": 0.9788546255506608,
              "2": 0.8207364341085271,
              "3": 0.8554455445544554,
              "4": 0.8279022403258656,
              "5": 0.8217488789237668,
              "6": 0.8977035490605428,
              "7": 0.9075875486381323,
              "8": 0.8090349075975359,
              "9": 0.8057482656095144
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 136,
              "wrong_to_correct": 92,
              "correct_both": 8587,
              "wrong_both": 1185
            }
          },
          "misleading": {
            "accuracy": 0.8664,
            "correct": 8664,
            "total": 10000,
            "mean_true_class_probability": 0.8276481337944585,
            "per_digit_accuracy": {
              "0": 0.9112244897959184,
              "1": 0.9656387665198238,
              "2": 0.7238372093023255,
              "3": 0.9227722772277228,
              "4": 0.785132382892057,
              "5": 0.7937219730941704,
              "6": 0.9248434237995825,
              "7": 0.9241245136186771,
              "8": 0.8357289527720739,
              "9": 0.8592666005946482
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 96,
              "wrong_to_correct": 111,
              "correct_both": 8553,
              "wrong_both": 1240
            }
          }
        },
        {
          "matched": {
            "accuracy": 0.9794,
            "correct": 9794,
            "total": 10000,
            "mean_true_class_probability": 0.9728599604926113,
            "per_digit_accuracy": {
              "0": 0.9887755102040816,
              "1": 0.9938325991189427,
              "2": 0.9786821705426356,
              "3": 0.9851485148514851,
              "4": 0.9725050916496945,
              "5": 0.9630044843049327,
              "6": 0.9822546972860126,
              "7": 0.9785992217898832,
              "8": 0.9702258726899384,
              "9": 0.977205153617443
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 17,
              "wrong_to_correct": 9,
              "correct_both": 9785,
              "wrong_both": 189
            }
          },
          "independent": {
            "accuracy": 0.8684,
            "correct": 8684,
            "total": 10000,
            "mean_true_class_probability": 0.832718941812345,
            "per_digit_accuracy": {
              "0": 0.939795918367347,
              "1": 0.9744493392070485,
              "2": 0.8391472868217055,
              "3": 0.8277227722772277,
              "4": 0.814663951120163,
              "5": 0.7993273542600897,
              "6": 0.9227557411273486,
              "7": 0.8764591439688716,
              "8": 0.8100616016427105,
              "9": 0.8602576808721506
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 99,
              "wrong_to_correct": 60,
              "correct_both": 8624,
              "wrong_both": 1217
            }
          },
          "misleading": {
            "accuracy": 0.8617,
            "correct": 8617,
            "total": 10000,
            "mean_true_class_probability": 0.8205804443599349,
            "per_digit_accuracy": {
              "0": 0.9336734693877551,
              "1": 0.9427312775330396,
              "2": 0.7790697674418605,
              "3": 0.8900990099009901,
              "4": 0.7596741344195519,
              "5": 0.7825112107623319,
              "6": 0.9331941544885177,
              "7": 0.892023346303502,
              "8": 0.8100616016427105,
              "9": 0.8771060455896927
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 93,
              "wrong_to_correct": 61,
              "correct_both": 8556,
              "wrong_both": 1290
            }
          }
        },
        {
          "matched": {
            "accuracy": 0.9796,
            "correct": 9796,
            "total": 10000,
            "mean_true_class_probability": 0.9720730411945053,
            "per_digit_accuracy": {
              "0": 0.9908163265306122,
              "1": 0.9911894273127754,
              "2": 0.9777131782945736,
              "3": 0.9871287128712871,
              "4": 0.9663951120162932,
              "5": 0.9742152466367713,
              "6": 0.9812108559498957,
              "7": 0.9747081712062257,
              "8": 0.9722792607802875,
              "9": 0.9781962338949455
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 23,
              "wrong_to_correct": 17,
              "correct_both": 9779,
              "wrong_both": 181
            }
          },
          "independent": {
            "accuracy": 0.8699,
            "correct": 8699,
            "total": 10000,
            "mean_true_class_probability": 0.8284716990747835,
            "per_digit_accuracy": {
              "0": 0.9377551020408164,
              "1": 0.9700440528634361,
              "2": 0.8352713178294574,
              "3": 0.8504950495049505,
              "4": 0.8014256619144603,
              "5": 0.820627802690583,
              "6": 0.9279749478079332,
              "7": 0.8608949416342413,
              "8": 0.8141683778234087,
              "9": 0.8642220019821606
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 116,
              "wrong_to_correct": 92,
              "correct_both": 8607,
              "wrong_both": 1185
            }
          },
          "misleading": {
            "accuracy": 0.8643,
            "correct": 8643,
            "total": 10000,
            "mean_true_class_probability": 0.8224065014800302,
            "per_digit_accuracy": {
              "0": 0.9346938775510204,
              "1": 0.947136563876652,
              "2": 0.7422480620155039,
              "3": 0.9237623762376238,
              "4": 0.7464358452138493,
              "5": 0.804932735426009,
              "6": 0.9394572025052192,
              "7": 0.892023346303502,
              "8": 0.8182751540041068,
              "9": 0.8800792864222002
            },
            "transitions_vs_baseline": {
              "correct_to_wrong": 97,
              "wrong_to_correct": 91,
              "correct_both": 8552,
              "wrong_both": 1260
            }
          }
        }
      ],
      "null_space_control_test": {
        "matched": {
          "accuracy": 0.1028,
          "correct": 1028,
          "total": 10000,
          "mean_true_class_probability": 0.09993723661676049,
          "per_digit_accuracy": {
            "0": 0.0,
            "1": 0.0,
            "2": 0.0,
            "3": 0.0,
            "4": 0.0,
            "5": 0.0,
            "6": 0.0,
            "7": 1.0,
            "8": 0.0,
            "9": 0.0
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 8797,
            "wrong_to_correct": 23,
            "correct_both": 1005,
            "wrong_both": 175
          }
        },
        "independent": {
          "accuracy": 0.1028,
          "correct": 1028,
          "total": 10000,
          "mean_true_class_probability": 0.0999372930586338,
          "per_digit_accuracy": {
            "0": 0.0,
            "1": 0.0,
            "2": 0.0,
            "3": 0.0,
            "4": 0.0,
            "5": 0.0,
            "6": 0.0,
            "7": 1.0,
            "8": 0.0,
            "9": 0.0
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 7831,
            "wrong_to_correct": 136,
            "correct_both": 892,
            "wrong_both": 1141
          }
        },
        "misleading": {
          "accuracy": 0.1028,
          "correct": 1028,
          "total": 10000,
          "mean_true_class_probability": 0.09993732610940934,
          "per_digit_accuracy": {
            "0": 0.0,
            "1": 0.0,
            "2": 0.0,
            "3": 0.0,
            "4": 0.0,
            "5": 0.0,
            "6": 0.0,
            "7": 1.0,
            "8": 0.0,
            "9": 0.0
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 7734,
            "wrong_to_correct": 113,
            "correct_both": 915,
            "wrong_both": 1238
          }
        }
      },
      "row_space_identity_test": {
        "matched": {
          "accuracy": 0.9802,
          "correct": 9802,
          "total": 10000,
          "mean_true_class_probability": 0.9730798736949801,
          "per_digit_accuracy": {
            "0": 0.9908163265306122,
            "1": 0.9938325991189427,
            "2": 0.9777131782945736,
            "3": 0.9881188118811881,
            "4": 0.9694501018329938,
            "5": 0.9719730941704036,
            "6": 0.9832985386221295,
            "7": 0.9776264591439688,
            "8": 0.9702258726899384,
            "9": 0.9762140733399405
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 0,
            "wrong_to_correct": 0,
            "correct_both": 9802,
            "wrong_both": 198
          }
        },
        "independent": {
          "accuracy": 0.8723,
          "correct": 8723,
          "total": 10000,
          "mean_true_class_probability": 0.836104607770556,
          "per_digit_accuracy": {
            "0": 0.9510204081632653,
            "1": 0.9770925110132158,
            "2": 0.8362403100775194,
            "3": 0.8514851485148515,
            "4": 0.8126272912423625,
            "5": 0.8183856502242153,
            "6": 0.9217118997912317,
            "7": 0.867704280155642,
            "8": 0.8049281314168378,
            "9": 0.8642220019821606
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 0,
            "wrong_to_correct": 0,
            "correct_both": 8723,
            "wrong_both": 1277
          }
        },
        "misleading": {
          "accuracy": 0.8649,
          "correct": 8649,
          "total": 10000,
          "mean_true_class_probability": 0.8283010846570492,
          "per_digit_accuracy": {
            "0": 0.936734693877551,
            "1": 0.9541850220264317,
            "2": 0.7587209302325582,
            "3": 0.9188118811881189,
            "4": 0.7647657841140529,
            "5": 0.7892376681614349,
            "6": 0.9405010438413361,
            "7": 0.8900778210116731,
            "8": 0.8100616016427105,
            "9": 0.869177403369673
          },
          "transitions_vs_baseline": {
            "correct_to_wrong": 0,
            "wrong_to_correct": 0,
            "correct_both": 8649,
            "wrong_both": 1351
          }
        }
      },
      "replacement": {
        "rank": 2,
        "test": {
          "matched": {
            "all_palette": {
              "accuracy": 0.95929,
              "correct": 95929,
              "total": 100000,
              "mean_true_class_probability": 0.9456781635327987,
              "per_digit_accuracy": {
                "0": 0.9874489795918368,
                "1": 0.9944493392070485,
                "2": 0.9061046511627907,
                "3": 0.9336633663366337,
                "4": 0.9460285132382892,
                "5": 0.930829596412556,
                "6": 0.9771398747390396,
                "7": 0.9747081712062257,
                "8": 0.9619096509240247,
                "9": 0.9753221010901884
              },
              "unique_images": 10000,
              "donors_per_image": 10,
              "changed_predictions": 3087,
              "transitions_vs_baseline": {
                "correct_to_wrong": 2468,
                "wrong_to_correct": 377,
                "correct_both": 95552,
                "wrong_both": 1603
              },
              "mean_donor_class_probability": 0.11010632236899179,
              "baseline_mean_donor_class_probability": 0.10000000048289764,
              "prediction_agreement_with_recoloring": 0.88117,
              "mean_abs_logit_difference_from_recoloring": 2.8057078511536697
            },
            "aligned_donor": {
              "accuracy": 0.9858,
              "correct": 9858,
              "total": 10000,
              "mean_true_class_probability": 0.9787189593497293,
              "per_digit_accuracy": {
                "0": 0.9959183673469387,
                "1": 0.9938325991189427,
                "2": 0.9854651162790697,
                "3": 0.995049504950495,
                "4": 0.984725050916497,
                "5": 0.984304932735426,
                "6": 0.9853862212943633,
                "7": 0.9815175097276264,
                "8": 0.971252566735113,
                "9": 0.979187314172448
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 91,
              "transitions_vs_baseline": {
                "correct_to_wrong": 10,
                "wrong_to_correct": 66,
                "correct_both": 9792,
                "wrong_both": 132
              },
              "mean_donor_class_probability": 0.9787189593497293,
              "baseline_mean_donor_class_probability": 0.9730798745489537,
              "prediction_agreement_with_recoloring": 0.9924,
              "mean_abs_logit_difference_from_recoloring": 0.26198883060216904
            },
            "shifted_donor": {
              "accuracy": 0.9204,
              "correct": 9204,
              "total": 10000,
              "mean_true_class_probability": 0.902469075740999,
              "per_digit_accuracy": {
                "0": 0.9806122448979592,
                "1": 0.9938325991189427,
                "2": 0.6540697674418605,
                "3": 0.906930693069307,
                "4": 0.9236252545824847,
                "5": 0.8721973094170403,
                "6": 0.9624217118997912,
                "7": 0.9669260700389105,
                "8": 0.9743326488706365,
                "9": 0.9653121902874133
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 658,
              "transitions_vs_baseline": {
                "correct_to_wrong": 612,
                "wrong_to_correct": 14,
                "correct_both": 9190,
                "wrong_both": 184
              },
              "mean_donor_class_probability": 0.03618167874863387,
              "baseline_mean_donor_class_probability": 0.00225632272740923,
              "prediction_agreement_with_recoloring": 0.8686,
              "mean_abs_logit_difference_from_recoloring": 3.1091516153714807
            },
            "complement_patch": {
              "all_palette": {
                "accuracy": 0.90941,
                "correct": 90941,
                "total": 100000,
                "mean_true_class_probability": 0.8820497708454202,
                "per_digit_accuracy": {
                  "0": 0.9627551020408164,
                  "1": 0.9656387665198238,
                  "2": 0.9114341085271318,
                  "3": 0.9552475247524752,
                  "4": 0.9195519348268839,
                  "5": 0.8942825112107623,
                  "6": 0.9532359081419625,
                  "7": 0.882295719844358,
                  "8": 0.774435318275154,
                  "9": 0.8662041625371655
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 8524,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 7607,
                  "wrong_to_correct": 528,
                  "correct_both": 90413,
                  "wrong_both": 1452
                },
                "mean_donor_class_probability": 0.13572644273409157,
                "baseline_mean_donor_class_probability": 0.10000000048289764,
                "prediction_agreement_with_recoloring": 0.9183,
                "mean_abs_logit_difference_from_recoloring": 1.106871101674406
              },
              "aligned_donor": {
                "accuracy": 0.9896,
                "correct": 9896,
                "total": 10000,
                "mean_true_class_probability": 0.9846828569806414,
                "per_digit_accuracy": {
                  "0": 0.9979591836734694,
                  "1": 0.9973568281938326,
                  "2": 0.9864341085271318,
                  "3": 0.9920792079207921,
                  "4": 0.9816700610997964,
                  "5": 0.984304932735426,
                  "6": 0.9864300626304802,
                  "7": 0.9902723735408561,
                  "8": 0.9845995893223819,
                  "9": 0.9930624380574826
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 127,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 11,
                  "wrong_to_correct": 105,
                  "correct_both": 9791,
                  "wrong_both": 93
                },
                "mean_donor_class_probability": 0.9846828569806414,
                "baseline_mean_donor_class_probability": 0.9730798745489537,
                "prediction_agreement_with_recoloring": 0.9963,
                "mean_abs_logit_difference_from_recoloring": 0.11069089128784836
              },
              "shifted_donor": {
                "accuracy": 0.9321,
                "correct": 9321,
                "total": 10000,
                "mean_true_class_probability": 0.9079402032344768,
                "per_digit_accuracy": {
                  "0": 0.9642857142857143,
                  "1": 0.9515418502202643,
                  "2": 0.9292635658914729,
                  "3": 0.9752475247524752,
                  "4": 0.9287169042769857,
                  "5": 0.9316143497757847,
                  "6": 0.9739039665970772,
                  "7": 0.9066147859922179,
                  "8": 0.8316221765913757,
                  "9": 0.9256689791873142
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 646,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 544,
                  "wrong_to_correct": 63,
                  "correct_both": 9258,
                  "wrong_both": 135
                },
                "mean_donor_class_probability": 0.021964217760903425,
                "baseline_mean_donor_class_probability": 0.00225632272740923,
                "prediction_agreement_with_recoloring": 0.9096,
                "mean_abs_logit_difference_from_recoloring": 1.5273712083994597
              }
            },
            "recolored_reference": {
              "all_palette": {
                "accuracy": 0.87204,
                "correct": 87204,
                "total": 100000,
                "mean_true_class_probability": 0.8356910797871492,
                "per_digit_accuracy": {
                  "0": 0.9419387755102041,
                  "1": 0.9706607929515418,
                  "2": 0.8499031007751938,
                  "3": 0.8574257425742574,
                  "4": 0.8205702647657841,
                  "5": 0.8154708520179372,
                  "6": 0.9225469728601252,
                  "7": 0.8747081712062257,
                  "8": 0.7897330595482546,
                  "9": 0.8593657086223984
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 12437,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 11423,
                  "wrong_to_correct": 607,
                  "correct_both": 86597,
                  "wrong_both": 1373
                },
                "mean_donor_class_probability": 0.17384744519948717,
                "baseline_mean_donor_class_probability": 0.10000000048289764,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "aligned_donor": {
                "accuracy": 0.9932,
                "correct": 9932,
                "total": 10000,
                "mean_true_class_probability": 0.9883561613966245,
                "per_digit_accuracy": {
                  "0": 0.9979591836734694,
                  "1": 0.9973568281938326,
                  "2": 0.9961240310077519,
                  "3": 0.997029702970297,
                  "4": 0.9887983706720977,
                  "5": 0.992152466367713,
                  "6": 0.988517745302714,
                  "7": 0.9912451361867705,
                  "8": 0.9887063655030801,
                  "9": 0.9930624380574826
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 132,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 0,
                  "wrong_to_correct": 130,
                  "correct_both": 9802,
                  "wrong_both": 68
                },
                "mean_donor_class_probability": 0.9883561613966245,
                "baseline_mean_donor_class_probability": 0.9730798745489537,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "shifted_donor": {
                "accuracy": 0.8631,
                "correct": 8631,
                "total": 10000,
                "mean_true_class_probability": 0.826564385091509,
                "per_digit_accuracy": {
                  "0": 0.939795918367347,
                  "1": 0.9515418502202643,
                  "2": 0.7412790697674418,
                  "3": 0.9227722772277228,
                  "4": 0.7556008146639511,
                  "5": 0.7903587443946188,
                  "6": 0.9394572025052192,
                  "7": 0.892023346303502,
                  "8": 0.8059548254620124,
                  "9": 0.8761149653121902
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1314,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 1219,
                  "wrong_to_correct": 48,
                  "correct_both": 8583,
                  "wrong_both": 150
                },
                "mean_donor_class_probability": 0.06242666428522647,
                "baseline_mean_donor_class_probability": 0.00225632272740923,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              }
            },
            "random_controls": [
              {
                "all_palette": {
                  "accuracy": 0.98003,
                  "correct": 98003,
                  "total": 100000,
                  "mean_true_class_probability": 0.9727634303820389,
                  "per_digit_accuracy": {
                    "0": 0.9918367346938776,
                    "1": 0.993920704845815,
                    "2": 0.9775193798449613,
                    "3": 0.9898019801980198,
                    "4": 0.969959266802444,
                    "5": 0.9714125560538116,
                    "6": 0.9838204592901879,
                    "7": 0.973443579766537,
                    "8": 0.9693018480492813,
                    "9": 0.9766105054509415
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 214,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 100,
                    "wrong_to_correct": 83,
                    "correct_both": 97920,
                    "wrong_both": 1897
                  },
                  "mean_donor_class_probability": 0.10017459111818101,
                  "baseline_mean_donor_class_probability": 0.10000000048289764,
                  "prediction_agreement_with_recoloring": 0.87613,
                  "mean_abs_logit_difference_from_recoloring": 3.011204524842185
                },
                "aligned_donor": {
                  "accuracy": 0.9807,
                  "correct": 9807,
                  "total": 10000,
                  "mean_true_class_probability": 0.9735998800847016,
                  "per_digit_accuracy": {
                    "0": 0.9908163265306122,
                    "1": 0.9947136563876652,
                    "2": 0.9777131782945736,
                    "3": 0.9900990099009901,
                    "4": 0.9704684317718941,
                    "5": 0.968609865470852,
                    "6": 0.9832985386221295,
                    "7": 0.9795719844357976,
                    "8": 0.9691991786447639,
                    "9": 0.979187314172448
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 15,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 4,
                    "wrong_to_correct": 9,
                    "correct_both": 9798,
                    "wrong_both": 189
                  },
                  "mean_donor_class_probability": 0.9735998800847016,
                  "baseline_mean_donor_class_probability": 0.9730798745489537,
                  "prediction_agreement_with_recoloring": 0.9873,
                  "mean_abs_logit_difference_from_recoloring": 0.2849644177480042
                },
                "shifted_donor": {
                  "accuracy": 0.9799,
                  "correct": 9799,
                  "total": 10000,
                  "mean_true_class_probability": 0.9723084628383545,
                  "per_digit_accuracy": {
                    "0": 0.9918367346938776,
                    "1": 0.9929515418502203,
                    "2": 0.9777131782945736,
                    "3": 0.9891089108910891,
                    "4": 0.9694501018329938,
                    "5": 0.9730941704035875,
                    "6": 0.9832985386221295,
                    "7": 0.9747081712062257,
                    "8": 0.9702258726899384,
                    "9": 0.9742319127849356
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 17,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 9,
                    "wrong_to_correct": 6,
                    "correct_both": 9793,
                    "wrong_both": 192
                  },
                  "mean_donor_class_probability": 0.00224761637945973,
                  "baseline_mean_donor_class_probability": 0.00225632272740923,
                  "prediction_agreement_with_recoloring": 0.8687,
                  "mean_abs_logit_difference_from_recoloring": 3.4325721067234873
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.98022,
                  "correct": 98022,
                  "total": 100000,
                  "mean_true_class_probability": 0.9728587718140675,
                  "per_digit_accuracy": {
                    "0": 0.9933673469387755,
                    "1": 0.9933039647577092,
                    "2": 0.977906976744186,
                    "3": 0.9878217821782178,
                    "4": 0.9709775967413442,
                    "5": 0.9700672645739911,
                    "6": 0.9831941544885178,
                    "7": 0.9768482490272373,
                    "8": 0.9685831622176592,
                    "9": 0.9773042616451932
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 235,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 94,
                    "wrong_to_correct": 96,
                    "correct_both": 97926,
                    "wrong_both": 1884
                  },
                  "mean_donor_class_probability": 0.09993589983850594,
                  "baseline_mean_donor_class_probability": 0.10000000048289764,
                  "prediction_agreement_with_recoloring": 0.87578,
                  "mean_abs_logit_difference_from_recoloring": 3.0797839453220686
                },
                "aligned_donor": {
                  "accuracy": 0.9803,
                  "correct": 9803,
                  "total": 10000,
                  "mean_true_class_probability": 0.9727828154513812,
                  "per_digit_accuracy": {
                    "0": 0.9938775510204082,
                    "1": 0.9929515418502203,
                    "2": 0.9777131782945736,
                    "3": 0.9881188118811881,
                    "4": 0.9684317718940937,
                    "5": 0.9708520179372198,
                    "6": 0.9832985386221295,
                    "7": 0.9785992217898832,
                    "8": 0.9702258726899384,
                    "9": 0.9762140733399405
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 17,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 6,
                    "wrong_to_correct": 7,
                    "correct_both": 9796,
                    "wrong_both": 191
                  },
                  "mean_donor_class_probability": 0.9727828154513812,
                  "baseline_mean_donor_class_probability": 0.9730798745489537,
                  "prediction_agreement_with_recoloring": 0.9869,
                  "mean_abs_logit_difference_from_recoloring": 0.2923831242608279
                },
                "shifted_donor": {
                  "accuracy": 0.9806,
                  "correct": 9806,
                  "total": 10000,
                  "mean_true_class_probability": 0.9734703691978932,
                  "per_digit_accuracy": {
                    "0": 0.9897959183673469,
                    "1": 0.9938325991189427,
                    "2": 0.9777131782945736,
                    "3": 0.9900990099009901,
                    "4": 0.9714867617107943,
                    "5": 0.9697309417040358,
                    "6": 0.9843423799582464,
                    "7": 0.9795719844357976,
                    "8": 0.9671457905544147,
                    "9": 0.979187314172448
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 18,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 6,
                    "wrong_to_correct": 10,
                    "correct_both": 9796,
                    "wrong_both": 188
                  },
                  "mean_donor_class_probability": 0.0021762179622020266,
                  "baseline_mean_donor_class_probability": 0.00225632272740923,
                  "prediction_agreement_with_recoloring": 0.8688,
                  "mean_abs_logit_difference_from_recoloring": 3.4875901129075513
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.97998,
                  "correct": 97998,
                  "total": 100000,
                  "mean_true_class_probability": 0.9728056956564661,
                  "per_digit_accuracy": {
                    "0": 0.993061224489796,
                    "1": 0.993568281938326,
                    "2": 0.9777131782945736,
                    "3": 0.9889108910891089,
                    "4": 0.9688391038696538,
                    "5": 0.9698430493273542,
                    "6": 0.9817327766179541,
                    "7": 0.9753891050583657,
                    "8": 0.9698151950718685,
                    "9": 0.977998017839445
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 358,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 169,
                    "wrong_to_correct": 147,
                    "correct_both": 97851,
                    "wrong_both": 1833
                  },
                  "mean_donor_class_probability": 0.1000492505988367,
                  "baseline_mean_donor_class_probability": 0.10000000048289764,
                  "prediction_agreement_with_recoloring": 0.87614,
                  "mean_abs_logit_difference_from_recoloring": 3.0411603362345323
                },
                "aligned_donor": {
                  "accuracy": 0.9804,
                  "correct": 9804,
                  "total": 10000,
                  "mean_true_class_probability": 0.9732578020262034,
                  "per_digit_accuracy": {
                    "0": 0.9938775510204082,
                    "1": 0.9938325991189427,
                    "2": 0.9767441860465116,
                    "3": 0.9910891089108911,
                    "4": 0.9725050916496945,
                    "5": 0.9652466367713004,
                    "6": 0.9832985386221295,
                    "7": 0.9766536964980544,
                    "8": 0.971252566735113,
                    "9": 0.9762140733399405
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 22,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 9,
                    "wrong_to_correct": 11,
                    "correct_both": 9793,
                    "wrong_both": 187
                  },
                  "mean_donor_class_probability": 0.9732578020262034,
                  "baseline_mean_donor_class_probability": 0.9730798745489537,
                  "prediction_agreement_with_recoloring": 0.987,
                  "mean_abs_logit_difference_from_recoloring": 0.2882900012880564
                },
                "shifted_donor": {
                  "accuracy": 0.9784,
                  "correct": 9784,
                  "total": 10000,
                  "mean_true_class_probability": 0.9709841358242416,
                  "per_digit_accuracy": {
                    "0": 0.9897959183673469,
                    "1": 0.9938325991189427,
                    "2": 0.9786821705426356,
                    "3": 0.9831683168316832,
                    "4": 0.9663951120162932,
                    "5": 0.9674887892376681,
                    "6": 0.9822546972860126,
                    "7": 0.9737354085603113,
                    "8": 0.9671457905544147,
                    "9": 0.9781962338949455
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 35,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 25,
                    "wrong_to_correct": 7,
                    "correct_both": 9777,
                    "wrong_both": 191
                  },
                  "mean_donor_class_probability": 0.002241995883107829,
                  "baseline_mean_donor_class_probability": 0.00225632272740923,
                  "prediction_agreement_with_recoloring": 0.8685,
                  "mean_abs_logit_difference_from_recoloring": 3.449321750989519
                }
              }
            ]
          },
          "independent": {
            "all_palette": {
              "accuracy": 0.85295,
              "correct": 85295,
              "total": 100000,
              "mean_true_class_probability": 0.8197947598199776,
              "per_digit_accuracy": {
                "0": 0.9393877551020409,
                "1": 0.9741850220264318,
                "2": 0.8268410852713178,
                "3": 0.8093069306930694,
                "4": 0.7920570264765784,
                "5": 0.7817264573991032,
                "6": 0.9153444676409186,
                "7": 0.866147859922179,
                "8": 0.781006160164271,
                "9": 0.822001982160555
              },
              "unique_images": 10000,
              "donors_per_image": 10,
              "changed_predictions": 9776,
              "transitions_vs_baseline": {
                "correct_to_wrong": 5095,
                "wrong_to_correct": 3160,
                "correct_both": 82135,
                "wrong_both": 9610
              },
              "mean_donor_class_probability": 0.12231207068368745,
              "baseline_mean_donor_class_probability": 0.10000000013915712,
              "prediction_agreement_with_recoloring": 0.85593,
              "mean_abs_logit_difference_from_recoloring": 2.9053001136115
            },
            "aligned_donor": {
              "accuracy": 0.9155,
              "correct": 9155,
              "total": 10000,
              "mean_true_class_probability": 0.88945705370094,
              "per_digit_accuracy": {
                "0": 0.9653061224489796,
                "1": 0.9726872246696036,
                "2": 0.9118217054263565,
                "3": 0.9643564356435643,
                "4": 0.9338085539714868,
                "5": 0.9002242152466368,
                "6": 0.9613778705636743,
                "7": 0.8842412451361867,
                "8": 0.7987679671457906,
                "9": 0.8543111992071358
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 815,
              "transitions_vs_baseline": {
                "correct_to_wrong": 133,
                "wrong_to_correct": 565,
                "correct_both": 8590,
                "wrong_both": 712
              },
              "mean_donor_class_probability": 0.88945705370094,
              "baseline_mean_donor_class_probability": 0.836104625771648,
              "prediction_agreement_with_recoloring": 0.92,
              "mean_abs_logit_difference_from_recoloring": 2.8075889364058524
            },
            "shifted_donor": {
              "accuracy": 0.788,
              "correct": 7880,
              "total": 10000,
              "mean_true_class_probability": 0.7523898075594164,
              "per_digit_accuracy": {
                "0": 0.9204081632653062,
                "1": 0.9718061674008811,
                "2": 0.6385658914728682,
                "3": 0.7207920792079208,
                "4": 0.6822810590631364,
                "5": 0.6771300448430493,
                "6": 0.8507306889352818,
                "7": 0.8472762645914397,
                "8": 0.813141683778234,
                "9": 0.7294350842418236
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 1260,
              "transitions_vs_baseline": {
                "correct_to_wrong": 971,
                "wrong_to_correct": 128,
                "correct_both": 7752,
                "wrong_both": 1149
              },
              "mean_donor_class_probability": 0.03780594636726352,
              "baseline_mean_donor_class_probability": 0.013839856134110871,
              "prediction_agreement_with_recoloring": 0.8308,
              "mean_abs_logit_difference_from_recoloring": 2.9078928081881257
            },
            "complement_patch": {
              "all_palette": {
                "accuracy": 0.85235,
                "correct": 85235,
                "total": 100000,
                "mean_true_class_probability": 0.8193069778160934,
                "per_digit_accuracy": {
                  "0": 0.9398979591836735,
                  "1": 0.9701321585903083,
                  "2": 0.8337209302325581,
                  "3": 0.808910891089109,
                  "4": 0.7841140529531568,
                  "5": 0.7927130044843049,
                  "6": 0.9110647181628393,
                  "7": 0.8657587548638133,
                  "8": 0.7674537987679672,
                  "9": 0.8290386521308226
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 14150,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 6842,
                  "wrong_to_correct": 4847,
                  "correct_both": 80388,
                  "wrong_both": 7923
                },
                "mean_donor_class_probability": 0.14788347560610898,
                "baseline_mean_donor_class_probability": 0.10000000013915712,
                "prediction_agreement_with_recoloring": 0.90144,
                "mean_abs_logit_difference_from_recoloring": 1.0869230688362774
              },
              "aligned_donor": {
                "accuracy": 0.9701,
                "correct": 9701,
                "total": 10000,
                "mean_true_class_probability": 0.9579472027499907,
                "per_digit_accuracy": {
                  "0": 0.9948979591836735,
                  "1": 0.9973568281938326,
                  "2": 0.9215116279069767,
                  "3": 0.9306930693069307,
                  "4": 0.955193482688391,
                  "5": 0.9540358744394619,
                  "6": 0.9822546972860126,
                  "7": 0.9854085603112841,
                  "8": 0.9845995893223819,
                  "9": 0.9920713577799801
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1152,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 61,
                  "wrong_to_correct": 1039,
                  "correct_both": 8662,
                  "wrong_both": 238
                },
                "mean_donor_class_probability": 0.9579472027499907,
                "baseline_mean_donor_class_probability": 0.836104625771648,
                "prediction_agreement_with_recoloring": 0.9755,
                "mean_abs_logit_difference_from_recoloring": 1.1156715643326565
              },
              "shifted_donor": {
                "accuracy": 0.9016,
                "correct": 9016,
                "total": 10000,
                "mean_true_class_probability": 0.8685386169477107,
                "per_digit_accuracy": {
                  "0": 0.9571428571428572,
                  "1": 0.960352422907489,
                  "2": 0.875968992248062,
                  "3": 0.9257425742574258,
                  "4": 0.8380855397148677,
                  "5": 0.8721973094170403,
                  "6": 0.9592901878914405,
                  "7": 0.9027237354085603,
                  "8": 0.795687885010267,
                  "9": 0.9177403369672944
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1045,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 273,
                  "wrong_to_correct": 566,
                  "correct_both": 8450,
                  "wrong_both": 711
                },
                "mean_donor_class_probability": 0.03675865725062451,
                "baseline_mean_donor_class_probability": 0.013839856134110871,
                "prediction_agreement_with_recoloring": 0.9312,
                "mean_abs_logit_difference_from_recoloring": 1.0840901874694135
              }
            },
            "recolored_reference": {
              "all_palette": {
                "accuracy": 0.87204,
                "correct": 87204,
                "total": 100000,
                "mean_true_class_probability": 0.8356910797871492,
                "per_digit_accuracy": {
                  "0": 0.9419387755102041,
                  "1": 0.9706607929515418,
                  "2": 0.8499031007751938,
                  "3": 0.8574257425742574,
                  "4": 0.8205702647657841,
                  "5": 0.8154708520179372,
                  "6": 0.9225469728601252,
                  "7": 0.8747081712062257,
                  "8": 0.7897330595482546,
                  "9": 0.8593657086223984
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 14278,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 5781,
                  "wrong_to_correct": 5755,
                  "correct_both": 81449,
                  "wrong_both": 7015
                },
                "mean_donor_class_probability": 0.17384744519948717,
                "baseline_mean_donor_class_probability": 0.10000000013915712,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "aligned_donor": {
                "accuracy": 0.9932,
                "correct": 9932,
                "total": 10000,
                "mean_true_class_probability": 0.9883561613966245,
                "per_digit_accuracy": {
                  "0": 0.9979591836734694,
                  "1": 0.9973568281938326,
                  "2": 0.9961240310077519,
                  "3": 0.997029702970297,
                  "4": 0.9887983706720977,
                  "5": 0.992152466367713,
                  "6": 0.988517745302714,
                  "7": 0.9912451361867705,
                  "8": 0.9887063655030801,
                  "9": 0.9930624380574826
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1223,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 1,
                  "wrong_to_correct": 1210,
                  "correct_both": 8722,
                  "wrong_both": 67
                },
                "mean_donor_class_probability": 0.9883561613966245,
                "baseline_mean_donor_class_probability": 0.836104625771648,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "shifted_donor": {
                "accuracy": 0.8631,
                "correct": 8631,
                "total": 10000,
                "mean_true_class_probability": 0.826564385091509,
                "per_digit_accuracy": {
                  "0": 0.939795918367347,
                  "1": 0.9515418502202643,
                  "2": 0.7412790697674418,
                  "3": 0.9227722772277228,
                  "4": 0.7556008146639511,
                  "5": 0.7903587443946188,
                  "6": 0.9394572025052192,
                  "7": 0.892023346303502,
                  "8": 0.8059548254620124,
                  "9": 0.8761149653121902
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1326,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 556,
                  "wrong_to_correct": 464,
                  "correct_both": 8167,
                  "wrong_both": 813
                },
                "mean_donor_class_probability": 0.06242666428522647,
                "baseline_mean_donor_class_probability": 0.013839856134110871,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              }
            },
            "random_controls": [
              {
                "all_palette": {
                  "accuracy": 0.87336,
                  "correct": 87336,
                  "total": 100000,
                  "mean_true_class_probability": 0.837491837779886,
                  "per_digit_accuracy": {
                    "0": 0.9503061224489796,
                    "1": 0.9771806167400882,
                    "2": 0.8393410852713178,
                    "3": 0.8515841584158416,
                    "4": 0.8165987780040733,
                    "5": 0.816591928251121,
                    "6": 0.9230688935281837,
                    "7": 0.8707198443579767,
                    "8": 0.8057494866529774,
                    "9": 0.8646184340931615
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1195,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 448,
                    "wrong_to_correct": 554,
                    "correct_both": 86782,
                    "wrong_both": 12216
                  },
                  "mean_donor_class_probability": 0.10103072813987511,
                  "baseline_mean_donor_class_probability": 0.10000000013915712,
                  "prediction_agreement_with_recoloring": 0.85901,
                  "mean_abs_logit_difference_from_recoloring": 3.070870986074929
                },
                "aligned_donor": {
                  "accuracy": 0.8781,
                  "correct": 8781,
                  "total": 10000,
                  "mean_true_class_probability": 0.8423709241408891,
                  "per_digit_accuracy": {
                    "0": 0.9489795918367347,
                    "1": 0.9797356828193833,
                    "2": 0.8459302325581395,
                    "3": 0.8504950495049505,
                    "4": 0.8177189409368636,
                    "5": 0.8060538116591929,
                    "6": 0.9196242171189979,
                    "7": 0.892023346303502,
                    "8": 0.8100616016427105,
                    "9": 0.889990089197225
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 138,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 30,
                    "wrong_to_correct": 88,
                    "correct_both": 8693,
                    "wrong_both": 1189
                  },
                  "mean_donor_class_probability": 0.8423709241408891,
                  "baseline_mean_donor_class_probability": 0.836104625771648,
                  "prediction_agreement_with_recoloring": 0.8834,
                  "mean_abs_logit_difference_from_recoloring": 3.0173636666749233
                },
                "shifted_donor": {
                  "accuracy": 0.8696,
                  "correct": 8696,
                  "total": 10000,
                  "mean_true_class_probability": 0.8334633783767192,
                  "per_digit_accuracy": {
                    "0": 0.9479591836734694,
                    "1": 0.9762114537444934,
                    "2": 0.8381782945736435,
                    "3": 0.8504950495049505,
                    "4": 0.814663951120163,
                    "5": 0.820627802690583,
                    "6": 0.9196242171189979,
                    "7": 0.8764591439688716,
                    "8": 0.8018480492813141,
                    "9": 0.8325074331020813
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 128,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 67,
                    "wrong_to_correct": 40,
                    "correct_both": 8656,
                    "wrong_both": 1237
                  },
                  "mean_donor_class_probability": 0.01408520291552637,
                  "baseline_mean_donor_class_probability": 0.013839856134110871,
                  "prediction_agreement_with_recoloring": 0.8674,
                  "mean_abs_logit_difference_from_recoloring": 3.0743527295846493
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.87022,
                  "correct": 87022,
                  "total": 100000,
                  "mean_true_class_probability": 0.8345474172809638,
                  "per_digit_accuracy": {
                    "0": 0.9492857142857143,
                    "1": 0.9746255506607929,
                    "2": 0.8310077519379845,
                    "3": 0.8517821782178218,
                    "4": 0.8120162932790224,
                    "5": 0.817713004484305,
                    "6": 0.9205636743215031,
                    "7": 0.8650778210116732,
                    "8": 0.8015400410677618,
                    "9": 0.8613478691774034
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1353,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 656,
                    "wrong_to_correct": 448,
                    "correct_both": 86574,
                    "wrong_both": 12322
                  },
                  "mean_donor_class_probability": 0.099562821227057,
                  "baseline_mean_donor_class_probability": 0.10000000013915712,
                  "prediction_agreement_with_recoloring": 0.85589,
                  "mean_abs_logit_difference_from_recoloring": 3.138968362770156
                },
                "aligned_donor": {
                  "accuracy": 0.8695,
                  "correct": 8695,
                  "total": 10000,
                  "mean_true_class_probability": 0.8336908784009746,
                  "per_digit_accuracy": {
                    "0": 0.9479591836734694,
                    "1": 0.9718061674008811,
                    "2": 0.8236434108527132,
                    "3": 0.8574257425742574,
                    "4": 0.8044806517311609,
                    "5": 0.8228699551569507,
                    "6": 0.9196242171189979,
                    "7": 0.867704280155642,
                    "8": 0.8028747433264887,
                    "9": 0.8602576808721506
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 152,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 75,
                    "wrong_to_correct": 47,
                    "correct_both": 8648,
                    "wrong_both": 1230
                  },
                  "mean_donor_class_probability": 0.8336908784009746,
                  "baseline_mean_donor_class_probability": 0.836104625771648,
                  "prediction_agreement_with_recoloring": 0.8749,
                  "mean_abs_logit_difference_from_recoloring": 3.0844136237223445
                },
                "shifted_donor": {
                  "accuracy": 0.8732,
                  "correct": 8732,
                  "total": 10000,
                  "mean_true_class_probability": 0.8387644203031647,
                  "per_digit_accuracy": {
                    "0": 0.9418367346938775,
                    "1": 0.9788546255506608,
                    "2": 0.8410852713178295,
                    "3": 0.8623762376237624,
                    "4": 0.814663951120163,
                    "5": 0.8139013452914798,
                    "6": 0.9227557411273486,
                    "7": 0.8803501945525292,
                    "8": 0.7987679671457906,
                    "9": 0.8582755203171457
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 134,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 53,
                    "wrong_to_correct": 62,
                    "correct_both": 8670,
                    "wrong_both": 1215
                  },
                  "mean_donor_class_probability": 0.013479349638770571,
                  "baseline_mean_donor_class_probability": 0.013839856134110871,
                  "prediction_agreement_with_recoloring": 0.8694,
                  "mean_abs_logit_difference_from_recoloring": 3.1317132639641687
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.87239,
                  "correct": 87239,
                  "total": 100000,
                  "mean_true_class_probability": 0.8370537010251912,
                  "per_digit_accuracy": {
                    "0": 0.9491836734693877,
                    "1": 0.9768281938325991,
                    "2": 0.8358527131782946,
                    "3": 0.8502970297029703,
                    "4": 0.8243380855397149,
                    "5": 0.8137892376681615,
                    "6": 0.9227557411273486,
                    "7": 0.8690661478599222,
                    "8": 0.7993839835728953,
                    "9": 0.864420218037661
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1881,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 811,
                    "wrong_to_correct": 820,
                    "correct_both": 86419,
                    "wrong_both": 11950
                  },
                  "mean_donor_class_probability": 0.10022949861104323,
                  "baseline_mean_donor_class_probability": 0.10000000013915712,
                  "prediction_agreement_with_recoloring": 0.85846,
                  "mean_abs_logit_difference_from_recoloring": 3.092978729264783
                },
                "aligned_donor": {
                  "accuracy": 0.8745,
                  "correct": 8745,
                  "total": 10000,
                  "mean_true_class_probability": 0.8397555450330383,
                  "per_digit_accuracy": {
                    "0": 0.9510204081632653,
                    "1": 0.9788546255506608,
                    "2": 0.8343023255813954,
                    "3": 0.8613861386138614,
                    "4": 0.8513238289205702,
                    "5": 0.7937219730941704,
                    "6": 0.9248434237995825,
                    "7": 0.8754863813229572,
                    "8": 0.8008213552361396,
                    "9": 0.8533201189296333
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 214,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 82,
                    "wrong_to_correct": 104,
                    "correct_both": 8641,
                    "wrong_both": 1173
                  },
                  "mean_donor_class_probability": 0.8397555450330383,
                  "baseline_mean_donor_class_probability": 0.836104625771648,
                  "prediction_agreement_with_recoloring": 0.8797,
                  "mean_abs_logit_difference_from_recoloring": 3.044132894794196
                },
                "shifted_donor": {
                  "accuracy": 0.8628,
                  "correct": 8628,
                  "total": 10000,
                  "mean_true_class_probability": 0.827849832971413,
                  "per_digit_accuracy": {
                    "0": 0.9336734693877551,
                    "1": 0.9753303964757709,
                    "2": 0.8362403100775194,
                    "3": 0.8247524752475247,
                    "4": 0.8014256619144603,
                    "5": 0.8105381165919282,
                    "6": 0.9227557411273486,
                    "7": 0.857976653696498,
                    "8": 0.7864476386036962,
                    "9": 0.8602576808721506
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 192,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 131,
                    "wrong_to_correct": 36,
                    "correct_both": 8592,
                    "wrong_both": 1241
                  },
                  "mean_donor_class_probability": 0.013871256233182945,
                  "baseline_mean_donor_class_probability": 0.013839856134110871,
                  "prediction_agreement_with_recoloring": 0.8649,
                  "mean_abs_logit_difference_from_recoloring": 3.0908443566953765
                }
              }
            ]
          },
          "misleading": {
            "all_palette": {
              "accuracy": 0.89614,
              "correct": 89614,
              "total": 100000,
              "mean_true_class_probability": 0.8652067789354592,
              "per_digit_accuracy": {
                "0": 0.9431632653061225,
                "1": 0.9581497797356828,
                "2": 0.8701550387596899,
                "3": 0.9233663366336634,
                "4": 0.8377800407331976,
                "5": 0.8658071748878924,
                "6": 0.9578288100208768,
                "7": 0.8929961089494164,
                "8": 0.8001026694045175,
                "9": 0.9009910802775025
              },
              "unique_images": 10000,
              "donors_per_image": 10,
              "changed_predictions": 7052,
              "transitions_vs_baseline": {
                "correct_to_wrong": 1267,
                "wrong_to_correct": 4391,
                "correct_both": 85223,
                "wrong_both": 9119
              },
              "mean_donor_class_probability": 0.11529157566646026,
              "baseline_mean_donor_class_probability": 0.10000000019150596,
              "prediction_agreement_with_recoloring": 0.88883,
              "mean_abs_logit_difference_from_recoloring": 2.893297533525763
            },
            "aligned_donor": {
              "accuracy": 0.933,
              "correct": 9330,
              "total": 10000,
              "mean_true_class_probability": 0.9097346003113955,
              "per_digit_accuracy": {
                "0": 0.9642857142857143,
                "1": 0.9541850220264317,
                "2": 0.9341085271317829,
                "3": 0.9762376237623762,
                "4": 0.9378818737270875,
                "5": 0.9394618834080718,
                "6": 0.9707724425887265,
                "7": 0.9036964980544747,
                "8": 0.8316221765913757,
                "9": 0.9157581764122894
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 915,
              "transitions_vs_baseline": {
                "correct_to_wrong": 52,
                "wrong_to_correct": 733,
                "correct_both": 8597,
                "wrong_both": 618
              },
              "mean_donor_class_probability": 0.9097346003113955,
              "baseline_mean_donor_class_probability": 0.8283011444443807,
              "prediction_agreement_with_recoloring": 0.9369,
              "mean_abs_logit_difference_from_recoloring": 3.087145840691626
            },
            "shifted_donor": {
              "accuracy": 0.8561,
              "correct": 8561,
              "total": 10000,
              "mean_true_class_probability": 0.819571243591119,
              "per_digit_accuracy": {
                "0": 0.9295918367346939,
                "1": 0.9550660792951542,
                "2": 0.7344961240310077,
                "3": 0.901980198019802,
                "4": 0.7535641547861507,
                "5": 0.7802690582959642,
                "6": 0.9352818371607515,
                "7": 0.8842412451361867,
                "8": 0.8100616016427105,
                "9": 0.8592666005946482
              },
              "unique_images": 10000,
              "donors_per_image": 1,
              "changed_predictions": 132,
              "transitions_vs_baseline": {
                "correct_to_wrong": 101,
                "wrong_to_correct": 13,
                "correct_both": 8548,
                "wrong_both": 1338
              },
              "mean_donor_class_probability": 0.05856092831661874,
              "baseline_mean_donor_class_probability": 0.056089306648746666,
              "prediction_agreement_with_recoloring": 0.9799,
              "mean_abs_logit_difference_from_recoloring": 0.3143055137643218
            },
            "complement_patch": {
              "all_palette": {
                "accuracy": 0.79614,
                "correct": 79614,
                "total": 100000,
                "mean_true_class_probability": 0.759948084065546,
                "per_digit_accuracy": {
                  "0": 0.919795918367347,
                  "1": 0.9659030837004405,
                  "2": 0.6635658914728683,
                  "3": 0.7534653465346535,
                  "4": 0.6906313645621182,
                  "5": 0.6855381165919282,
                  "6": 0.8564718162839249,
                  "7": 0.856420233463035,
                  "8": 0.8046201232032855,
                  "9": 0.7369672943508424
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 16943,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 10438,
                  "wrong_to_correct": 3562,
                  "correct_both": 76052,
                  "wrong_both": 9948
                },
                "mean_donor_class_probability": 0.15682718823928096,
                "baseline_mean_donor_class_probability": 0.10000000019150596,
                "prediction_agreement_with_recoloring": 0.8754,
                "mean_abs_logit_difference_from_recoloring": 1.090516323561797
              },
              "aligned_donor": {
                "accuracy": 0.937,
                "correct": 9370,
                "total": 10000,
                "mean_true_class_probability": 0.9218634066291576,
                "per_digit_accuracy": {
                  "0": 0.9897959183673469,
                  "1": 0.9964757709251101,
                  "2": 0.6802325581395349,
                  "3": 0.9297029702970298,
                  "4": 0.9429735234215886,
                  "5": 0.897982062780269,
                  "6": 0.9749478079331941,
                  "7": 0.980544747081712,
                  "8": 0.9856262833675564,
                  "9": 0.9900891972249752
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1276,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 241,
                  "wrong_to_correct": 962,
                  "correct_both": 8408,
                  "wrong_both": 389
                },
                "mean_donor_class_probability": 0.9218634066291576,
                "baseline_mean_donor_class_probability": 0.8283011444443807,
                "prediction_agreement_with_recoloring": 0.9419,
                "mean_abs_logit_difference_from_recoloring": 1.5303135102203675
              },
              "shifted_donor": {
                "accuracy": 0.8673,
                "correct": 8673,
                "total": 10000,
                "mean_true_class_probability": 0.8312907984231959,
                "per_digit_accuracy": {
                  "0": 0.939795918367347,
                  "1": 0.9533039647577093,
                  "2": 0.7596899224806202,
                  "3": 0.9247524752475248,
                  "4": 0.7678207739307535,
                  "5": 0.7937219730941704,
                  "6": 0.941544885177453,
                  "7": 0.8929961089494164,
                  "8": 0.8039014373716632,
                  "9": 0.8790882061446977
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 139,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 44,
                  "wrong_to_correct": 68,
                  "correct_both": 8605,
                  "wrong_both": 1283
                },
                "mean_donor_class_probability": 0.05916256944700776,
                "baseline_mean_donor_class_probability": 0.056089306648746666,
                "prediction_agreement_with_recoloring": 0.9929,
                "mean_abs_logit_difference_from_recoloring": 0.11540003855232149
              }
            },
            "recolored_reference": {
              "all_palette": {
                "accuracy": 0.87204,
                "correct": 87204,
                "total": 100000,
                "mean_true_class_probability": 0.8356910797871492,
                "per_digit_accuracy": {
                  "0": 0.9419387755102041,
                  "1": 0.9706607929515418,
                  "2": 0.8499031007751938,
                  "3": 0.8574257425742574,
                  "4": 0.8205702647657841,
                  "5": 0.8154708520179372,
                  "6": 0.9225469728601252,
                  "7": 0.8747081712062257,
                  "8": 0.7897330595482546,
                  "9": 0.8593657086223984
                },
                "unique_images": 10000,
                "donors_per_image": 10,
                "changed_predictions": 13639,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 4946,
                  "wrong_to_correct": 5660,
                  "correct_both": 81544,
                  "wrong_both": 7850
                },
                "mean_donor_class_probability": 0.17384744519948717,
                "baseline_mean_donor_class_probability": 0.10000000019150596,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "aligned_donor": {
                "accuracy": 0.9932,
                "correct": 9932,
                "total": 10000,
                "mean_true_class_probability": 0.9883561613966245,
                "per_digit_accuracy": {
                  "0": 0.9979591836734694,
                  "1": 0.9973568281938326,
                  "2": 0.9961240310077519,
                  "3": 0.997029702970297,
                  "4": 0.9887983706720977,
                  "5": 0.992152466367713,
                  "6": 0.988517745302714,
                  "7": 0.9912451361867705,
                  "8": 0.9887063655030801,
                  "9": 0.9930624380574826
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 1303,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 2,
                  "wrong_to_correct": 1285,
                  "correct_both": 8647,
                  "wrong_both": 66
                },
                "mean_donor_class_probability": 0.9883561613966245,
                "baseline_mean_donor_class_probability": 0.8283011444443807,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              },
              "shifted_donor": {
                "accuracy": 0.8631,
                "correct": 8631,
                "total": 10000,
                "mean_true_class_probability": 0.826564385091509,
                "per_digit_accuracy": {
                  "0": 0.939795918367347,
                  "1": 0.9515418502202643,
                  "2": 0.7412790697674418,
                  "3": 0.9227722772277228,
                  "4": 0.7556008146639511,
                  "5": 0.7903587443946188,
                  "6": 0.9394572025052192,
                  "7": 0.892023346303502,
                  "8": 0.8059548254620124,
                  "9": 0.8761149653121902
                },
                "unique_images": 10000,
                "donors_per_image": 1,
                "changed_predictions": 163,
                "transitions_vs_baseline": {
                  "correct_to_wrong": 72,
                  "wrong_to_correct": 54,
                  "correct_both": 8577,
                  "wrong_both": 1297
                },
                "mean_donor_class_probability": 0.06242666428522647,
                "baseline_mean_donor_class_probability": 0.056089306648746666,
                "prediction_agreement_with_recoloring": 1.0,
                "mean_abs_logit_difference_from_recoloring": 0.0
              }
            },
            "random_controls": [
              {
                "all_palette": {
                  "accuracy": 0.86831,
                  "correct": 86831,
                  "total": 100000,
                  "mean_true_class_probability": 0.8326956674490831,
                  "per_digit_accuracy": {
                    "0": 0.9416326530612245,
                    "1": 0.9604405286343612,
                    "2": 0.7609496124031008,
                    "3": 0.9198019801980198,
                    "4": 0.7662932790224033,
                    "5": 0.7838565022421524,
                    "6": 0.94258872651357,
                    "7": 0.8888132295719844,
                    "8": 0.821047227926078,
                    "9": 0.8798810703666997
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1094,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 269,
                    "wrong_to_correct": 610,
                    "correct_both": 86221,
                    "wrong_both": 12900
                  },
                  "mean_donor_class_probability": 0.10083742016236867,
                  "baseline_mean_donor_class_probability": 0.10000000019150596,
                  "prediction_agreement_with_recoloring": 0.86652,
                  "mean_abs_logit_difference_from_recoloring": 3.0607308727592977
                },
                "aligned_donor": {
                  "accuracy": 0.8704,
                  "correct": 8704,
                  "total": 10000,
                  "mean_true_class_probability": 0.836131744908127,
                  "per_digit_accuracy": {
                    "0": 0.9316326530612244,
                    "1": 0.9656387665198238,
                    "2": 0.7684108527131783,
                    "3": 0.9198019801980198,
                    "4": 0.7637474541751528,
                    "5": 0.7701793721973094,
                    "6": 0.941544885177453,
                    "7": 0.9075875486381323,
                    "8": 0.8223819301848049,
                    "9": 0.8919722497522299
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 142,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 31,
                    "wrong_to_correct": 86,
                    "correct_both": 8618,
                    "wrong_both": 1265
                  },
                  "mean_donor_class_probability": 0.836131744908127,
                  "baseline_mean_donor_class_probability": 0.8283011444443807,
                  "prediction_agreement_with_recoloring": 0.8752,
                  "mean_abs_logit_difference_from_recoloring": 3.413080298335869
                },
                "shifted_donor": {
                  "accuracy": 0.8641,
                  "correct": 8641,
                  "total": 10000,
                  "mean_true_class_probability": 0.8278093571314735,
                  "per_digit_accuracy": {
                    "0": 0.936734693877551,
                    "1": 0.9541850220264317,
                    "2": 0.7567829457364341,
                    "3": 0.9188118811881189,
                    "4": 0.7637474541751528,
                    "5": 0.7892376681614349,
                    "6": 0.9405010438413361,
                    "7": 0.8861867704280155,
                    "8": 0.8090349075975359,
                    "9": 0.869177403369673
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 14,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 9,
                    "wrong_to_correct": 1,
                    "correct_both": 8640,
                    "wrong_both": 1350
                  },
                  "mean_donor_class_probability": 0.05613328260977223,
                  "baseline_mean_donor_class_probability": 0.056089306648746666,
                  "prediction_agreement_with_recoloring": 0.9837,
                  "mean_abs_logit_difference_from_recoloring": 0.33217965402334926
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.86104,
                  "correct": 86104,
                  "total": 100000,
                  "mean_true_class_probability": 0.8214221966860841,
                  "per_digit_accuracy": {
                    "0": 0.9422448979591836,
                    "1": 0.9432599118942732,
                    "2": 0.7523255813953489,
                    "3": 0.9078217821782179,
                    "4": 0.7596741344195519,
                    "5": 0.7949551569506726,
                    "6": 0.9376826722338205,
                    "7": 0.8768482490272373,
                    "8": 0.8104722792607802,
                    "9": 0.8710604558969276
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1328,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 728,
                    "wrong_to_correct": 342,
                    "correct_both": 85762,
                    "wrong_both": 13168
                  },
                  "mean_donor_class_probability": 0.09905152889045843,
                  "baseline_mean_donor_class_probability": 0.10000000019150596,
                  "prediction_agreement_with_recoloring": 0.86095,
                  "mean_abs_logit_difference_from_recoloring": 3.118879263991204
                },
                "aligned_donor": {
                  "accuracy": 0.8593,
                  "correct": 8593,
                  "total": 10000,
                  "mean_true_class_probability": 0.818083525490622,
                  "per_digit_accuracy": {
                    "0": 0.9387755102040817,
                    "1": 0.9303964757709251,
                    "2": 0.752906976744186,
                    "3": 0.9118811881188119,
                    "4": 0.7535641547861507,
                    "5": 0.797085201793722,
                    "6": 0.9394572025052192,
                    "7": 0.8803501945525292,
                    "8": 0.8080082135523614,
                    "9": 0.8681863230921705
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 160,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 91,
                    "wrong_to_correct": 35,
                    "correct_both": 8558,
                    "wrong_both": 1316
                  },
                  "mean_donor_class_probability": 0.818083525490622,
                  "baseline_mean_donor_class_probability": 0.8283011444443807,
                  "prediction_agreement_with_recoloring": 0.8644,
                  "mean_abs_logit_difference_from_recoloring": 3.468660366118802
                },
                "shifted_donor": {
                  "accuracy": 0.8652,
                  "correct": 8652,
                  "total": 10000,
                  "mean_true_class_probability": 0.8284772146595138,
                  "per_digit_accuracy": {
                    "0": 0.9357142857142857,
                    "1": 0.9550660792951542,
                    "2": 0.7587209302325582,
                    "3": 0.9188118811881189,
                    "4": 0.7647657841140529,
                    "5": 0.7881165919282511,
                    "6": 0.9394572025052192,
                    "7": 0.8929961089494164,
                    "8": 0.8100616016427105,
                    "9": 0.8711595639246779
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 11,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 3,
                    "wrong_to_correct": 6,
                    "correct_both": 8646,
                    "wrong_both": 1345
                  },
                  "mean_donor_class_probability": 0.056047136734200094,
                  "baseline_mean_donor_class_probability": 0.056089306648746666,
                  "prediction_agreement_with_recoloring": 0.9837,
                  "mean_abs_logit_difference_from_recoloring": 0.33840232657551766
                }
              },
              {
                "all_palette": {
                  "accuracy": 0.86872,
                  "correct": 86872,
                  "total": 100000,
                  "mean_true_class_probability": 0.8337938299869775,
                  "per_digit_accuracy": {
                    "0": 0.9413265306122449,
                    "1": 0.9539207048458149,
                    "2": 0.7545542635658915,
                    "3": 0.9225742574257426,
                    "4": 0.7808553971486761,
                    "5": 0.7933856502242153,
                    "6": 0.9407098121085595,
                    "7": 0.8974708171206226,
                    "8": 0.8116016427104723,
                    "9": 0.8748265609514371
                  },
                  "unique_images": 10000,
                  "donors_per_image": 10,
                  "changed_predictions": 1668,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 492,
                    "wrong_to_correct": 874,
                    "correct_both": 85998,
                    "wrong_both": 12636
                  },
                  "mean_donor_class_probability": 0.09992792047044426,
                  "baseline_mean_donor_class_probability": 0.10000000019150596,
                  "prediction_agreement_with_recoloring": 0.86506,
                  "mean_abs_logit_difference_from_recoloring": 3.078017849453168
                },
                "aligned_donor": {
                  "accuracy": 0.8668,
                  "correct": 8668,
                  "total": 10000,
                  "mean_true_class_probability": 0.8333588704466421,
                  "per_digit_accuracy": {
                    "0": 0.9346938775510204,
                    "1": 0.9559471365638766,
                    "2": 0.7403100775193798,
                    "3": 0.9257425742574258,
                    "4": 0.8085539714867617,
                    "5": 0.7645739910313901,
                    "6": 0.94258872651357,
                    "7": 0.9046692607003891,
                    "8": 0.811088295687885,
                    "9": 0.8612487611496531
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 230,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 86,
                    "wrong_to_correct": 105,
                    "correct_both": 8563,
                    "wrong_both": 1246
                  },
                  "mean_donor_class_probability": 0.8333588704466421,
                  "baseline_mean_donor_class_probability": 0.8283011444443807,
                  "prediction_agreement_with_recoloring": 0.8716,
                  "mean_abs_logit_difference_from_recoloring": 3.430556076871436
                },
                "shifted_donor": {
                  "accuracy": 0.8637,
                  "correct": 8637,
                  "total": 10000,
                  "mean_true_class_probability": 0.8274151162598328,
                  "per_digit_accuracy": {
                    "0": 0.936734693877551,
                    "1": 0.9533039647577093,
                    "2": 0.7577519379844961,
                    "3": 0.9158415841584159,
                    "4": 0.7617107942973523,
                    "5": 0.7869955156950673,
                    "6": 0.9405010438413361,
                    "7": 0.8881322957198443,
                    "8": 0.811088295687885,
                    "9": 0.8681863230921705
                  },
                  "unique_images": 10000,
                  "donors_per_image": 1,
                  "changed_predictions": 20,
                  "transitions_vs_baseline": {
                    "correct_to_wrong": 14,
                    "wrong_to_correct": 2,
                    "correct_both": 8635,
                    "wrong_both": 1349
                  },
                  "mean_donor_class_probability": 0.056056723248459346,
                  "baseline_mean_donor_class_probability": 0.056089306648746666,
                  "prediction_agreement_with_recoloring": 0.983,
                  "mean_abs_logit_difference_from_recoloring": 0.3338189757671207
                }
              }
            ]
          }
        },
        "checks": {
          "self_patch_logits_max_abs_diff": 0.0,
          "selected_coordinates_max_abs_diff": 3.814697265625e-05,
          "complementary_logit_effects_max_abs_diff": 3.814697265625e-05,
          "perpendicular_component_max_abs_diff": 7.62939453125e-06,
          "full_patch_vs_donor_logits_max_abs_diff": 3.814697265625e-06
        }
      },
      "checks": {
        "learned_tensors_identical": true,
        "identity_at_zero_strength_matches_baseline": true,
        "U_orthonormal": true,
        "UUT_idempotent": true,
        "wrapped_vs_direct_logits_max_abs_diff": 7.62939453125e-06,
        "null_projection_logits_minus_bias_max_abs": 2.2681429982185364e-05,
        "row_projection_logits_max_abs_diff": 3.24249267578125e-05,
        "head_rank": 10,
        "row_space_dim": 10,
        "null_space_dim": 118
      }
    }
  ],
  "elapsed_seconds": 44.50861348910257
}