#!/usr/bin/env python3
"""
build_data.py  --  Clean & reshape the hand-coded Catholic clergy scandal data
====================================================================================
Source : Scandals_v0.txt  (tab-delimited, one row per accused clergy member)
Output : event-level dataset of *scandals* as defined in
         Bottan & Perez-Truglia (2015, J. Public Economics).

Definitions (paper, Section 2.1):
  * type-A scandal (RED):  a clergy member CURRENTLY working in a Catholic
                           institution is accused for the first time. Location =
                           institution where the clergy works at time of accusation.
                           -> comes from the "scandal{1,2}" column blocks.
  * type-B scandal (BLUE): a clergy member is accused of abuse committed while
                           working in an institution in the PAST. Location =
                           institution where the abuse allegedly occurred.
                           -> comes from the "abuse{1..10}" column blocks.

The event date is the date the accusation became public (date_scandal* /
date_scandal_abuse*), i.e. the "scandal", not the abuse itself.
"""
import csv, json, re, sys
from pathlib import Path

SRC = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("Scandals_v0.txt")
OUT = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("out")
OUT.mkdir(parents=True, exist_ok=True)

# ----------------------------------------------------------------------------- read
csv.field_size_limit(10_000_000)
with open(SRC, encoding="utf-8", errors="replace", newline="") as f:
    reader = csv.reader(f, delimiter="\t", quotechar='"')
    rows = list(reader)
header = rows[0]
data = rows[1:]
print(f"raw rows parsed: {len(data)} (each should map to one clergy member)")

def clean(s):
    if s is None:
        return ""
    s = s.replace("\r", " ").replace("\n", " ").replace('"', " ")
    s = re.sub(r"\s+", " ", s).strip()
    return s

def num(s):
    s = clean(s)
    if s in ("", "0", "0.0"):
        return None
    try:
        return float(s)
    except ValueError:
        return None

DATE_RE = re.compile(r"^\s*(\d{1,2})/(\d{1,2})/(\d{4})\s*$")
def parse_date(s):
    """Return (iso, year, month, day) or (None,None,None,None)."""
    s = clean(s)
    m = DATE_RE.match(s)
    if m:
        mm, dd, yy = int(m.group(1)), int(m.group(2)), int(m.group(3))
        if 1 <= mm <= 12 and 1 <= dd <= 31 and 1950 <= yy <= 2010:
            return (f"{yy:04d}-{mm:02d}-{dd:02d}", yy, mm, dd)
    m2 = re.match(r"^\s*(\d{4})\s*$", s)
    if m2:
        yy = int(m2.group(1))
        if 1950 <= yy <= 2010:
            return (f"{yy:04d}-07-01", yy, 7, 1)  # mid-year fallback
    return (None, None, None, None)

# Column block layouts (start indices verified from header dump) -----------------
# Paper construction (create_scandals.do): type-A = scandal1 only; type-B = abuse1..6.
SCANDAL_BLOCKS = [
    dict(date=16, name=21, addr=22, postal=23, county=24, state=25, lat=26, lon=27,
         pos=14, imp=19, flag=20),  # scandal1
]
ABUSE_BLOCKS = [
    dict(link=50,  sdate=51,  name=58,  addr=59,  postal=60,  county=61,  state=62,  lat=63,  lon=64,  pos=47,  flag=57),
    dict(link=69,  sdate=70,  name=77,  addr=78,  postal=79,  county=80,  state=81,  lat=82,  lon=83,  pos=66,  flag=76),
    dict(link=88,  sdate=89,  name=97,  addr=98,  postal=99,  county=100, state=101, lat=102, lon=103, pos=85,  flag=94),
    dict(link=108, sdate=109, name=115, addr=116, postal=117, county=118, state=119, lat=120, lon=121, pos=105, flag=114),
    dict(link=126, sdate=127, name=133, addr=134, postal=135, county=136, state=137, lat=138, lon=139, pos=123, flag=132),
    dict(link=144, sdate=145, name=151, addr=152, postal=153, county=154, state=155, lat=156, lon=157, pos=141, flag=150),
]

US_LAT = (17.0, 72.0)   # incl. Alaska/Hawaii
US_LON = (-180.0, -64.0)

def valid_coord(lat, lon):
    return (lat is not None and lon is not None
            and US_LAT[0] <= lat <= US_LAT[1] and US_LON[0] <= lon <= US_LON[1])

def get(row, idx):
    return row[idx] if idx < len(row) else ""

events = []
priests = set()
skipped_no_coord = 0
skipped_no_date = 0

for r in data:
    last = clean(get(r, 1)); first = clean(get(r, 2))
    if not last and not first:
        continue
    priest = f"{first} {last}".strip()
    role = clean(get(r, 4))
    diocese = clean(get(r, 7))
    minor = clean(get(r, 8))
    gender = clean(get(r, 9))
    naccus = clean(get(r, 10))
    if clean(get(r, 13)) != "":          # paper criterion: keep if pornography==""
        continue
    priests.add(priest.lower())

    ba_link = ""
    for b in ABUSE_BLOCKS:
        l = clean(get(r, b["link"]))
        if "bishop" in l.lower() or l.lower().startswith("http"):
            ba_link = l
            break

    def make_event(kind, date_raw, name, addr, postal, county, state, lat, lon, link="",
                   pos="", imp="", flag=""):
        global skipped_no_coord, skipped_no_date
        # paper sample-selection (create_scandals.do): drop position J/OD, importance==1, non-geocoded
        if clean(pos) in ("J", "OD"):
            return None
        if clean(imp) == "1":
            return None
        if clean(flag) not in ("", "0", "0.0"):
            return None
        iso, yy, mm, dd = parse_date(date_raw)
        latf, lonf = num(lat), num(lon)
        if not valid_coord(latf, lonf):
            skipped_no_coord += 1
            return None
        if iso is None:
            skipped_no_date += 1
            return None
        return dict(
            type=kind, priest=priest, last_name=last, first_name=first,
            role=role, diocese=diocese, minor_or_adult=minor, gender=gender,
            n_accusations=naccus,
            date=iso, year=yy, month=mm, day=dd,
            institution=clean(name), address=clean(addr), postal=clean(postal),
            county=clean(county), state=clean(state),
            lat=round(latf, 6), lon=round(lonf, 6),
            news_link=clean(link) or ba_link, ba_link=ba_link,
        )

    for b in SCANDAL_BLOCKS:
        ev = make_event("A", get(r, b["date"]), get(r, b["name"]), get(r, b["addr"]),
                        get(r, b["postal"]), get(r, b["county"]), get(r, b["state"]),
                        get(r, b["lat"]), get(r, b["lon"]),
                        pos=get(r, b["pos"]), imp=get(r, b["imp"]), flag=get(r, b["flag"]))
        if ev:
            events.append(ev)
    for b in ABUSE_BLOCKS:
        ev = make_event("B", get(r, b["sdate"]), get(r, b["name"]), get(r, b["addr"]),
                        get(r, b["postal"]), get(r, b["county"]), get(r, b["state"]),
                        get(r, b["lat"]), get(r, b["lon"]), link=get(r, b["link"]),
                        pos=get(r, b["pos"]), flag=get(r, b["flag"]))
        if ev:
            events.append(ev)

events.sort(key=lambda e: (e["date"], e["priest"]))
for i, e in enumerate(events):
    e["id"] = i + 1

nA = sum(1 for e in events if e["type"] == "A")
nB = sum(1 for e in events if e["type"] == "B")
print(f"clergy members with a name: {len(priests)}")
print(f"geocoded+dated events: {len(events)}  (type-A={nA}, type-B={nB})")
print(f"skipped (no/invalid coord): {skipped_no_coord} | skipped (no parseable date): {skipped_no_date}")
yrs = [e['year'] for e in events]
print(f"year range: {min(yrs)}-{max(yrs)}")

import csv as _csv
fields = ["id","type","priest","last_name","first_name","role","diocese",
          "minor_or_adult","gender","n_accusations","date","year","month","day",
          "institution","address","postal","county","state","lat","lon",
          "news_link","ba_link"]
with open(OUT/"scandals_events.csv", "w", newline="", encoding="utf-8") as f:
    w = _csv.DictWriter(f, fieldnames=fields); w.writeheader()
    for e in events: w.writerow(e)

with open(OUT/"scandals_events.json", "w", encoding="utf-8") as f:
    json.dump(events, f, ensure_ascii=False)

compact = [dict(i=e["id"], t=e["type"], p=e["priest"], d=e["date"],
                y=e["year"], m=e["month"],
                lat=e["lat"], lon=e["lon"], inst=e["institution"],
                city=e["address"], county=e["county"], st=e["state"],
                role=e["role"], dio=e["diocese"], link=e["ba_link"])
           for e in events]
with open(OUT/"scandals_map.json", "w", encoding="utf-8") as f:
    json.dump(compact, f, ensure_ascii=False, separators=(",", ":"))

# GeoJSON ----------------------------------------------------------------------
geo = {"type": "FeatureCollection", "features": [
    {"type": "Feature",
     "geometry": {"type": "Point", "coordinates": [e["lon"], e["lat"]]},
     "properties": {k: e[k] for k in fields if k not in ("lat", "lon")}}
    for e in events]}
with open(OUT/"scandals_events.geojson", "w", encoding="utf-8") as f:
    json.dump(geo, f, ensure_ascii=False)

# ---------------------------------------------------------------- aggregations
from collections import defaultdict

def aggregate(keyfn, keynames):
    agg = defaultdict(lambda: {"n_total": 0, "n_typeA": 0, "n_typeB": 0,
                               "n_priests": set()})
    for e in events:
        k = keyfn(e)
        if k is None:
            continue
        a = agg[k]
        a["n_total"] += 1
        a["n_typeA"] += (e["type"] == "A")
        a["n_typeB"] += (e["type"] == "B")
        a["n_priests"].add(e["priest"])
    rows = []
    for k, a in agg.items():
        kk = k if isinstance(k, tuple) else (k,)
        row = dict(zip(keynames, kk))
        row.update(n_total=a["n_total"], n_typeA=a["n_typeA"],
                   n_typeB=a["n_typeB"], n_priests=len(a["n_priests"]))
        rows.append(row)
    rows.sort(key=lambda r: tuple(str(r[n]) for n in keynames))
    return rows, keynames + ["n_total", "n_typeA", "n_typeB", "n_priests"]

def write_csv(path, rows, cols):
    with open(path, "w", newline="", encoding="utf-8") as f:
        w = _csv.DictWriter(f, fieldnames=cols); w.writeheader()
        for r in rows: w.writerow(r)

# Panels are SPARSE: one row per (unit, year) cell that has >=1 scandal event.
# Cells with zero events are omitted (the public scandal file has no universe of
# all zips/counties; cross-sectional totals are recoverable by summing over year).
aggs = {
    "by_year":   aggregate(lambda e: e["year"], ["year"]),
    "by_state":  aggregate(lambda e: e["state"] or None, ["state"]),
    "by_year_state": aggregate(lambda e: (e["year"], e["state"]) if e["state"] else None,
                               ["year", "state"]),
    "by_county_year": aggregate(
        lambda e: (e["state"], e["county"], e["year"]) if e["county"] else None,
        ["state", "county", "year"]),
    "by_zip_year": aggregate(
        lambda e: (e["postal"], e["year"]) if e["postal"] else None,
        ["zip", "year"]),
}
for name, (rows, cols) in aggs.items():
    write_csv(OUT/f"scandals_{name}.csv", rows, cols)
    print(f"  agg {name}: {len(rows)} rows -> scandals_{name}.csv")

# ---------------------------------------------------------------- summary stats
by_year_rows = aggs["by_year"][0]
summary = {
    "n_events": len(events), "n_typeA": nA, "n_typeB": nB,
    "n_priests": len(set(e["priest"] for e in events)),
    "n_states": len(set(e["state"] for e in events if e["state"])),
    "n_with_link": sum(1 for e in events if e["ba_link"]),
    "year_min": min(yrs), "year_max": max(yrs),
    "events_1980_2010": sum(1 for e in events if 1980 <= e["year"] <= 2010),
    "typeA_1980_2010": sum(1 for e in events if e["type"]=="A" and 1980<=e["year"]<=2010),
    "typeB_1980_2010": sum(1 for e in events if e["type"]=="B" and 1980<=e["year"]<=2010),
    "paper_typeA": 1125, "paper_typeB": 1899, "paper_total": 3024,
    "by_year": [{"year": r["year"], "A": r["n_typeA"], "B": r["n_typeB"],
                 "total": r["n_total"]} for r in by_year_rows],
}
with open(OUT/"summary.json", "w", encoding="utf-8") as f:
    json.dump(summary, f, ensure_ascii=False, indent=2)

print("wrote events (csv/json/geojson), compact map json, 5 aggregations, summary.json")