#!/usr/bin/env python3
"""
01_fetch_usgs.py — download the USGS ComCat earthquake catalogue, 1980–2025
inclusive, magnitude >= 4.5, as one CSV.

Standard library only. Queries the FDSN event service month by month
(the service caps a single reply at 20,000 events; the largest month
at M>=4.5 is far below that), retries politely, de-duplicates on the
USGS event id, and writes:

    _to_delete/eq/usgs_1980_2025_m45.csv

Columns kept: time (UTC ISO-8601), latitude, longitude, depth (km),
mag, magType, place, id, type.

Re-running resumes: months already saved under _to_delete/eq/months/
are not fetched again, so an interrupted run can simply be restarted.

Usage (from the repository root):
    python3 _to_delete/eq/fetch_usgs.py
Optional:
    python3 _to_delete/eq/fetch_usgs.py --minmag 4.5 --start 1980 --end 2025
"""
import argparse
import csv
import io
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

BASE = "https://earthquake.usgs.gov/fdsnws/event/1/query"
KEEP = ["time", "latitude", "longitude", "depth", "mag", "magType",
        "place", "id", "type"]


def month_ranges(y0, y1):
    for y in range(y0, y1 + 1):
        for m in range(1, 13):
            start = f"{y:04d}-{m:02d}-01"
            ny, nm = (y + 1, 1) if m == 12 else (y, m + 1)
            end = f"{ny:04d}-{nm:02d}-01"
            yield start, end


def fetch(start, end, minmag, tries=6):
    params = {
        "format": "csv",
        "starttime": start,
        "endtime": end,          # exclusive
        "minmagnitude": f"{minmag:g}",
        "orderby": "time-asc",
        "eventtype": "earthquake",
    }
    url = BASE + "?" + urllib.parse.urlencode(params)
    delay = 5
    for attempt in range(1, tries + 1):
        try:
            req = urllib.request.Request(
                url, headers={"User-Agent": "ayurastro-bhukampa/1.0"})
            with urllib.request.urlopen(req, timeout=180) as r:
                return r.read().decode("utf-8")
        except (urllib.error.URLError, urllib.error.HTTPError,
                TimeoutError, ConnectionError) as e:
            if attempt == tries:
                raise
            print(f"   retry {attempt}/{tries} after error: {e}",
                  file=sys.stderr)
            time.sleep(delay)
            delay = min(delay * 2, 120)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--start", type=int, default=1980)
    ap.add_argument("--end", type=int, default=2025)
    ap.add_argument("--minmag", type=float, default=4.5)
    ap.add_argument("--outdir", default=os.path.join("_to_delete", "eq"))
    a = ap.parse_args()

    monthdir = os.path.join(a.outdir, "months")
    os.makedirs(monthdir, exist_ok=True)
    tag = f"m{a.minmag:g}".replace(".", "")
    out_path = os.path.join(a.outdir, f"usgs_{a.start}_{a.end}_{tag}.csv")

    ranges = list(month_ranges(a.start, a.end))
    total = len(ranges)
    for i, (s, e) in enumerate(ranges, 1):
        f = os.path.join(monthdir, f"{s[:7]}.csv")
        if os.path.exists(f) and os.path.getsize(f) > 0:
            continue
        txt = fetch(s, e, a.minmag)
        n = max(0, txt.count("\n") - 1)
        if n >= 20000:
            sys.exit(f"month {s[:7]} hit the 20,000-event cap; "
                     f"lower the range or raise --minmag")
        with open(f, "w", encoding="utf-8", newline="") as fh:
            fh.write(txt)
        print(f"[{i:3d}/{total}] {s[:7]}  {n:5d} events")
        time.sleep(0.4)   # be gentle with the service

    seen = set()
    rows = 0
    with open(out_path, "w", encoding="utf-8", newline="") as out:
        w = csv.DictWriter(out, fieldnames=KEEP)
        w.writeheader()
        for s, e in ranges:
            f = os.path.join(monthdir, f"{s[:7]}.csv")
            with open(f, encoding="utf-8", newline="") as fh:
                for row in csv.DictReader(fh):
                    if row.get("id") in seen or not row.get("mag"):
                        continue
                    seen.add(row["id"])
                    w.writerow({k: row.get(k, "") for k in KEEP})
                    rows += 1
    print(f"\nwrote {rows} unique events -> {out_path}")


if __name__ == "__main__":
    main()