#!/usr/bin/env python3
"""Share of household spending on a proxy for 'essential' categories, by income quintile.

Source: U.S. Bureau of Labor Statistics, Consumer Expenditure Surveys (CE),
annual mean expenditures, 1984-2023, via the public BLS API v1
(https://api.bls.gov/publicAPI/v1/timeseries/data/, no key, 25 queries/day).

Series id = "CXU" + item + demographic + "M":
  items:   TOTALEXP total expenditures; FOODTOTL food; SHELTER shelter; UTILS utilities;
           TRANS transportation; HEALTH health care; INCAFTTX income after taxes
  groups:  LB0101 all consumer units; LB0102..LB0106 income-before-taxes quintiles 1 (lowest) to 5

Proxy = (FOODTOTL + SHELTER + UTILS + TRANS + HEALTH) / TOTALEXP.
It omits childcare and communications and counts all vehicle costs, so it is a coarse
stand-in for the book's GEC, not the GEC itself.

Usage:
  python3 fetch_and_compute.py            # fetch missing series into ce_raw.csv, then compute
  python3 fetch_and_compute.py --offline  # compute from ce_raw.csv only
Outputs: ce_raw.csv, ce_shares.csv
"""
import csv, json, subprocess, sys
from pathlib import Path

HERE = Path(__file__).resolve().parent
RAW = HERE / "ce_raw.csv"
OUT = HERE / "ce_shares.csv"
ITEMS = ["TOTALEXP", "FOODTOTL", "SHELTER", "UTILS", "TRANS", "HEALTH"]
GROUPS = {"all": "LB0101", "q1_lowest": "LB0102", "q2": "LB0103", "q3_middle": "LB0104", "q4": "LB0105", "q5_highest": "LB0106"}
WINDOWS = [(1984, 1993), (1994, 2003), (2004, 2013), (2014, 2023)]
ESSENTIAL = ["FOODTOTL", "SHELTER", "UTILS", "TRANS", "HEALTH"]


def sid(item, demo):
    return f"CXU{item}{demo}M"


def load():
    data = {}
    if RAW.exists():
        for r in csv.DictReader(open(RAW)):
            data.setdefault(r["series_id"], {})[int(r["year"])] = float(r["value_usd"])
    return data


def fetch(ids, a, b, data):
    payload = json.dumps({"seriesid": ids, "startyear": str(a), "endyear": str(b)})
    out = subprocess.run(["curl", "-sS", "-m", "60", "-X", "POST", "-H", "Content-type: application/json",
                          "-d", payload, "https://api.bls.gov/publicAPI/v1/timeseries/data/"],
                         capture_output=True, text=True).stdout
    d = json.loads(out)
    if d["status"] != "REQUEST_SUCCEEDED" or d["message"]:
        sys.exit(f"BLS API problem for {a}-{b}: {d['status']} {d['message'][:3]}")
    for s in d["Results"]["series"]:
        for r in s["data"]:
            data.setdefault(s["seriesID"], {})[int(r["year"])] = float(r["value"])


def save(data):
    with open(RAW, "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["series_id", "year", "value_usd"])
        for k in sorted(data):
            for y in sorted(data[k]):
                v = data[k][y]
                w.writerow([k, y, int(v) if v == int(v) else v])


def main():
    data = load()
    if "--offline" not in sys.argv:
        for a, b in WINDOWS:
            missing = [sid(i, g) for g in GROUPS.values() for i in ITEMS
                       if not all(y in data.get(sid(i, g), {}) for y in range(a, b + 1))]
            for k in range(0, len(missing), 25):  # API limit: 25 series per query
                fetch(missing[k:k + 25], a, b, data)
        save(data)
    rows = []
    for name, g in GROUPS.items():
        for y in range(1984, 2024):
            tot = data[sid("TOTALEXP", g)][y]
            ess = sum(data[sid(i, g)][y] for i in ESSENTIAL)
            row = {"group": name, "year": y, "total_expenditure": tot,
                   "essential_proxy_pct": round(ess / tot * 100, 2)}
            for i in ESSENTIAL:
                row[i.lower() + "_pct"] = round(data[sid(i, g)][y] / tot * 100, 2)
            rows.append(row)
    with open(OUT, "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=rows[0].keys())
        w.writeheader()
        w.writerows(rows)
    print("1984 -> 2023, food share / essential proxy (% of total expenditure)")
    for name in GROUPS:
        a = next(r for r in rows if r["group"] == name and r["year"] == 1984)
        b = next(r for r in rows if r["group"] == name and r["year"] == 2023)
        print(f"{name:12s} food {a['foodtotl_pct']:5.1f} -> {b['foodtotl_pct']:5.1f}   "
              f"proxy {a['essential_proxy_pct']:5.1f} -> {b['essential_proxy_pct']:5.1f}")


if __name__ == "__main__":
    main()