"""URL plus column-spec scraper.

Takes a URL and a JSON column spec, fetches the page with Playwright,
extracts one row per element matched by the spec row selector, writes
CSV, and writes a short run log with row counts, empty-field counts,
and failures.

Spec format:

    {
      "row_selector": ".book",
      "columns": {
        "title": {"selector": ".title", "required": true},
        "price": {"selector": ".price"},
        "link": {"selector": "a", "attribute": "href"}
      }
    }

A flat mapping is also accepted, with the row selector passed by flag:

    {"title": ".title", "link": {"selector": "a", "attribute": "href"}}

If any column marked "required" is empty in any row, the run is
treated as failed: the log records the empty counts and the CLI
exits with status 2. Silent empty columns are the failure this
guard exists to catch.
"""

from __future__ import annotations

import argparse
import csv
import json
import sys
from pathlib import Path

from playwright.sync_api import sync_playwright

DEFAULT_PROXY = "http://127.0.0.1:18082"


class EmptyFieldError(Exception):
    """Raised when a required column has empty values."""


def load_spec(path: str | Path) -> dict:
    """Load and normalize a column spec file."""
    raw = json.loads(Path(path).read_text(encoding="utf-8"))
    if "columns" in raw:
        row_selector = raw.get("row_selector")
        columns_raw = raw["columns"]
    else:
        row_selector = None
        columns_raw = raw
    columns: dict[str, dict] = {}
    for name, value in columns_raw.items():
        if isinstance(value, str):
            columns[name] = {"selector": value}
        else:
            columns[name] = dict(value)
    return {"row_selector": row_selector, "columns": columns}


def extract_rows(page, spec: dict, row_selector: str) -> list[dict]:
    """Extract rows from a loaded Playwright page using the spec."""
    rows: list[dict] = []
    row_elements = page.query_selector_all(row_selector)
    for row_el in row_elements:
        row: dict[str, str] = {}
        for name, col in spec["columns"].items():
            el = row_el.query_selector(col["selector"])
            if el is None:
                row[name] = ""
                continue
            attribute = col.get("attribute")
            if attribute:
                row[name] = (el.get_attribute(attribute) or "").strip()
            else:
                row[name] = (el.inner_text() or "").strip()
        rows.append(row)
    return rows


def empty_counts(rows: list[dict], columns: dict) -> dict[str, int]:
    return {name: sum(1 for r in rows if not r.get(name)) for name in columns}


def check_required(rows: list[dict], spec: dict) -> None:
    counts = empty_counts(rows, spec["columns"])
    bad = [
        name
        for name, col in spec["columns"].items()
        if col.get("required") and counts[name] > 0
    ]
    if bad:
        raise EmptyFieldError(
            "required column(s) empty: "
            + ", ".join(f"{name} ({counts[name]} empty)" for name in bad)
        )


def write_csv(rows: list[dict], columns: dict, out_path: str | Path) -> None:
    with open(out_path, "w", newline="", encoding="utf-8") as fh:
        writer = csv.DictWriter(fh, fieldnames=list(columns.keys()))
        writer.writeheader()
        writer.writerows(rows)


def write_log(
    log_path: str | Path,
    url: str,
    rows: list[dict],
    spec: dict,
    failures: list[str],
) -> None:
    counts = empty_counts(rows, spec["columns"])
    lines = [
        f"url: {url}",
        f"rows found: {len(rows)}",
        "empty fields per column:",
    ]
    lines += [f"  {name}: {count}" for name, count in counts.items()]
    lines.append("failures:")
    lines += [f"  {item}" for item in failures] if failures else ["  none"]
    Path(log_path).write_text("\n".join(lines) + "\n", encoding="utf-8")


def scrape(url: str, spec: dict, row_selector: str, proxy: str | None) -> list[dict]:
    launch_kwargs: dict = {}
    if proxy and url.startswith("http"):
        launch_kwargs["proxy"] = {"server": proxy}
    with sync_playwright() as pw:
        browser = pw.chromium.launch(**launch_kwargs)
        context = browser.new_context(ignore_https_errors=True)
        page = context.new_page()
        page.goto(url)
        rows = extract_rows(page, spec, row_selector)
        browser.close()
    return rows


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description="URL plus column-spec scraper")
    parser.add_argument("--url", required=True, help="Page URL or file:// URL")
    parser.add_argument("--spec", required=True, help="Path to column spec JSON")
    parser.add_argument("--out", default="output.csv", help="CSV output path")
    parser.add_argument("--log", default="run-log.txt", help="Run log path")
    parser.add_argument("--row-selector", default=None, help="Overrides spec row_selector")
    parser.add_argument("--proxy", default=DEFAULT_PROXY, help="Proxy for http(s) URLs")
    parser.add_argument("--no-proxy", action="store_true", help="Disable proxy")
    args = parser.parse_args(argv)

    spec = load_spec(args.spec)
    row_selector = args.row_selector or spec["row_selector"]
    if not row_selector:
        print("error: no row selector in spec and none passed by flag", file=sys.stderr)
        return 1

    failures: list[str] = []
    rows: list[dict] = []
    try:
        rows = scrape(args.url, spec, row_selector, None if args.no_proxy else args.proxy)
        check_required(rows, spec)
    except EmptyFieldError as exc:
        failures.append(str(exc))
        write_csv(rows, spec["columns"], args.out)
        write_log(args.log, args.url, rows, spec, failures)
        print(f"failed: {exc}", file=sys.stderr)
        return 2
    except Exception as exc:  # fetch or extraction failure
        failures.append(f"{type(exc).__name__}: {exc}")
        write_log(args.log, args.url, rows, spec, failures)
        print(f"failed: {exc}", file=sys.stderr)
        return 1

    write_csv(rows, spec["columns"], args.out)
    write_log(args.log, args.url, rows, spec, failures)
    print(f"wrote {len(rows)} rows to {args.out}, log at {args.log}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
