scraper.py

import anthropic
import httpx
import json
import re
import time
import csv
import logging
import pdfplumber
import phonenumbers
import extruct
from io import BytesIO
from pathlib import Path
from bs4 import BeautifulSoup
from typing import Optional, Dict, List, Tuple, Set
from urllib.parse import urlparse, urljoin

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[logging.FileHandler("scraper_v2_effective.log"), logging.StreamHandler()]
)
log = logging.getLogger(__name__)

# ── Config ────────────────────────────────────────────────────────────────────
ANTHROPIC_API_KEY = "YOUR_ANTHROPIC_API_KEY"
SERP_API_KEY      = "YOUR_SERP_API_KEY"
COUNTY_SITES_JSON = "county_sites.json"

CLAUDE_MODEL      = "claude-sonnet-4-6"
REQUEST_DELAY     = 0.6
FETCH_TIMEOUT     = 10
MAX_BYTES         = 150000

# Split confidence thresholds by source trust
CONFIDENCE_GOV    = 0.60   # .gov sources — lower threshold, already high trust
CONFIDENCE_OTHER  = 0.72   # everything else

MAX_SOURCES       = 8      # increased from 6
USE_PLAYWRIGHT    = True

# ── Known Michigan county email domain patterns ───────────────────────────────
# Many counties follow predictable email patterns.
# We try these directly if we know the domain.
EMAIL_DOMAIN_PATTERNS = [
    "{first}.{last}@{slug}county.gov",
    "{first}.{last}@{slug}countymi.gov",
    "{first}{last}@{slug}county.gov",
    "{f}{last}@{slug}county.gov",
    "{last}@{slug}county.gov",
    "{first}.{last}@{slug}.gov",
    "{f}{last}@{slug}countymi.gov",
    "{last}@{slug}countymi.gov",
]

# ── Authoritative directories ─────────────────────────────────────────────────
AUTHORITATIVE_SOURCES: Dict[str, List[str]] = {
    "Prosecuting Attorney": [
        "https://michiganprosecutor.org/about-us/prosecutor-directory/",
    ],
    "Sheriff": [
        "https://www.misheriff.org/sheriffs-offices/",
    ],
    "_all": [],
}

# ── Per-position directory paths ──────────────────────────────────────────────
# Tried in order on the county .gov site BEFORE generic patterns
POSITION_SPECIFIC_PATHS: Dict[str, List[str]] = {
    "Board of Commissioners Chairman": [
        "/board-of-commissioners", "/boc", "/commissioners",
        "/government/board_of_commissioners", "/government/commissioners",
        "/government/elected-officials", "/elected-officials",
        "/government/board-of-commissioners",
    ],
    "County Administrator": [
        "/administration", "/administrator", "/county-administrator",
        "/departments/administration", "/government/administration",
        "/management", "/departments/county-administration",
    ],
    "County Clerk": [
        "/clerk", "/county-clerk", "/departments/clerk",
        "/government/clerk", "/elected-officials/clerk",
        "/government/county-clerk",
    ],
    "County Treasurer": [
        "/treasurer", "/county-treasurer", "/departments/treasurer",
        "/government/treasurer", "/elected-officials/treasurer",
    ],
    "Prosecuting Attorney": [
        "/prosecutor", "/prosecuting-attorney", "/pa",
        "/departments/prosecutor", "/government/prosecutor",
        "/departments/prosecuting-attorney",
    ],
    "Sheriff": [
        "/sheriff", "/sheriffs-office", "/sheriff-office",
        "/departments/sheriff", "/government/sheriff",
    ],
    "Register of Deeds": [
        "/register-of-deeds", "/rod",
        "/departments/register-of-deeds",
        "/government/register-of-deeds",
    ],
    "Chief Executive Officer": [
        "/county-executive", "/executive",
        "/government/county-executive", "/government/executive",
    ],
}

# Generic fallback paths tried after position-specific ones
GENERIC_PATHS: List[str] = [
    "/directory", "/staff-directory", "/staff",
    "/contact", "/contact-us",
    "/elected-officials", "/government/elected-officials",
    "/government/staff", "/departments",
    "/about/staff", "/about",
    "/government", "/government/departments",
]

GARBAGE_URL_PATTERNS = [
    "archive.org", "ancestry.com", "findagrave", "genealogy",
    "newspapers.com", "wikitree", "historicmapworks", "fold3.com",
    "govinfo.gov/content/pkg/CDIR-191",
    "govinfo.gov/content/pkg/CDIR-192",
    "govinfo.gov/content/pkg/CDIR-193",
    "govinfo.gov/content/pkg/CDIR-194",
    "govinfo.gov/content/pkg/CDIR-195",
    "govinfo.gov/content/pkg/CDIR-196",
    "govinfo.gov/content/pkg/CDIR-197",
    "govinfo.gov/content/pkg/CDIR-198",
    "govinfo.gov/content/pkg/CDIR-199",
    "govinfo.gov/content/pkg/CDIR-200",
    "govinfo.gov/content/pkg/CDIR-201",
    "1918", "historicmaps",
]

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
}

TRUSTED_DOMAINS = [
    (".gov",                   100),
    ("michiganprosecutor.org",  95),
    ("misheriff.org",           90),
    ("michigan.gov",            85),
    ("ballotpedia.org",         60),
    ("facebook.com",            50),
    ("instagram.com",           50),
    ("twitter.com",             45),
    ("linkedin.com",            45),
]

def domain_score(url: str) -> int:
    for domain, score in TRUSTED_DOMAINS:
        if domain in url:
            return score
    return 30

def is_gov_url(url: str) -> bool:
    return ".gov" in url or "michiganprosecutor.org" in url or "misheriff.org" in url

# ── County sites ──────────────────────────────────────────────────────────────
KNOWN_COUNTY_SITES: Dict[str, dict] = {}

def load_county_sites():
    global KNOWN_COUNTY_SITES
    p = Path(COUNTY_SITES_JSON)
    if not p.exists():
        log.info("county_sites.json not found")
        return
    with open(p) as f:
        data = json.load(f)
    for county, info in data.items():
        if info.get("base_url") and info.get("confidence", 0) >= 0.5:
            KNOWN_COUNTY_SITES[county] = info
    log.info(f"Loaded {len(KNOWN_COUNTY_SITES)} county sites")

def get_county_base_url(county: str) -> str:
    return KNOWN_COUNTY_SITES.get(county, {}).get("base_url", "")

def get_county_slug(county: str) -> str:
    return county.lower().replace(" county","").replace(" ","").replace(".","").replace("-","")

# ── Email pattern inference ───────────────────────────────────────────────────
def infer_email_candidates(name: str, county: str) -> List[str]:
    """
    Generate likely email addresses based on known county domain patterns.
    These are tested by checking MX records / SMTP, or just passed to Claude as hints.
    """
    parts = name.lower().split()
    if len(parts) < 2:
        return []
    first = parts[0]
    last  = parts[-1]
    f     = first[0]
    slug  = get_county_slug(county)

    candidates = []
    for pattern in EMAIL_DOMAIN_PATTERNS:
        try:
            email = pattern.format(first=first, last=last, f=f, slug=slug)
            candidates.append(email)
        except Exception:
            pass

    # Also try county-specific known domains from county_sites
    base_url = get_county_base_url(county)
    if base_url:
        domain = urlparse(base_url).netloc.replace("www.", "")
        for tmpl in [
            f"{first}.{last}@{domain}",
            f"{f}{last}@{domain}",
            f"{last}@{domain}",
        ]:
            if tmpl not in candidates:
                candidates.append(tmpl)

    return candidates[:8]

# ── Phone/email/address extraction ───────────────────────────────────────────
EMAIL_RE = re.compile(r'\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b')
PHONE_RE = re.compile(
    r'(?:\+?1[\s.\-]?)?'
    r'(?:\(?\d{3}\)?[\s.\-]?)'
    r'\d{3}[\s.\-]?\d{4}'
)
ADDRESS_RE = re.compile(
    r'\d{1,5}\s+[A-Za-z0-9\s.,#]+'
    r'(?:St|Street|Ave|Avenue|Blvd|Boulevard|Rd|Road|Dr|Drive|'
    r'Ln|Lane|Way|Ct|Court|Pl|Place|Hwy|Highway|Suite|Ste)\b'
    r'[.,\s]*(?:[A-Za-z\s]+,\s*MI\s*\d{5})?',
    re.IGNORECASE
)
SOCIAL_RE = {
    "facebook": re.compile(r'facebook\.com/(?!sharer|share|login|home|watch)([A-Za-z0-9._\-/]+)', re.IGNORECASE),
    "instagram": re.compile(r'instagram\.com/([A-Za-z0-9._]+)', re.IGNORECASE),
    "twitter_x": re.compile(r'(?:twitter|x)\.com/([A-Za-z0-9_]+)', re.IGNORECASE),
}

def regex_preextract(text: str) -> dict:
    found: Dict[str, list] = {}

    emails = EMAIL_RE.findall(text)
    real = [e for e in emails if not any(j in e.lower() for j in
            ["noreply","no-reply","webmaster","info@","admin@","support@",
             "news@","example@","donotreply","help@","contact@"])]
    if real:
        found["email_candidates"] = real[:8]

    phones = PHONE_RE.findall(text)
    if phones:
        found["phone_candidates"] = list(dict.fromkeys(
            normalise_phone(p) for p in phones
        ))[:10]

    addresses = ADDRESS_RE.findall(text)
    if addresses:
        found["address_candidates"] = [a.strip() for a in addresses[:5]]

    for platform, rx in SOCIAL_RE.items():
        matches = rx.findall(text)
        if matches:
            found[f"{platform}_candidates"] = [
                f"https://{platform.replace('twitter_x','x')}.com/{m.rstrip('/')}"
                for m in matches[:3]
            ]

    return found

def normalise_phone(raw: str) -> str:
    try:
        cleaned = re.sub(r'[^\d+]', '', raw)
        if len(cleaned) == 10:
            cleaned = "1" + cleaned
        p = phonenumbers.parse("+" + cleaned, "US")
        if phonenumbers.is_valid_number(p):
            return phonenumbers.format_number(p, phonenumbers.PhoneNumberFormat.NATIONAL)
    except Exception:
        pass
    digits = re.sub(r'\D', '', raw)
    if len(digits) == 10:
        return f"{digits[:3]}-{digits[3:6]}-{digits[6:]}"
    return raw.strip()

def normalise_email(raw: str) -> str:
    return raw.strip().lower().replace("mailto:", "")

# ── Page section focus ────────────────────────────────────────────────────────
def extract_name_context(text: str, name: str, window: int = 2000) -> str:
    """
    Extract a window of text around the official's name.
    Gives Claude focused context instead of 6000 chars of noise.
    """
    last_name = name.split()[-1].lower()
    idx = text.lower().find(last_name)
    if idx == -1:
        return text[:window]
    start = max(0, idx - window // 2)
    end   = min(len(text), idx + window // 2)
    return text[start:end]

# ── Obfuscation decoding ──────────────────────────────────────────────────────
def decode_cloudflare_email(encoded: str) -> str:
    try:
        r = int(encoded[:2], 16)
        return "".join(chr(int(encoded[i:i+2], 16) ^ r) for i in range(2, len(encoded), 2))
    except Exception:
        return ""

def decode_all_obfuscated_emails(html: str) -> str:
    def cf_replace(m):
        decoded = decode_cloudflare_email(m.group(1))
        return decoded if decoded else ""
    html = re.sub(r'data-cfemail="([0-9a-f]+)"', cf_replace, html)
    html = re.sub(r'\[email[^]]*protected\]', '', html)
    html = re.sub(
        r'(\w[\w.+\-]*)\s*(?:\[?AT\]?|&#64;)\s*(\w[\w.\-]*)\s*(?:\[?DOT\]?|\.)\s*(\w{2,6})',
        lambda m: f"{m.group(1)}@{m.group(2)}.{m.group(3)}",
        html, flags=re.IGNORECASE
    )
    return html

# ── Structured data ───────────────────────────────────────────────────────────
def extract_structured_data(html: str, url: str) -> dict:
    try:
        data = extruct.extract(html, base_url=url, syntaxes=["json-ld", "microdata"])
        contacts: Dict[str, str] = {}
        for item in data.get("json-ld", []) + data.get("microdata", []):
            t = item.get("@type", "") or item.get("type", "")
            if t in ("Person","GovernmentOrganization","LocalBusiness","Organization","GovernmentOffice"):
                if item.get("email"):
                    contacts["email"] = item["email"].replace("mailto:", "")
                if item.get("telephone"):
                    contacts["phone"] = item["telephone"]
                if item.get("address"):
                    addr = item["address"]
                    if isinstance(addr, dict):
                        parts = [addr.get("streetAddress",""), addr.get("addressLocality",""),
                                 addr.get("addressRegion",""), addr.get("postalCode","")]
                        contacts["office_address"] = ", ".join(p for p in parts if p)
                if item.get("url"):
                    contacts["website"] = item["url"]
        return contacts
    except Exception:
        return {}

# ── PDF parsing ───────────────────────────────────────────────────────────────
def fetch_pdf_text(url: str) -> str:
    try:
        time.sleep(REQUEST_DELAY)
        with httpx.Client(timeout=FETCH_TIMEOUT, follow_redirects=True, headers=HEADERS) as client:
            with client.stream("GET", url) as r:
                r.raise_for_status()
                chunks, total = [], 0
                for chunk in r.iter_bytes(8192):
                    chunks.append(chunk)
                    total += len(chunk)
                    if total >= MAX_BYTES:
                        break
        with pdfplumber.open(BytesIO(b"".join(chunks))) as pdf:
            return "\n".join(p.extract_text() or "" for p in pdf.pages[:8])[:6000]
    except Exception as e:
        log.debug(f"PDF failed {url}: {e}")
        return ""

# ── Playwright ────────────────────────────────────────────────────────────────
def fetch_with_playwright(url: str) -> str:
    if not USE_PLAYWRIGHT:
        return ""
    try:
        from playwright.sync_api import sync_playwright
        with sync_playwright() as p:
            browser = p.chromium.launch(headless=True)
            page = browser.new_page()
            page.set_extra_http_headers({"User-Agent": HEADERS["User-Agent"]})
            page.goto(url, timeout=8000, wait_until="domcontentloaded")
            html = page.content()
            browser.close()
        html = decode_all_obfuscated_emails(html)
        soup = BeautifulSoup(html, "html.parser")
        for tag in soup(["script","style","nav","footer"]):
            tag.decompose()
        return re.sub(r'\s+', ' ', soup.get_text(separator=" ", strip=True))[:6000]
    except Exception as e:
        log.warning(f"Playwright failed {url}: {e}")
        return ""

# ── Page fetcher ──────────────────────────────────────────────────────────────
def fetch_page(url: str) -> Tuple[str, dict]:
    time.sleep(REQUEST_DELAY)
    try:
        with httpx.Client(timeout=FETCH_TIMEOUT, follow_redirects=True, headers=HEADERS) as client:
            try:
                head = client.head(url)
                ct = head.headers.get("content-type","")
                if "pdf" in ct or url.lower().endswith(".pdf"):
                    return fetch_pdf_text(url), {}
                if any(s in ct for s in ["image/","video/","audio/","application/zip"]):
                    return "", {}
            except Exception:
                pass

            with client.stream("GET", url) as r:
                r.raise_for_status()
                chunks, total = [], 0
                for chunk in r.iter_bytes(8192):
                    chunks.append(chunk)
                    total += len(chunk)
                    if total >= MAX_BYTES:
                        break
                raw_bytes = b"".join(chunks)

        raw_html = decode_all_obfuscated_emails(raw_bytes.decode("utf-8", errors="ignore"))
        structured = extract_structured_data(raw_html, url)

        soup = BeautifulSoup(raw_html, "lxml")
        for tag in soup(["script","style","nav","footer","header","aside","iframe"]):
            tag.decompose()
        text = re.sub(r'\s+', ' ', soup.get_text(separator=" ", strip=True))

        if len(text) < 200 and USE_PLAYWRIGHT:
            log.info(f"  Short page, trying Playwright: {url}")
            text = fetch_with_playwright(url)

        return text[:8000], structured

    except httpx.HTTPStatusError as e:
        if e.response.status_code in (403,429) and USE_PLAYWRIGHT:
            return fetch_with_playwright(url), {}
        log.debug(f"HTTP {e.response.status_code} for {url}")
        return "", {}
    except Exception as e:
        log.debug(f"Fetch failed {url}: {e}")
        return "", {}

# ── URL probe ─────────────────────────────────────────────────────────────────
def probe_url(url: str) -> bool:
    try:
        r = httpx.head(url, timeout=4, follow_redirects=True, headers=HEADERS)
        return r.status_code == 200
    except Exception:
        return False

# ── URL discovery ─────────────────────────────────────────────────────────────
def discover_county_urls(county: str, position: str) -> List[str]:
    info = KNOWN_COUNTY_SITES.get(county, {})
    base = info.get("base_url","")
    if not base:
        return []

    # 1. Already-discovered directory pages from county_sites.json
    dir_pages = info.get("directory_pages", {})
    pos_to_dir = {
        "Board of Commissioners Chairman": "commissioners",
        "County Administrator":            "commissioners",
        "County Clerk":                    "clerk",
        "County Treasurer":                "treasurer",
        "Prosecuting Attorney":            "prosecuting_attorney",
        "Sheriff":                         "sheriff",
        "Register of Deeds":              "staff_directory",
    }
    discovered: List[str] = []
    key = pos_to_dir.get(position,"")
    if key and key in dir_pages:
        discovered.append(dir_pages[key])
    if "staff_directory" in dir_pages:
        sd = dir_pages["staff_directory"]
        if sd not in discovered:
            discovered.append(sd)

    # 2. Position-specific paths + generic paths
    candidates = (
        POSITION_SPECIFIC_PATHS.get(position, []) +
        GENERIC_PATHS
    )
    with httpx.Client(timeout=4, follow_redirects=True, headers=HEADERS) as http_client:
        for path in candidates:
            url = base.rstrip("/") + path
            if url in discovered:
                continue
            try:
                r = http_client.head(url)
                if r.status_code == 200:
                    discovered.append(url)
                    if len(discovered) >= 5:
                        break
            except Exception:
                pass

    log.info(f"  Found {len(discovered)} direct county URLs")
    return discovered[:5]

# ── Search ────────────────────────────────────────────────────────────────────
def search_web(query: str, num: int = 6, recent_only: bool = False) -> List[dict]:
    try:
        from serpapi import GoogleSearch
        params = {"q": query, "num": num, "api_key": SERP_API_KEY}
        if recent_only:
            params["tbs"] = "qdr:y"
        results = GoogleSearch(params).get_dict().get("organic_results", [])
        filtered = []
        for r in results:
            url = r.get("link","")
            if any(g in url for g in GARBAGE_URL_PATTERNS):
                continue
            filtered.append({"url": url, "title": r.get("title",""), "snippet": r.get("snippet","")})
        return filtered
    except Exception as e:
        log.error(f"Search error: {e}")
        return []

def build_search_urls(name: str, position: str, county: str) -> List[Tuple[int, str]]:
    """
    Multi-pass search strategy:
    Pass 1: Targeted (name + county + position + contact)
    Pass 2: Broad (position + county directory)
    Pass 3: Social media dedicated
    Returns ranked list of (score, url)
    """
    county_clean = county.replace(" County","")
    last_name    = name.split()[-1]
    slug         = get_county_slug(county)

    ranked: List[Tuple[int, str]] = []
    seen: Set[str] = set()

    def add_results(results, bonus=0):
        for r in results:
            url = r.get("url","")
            if not url or url in seen:
                continue
            seen.add(url)
            score = domain_score(url) + bonus
            if last_name.lower() in (r.get("snippet","") + r.get("title","")).lower():
                score += 25
            if slug in url.lower() or county_clean.lower().replace(" ","") in url.lower():
                score += 15
            ranked.append((score, url))

    # Pass 1: Targeted name search
    queries_targeted = [
        f'"{name}" "{county_clean} County" Michigan {position} email phone',
        f'"{name}" {county_clean} Michigan {position} contact address',
        f'"{name}" {position} site:{slug}county.gov',
        f'"{name}" {position} site:{slug}countymi.gov',
        f'"{name}" Michigan {position} "@{slug}" OR "{slug}county.gov"',
    ]
    for q in queries_targeted[:3]:
        add_results(search_web(q, num=5), bonus=10)

    # Pass 2: Broad position+county search (finds directory pages)
    queries_broad = [
        f'{county_clean} County Michigan {position} contact 2025',
        f'{county_clean} County Michigan {position} office phone email address',
        f'site:{slug}county.gov {position}',
    ]
    for q in queries_broad[:2]:
        add_results(search_web(q, num=5))

    # Pass 3: Social media dedicated
    queries_social = [
        f'"{name}" {position} {county_clean} Michigan Facebook',
        f'"{name}" {county_clean} Michigan official Instagram Twitter',
    ]
    for q in queries_social[:2]:
        add_results(search_web(q, num=4))

    return ranked

# ── Claude client ─────────────────────────────────────────────────────────────
# Instantiated in run() so the key can be set/overridden before the first call
client: anthropic.Anthropic = None  # type: ignore

SYSTEM = """You are a precise government records researcher for Michigan county officials.
Extract ONLY explicitly stated contact information. Never guess, infer, or hallucinate.
null is ALWAYS better than a wrong value. Return valid JSON only."""

EXTRACT_PROMPT = """Find contact details for this Michigan county official in the page text below.

Official:
  Name:     {name}
  Position: {position}
  County:   {county}

Source URL: {url}
Domain trust score: {domain_score}/100

Regex-extracted candidates from this page (use as anchors):
{hints}

Inferred email patterns to look for (check if any appear in the text):
{inferred_emails}

Focused page text (around the official's name):
---
{context}
---

Full page text (for address/website):
---
{text}
---

Return ONLY this JSON (null if not found or uncertain):
{{
  "email":                   null,
  "phone":                   null,
  "office_address":          null,
  "website":                 null,
  "instagram":               null,
  "twitter_x":               null,
  "facebook":                null,
  "confidence":              0.0,
  "is_about_correct_person": true,
  "is_current":              true,
  "notes":                   null
}}

Rules:
- confidence=0.0 if page is NOT about {name} as {position} in {county}
- is_current=false if {name} is described as FORMER/EX/RETIRED/PREVIOUS
- Phone: NNN-NNN-NNNN format
- Email: lowercase
- Website: the specific office page, not county homepage
- For social media: provide full URL (e.g. https://facebook.com/pagename)
- If multiple phones on page pick the one directly under {name}'s name/title
- null is ALWAYS better than wrong"""

RECONCILE_PROMPT = """Pick the best verified contact info for this Michigan official.

Official: {name}, {position}, {county}

Sources (higher domain_score = more trustworthy):
{sources}

For each field:
- Prefer .gov sources (score 100) over all others
- If two sources agree → very high confidence
- If only one source has value from .gov → high confidence (0.85+)
- If only non-gov sources agree → medium confidence (0.65-0.75)
- If single non-gov source only → low confidence (0.50-0.60)

Return ONLY JSON:
{{
  "email":          {{"value": null, "confidence": 0.0}},
  "phone":          {{"value": null, "confidence": 0.0}},
  "office_address": {{"value": null, "confidence": 0.0}},
  "website":        {{"value": null, "confidence": 0.0}},
  "instagram":      {{"value": null, "confidence": 0.0}},
  "twitter_x":      {{"value": null, "confidence": 0.0}},
  "facebook":       {{"value": null, "confidence": 0.0}},
  "overall_confidence": 0.0,
  "flags": []
}}"""

VERIFY_PROMPT = """Verify this contact detail for a Michigan county official.

Official: {name}, {position}, {county}
Field: {field} = "{value}"
Source: {source_url} (domain score: {score}/100)
Context: "{context}"

Return JSON:
{{
  "verified": true,
  "confidence": 0.0,
  "reason": "",
  "corrected_value": null
}}"""


def stage1_extract(name: str, position: str, county: str, url: str,
                   text: str, hints: dict, inferred_emails: List[str]) -> dict:
    if not text or len(text) < 80:
        return {}

    # Name presence check before API call
    last_name  = name.split()[-1].lower()
    first_name = name.split()[0].lower()
    if last_name not in text.lower() and first_name not in text.lower():
        log.debug(f"  Name not on page, skipping: {url}")
        return {}

    # Extract focused context window around the name
    context = extract_name_context(text, name, window=2500)

    try:
        prompt = EXTRACT_PROMPT.format(
            name=name, position=position, county=county,
            url=url, domain_score=domain_score(url),
            hints=json.dumps(hints, indent=2) if hints else "none",
            inferred_emails=json.dumps(inferred_emails) if inferred_emails else "none",
            context=context[:2500],
            text=text[:4000],
        )
        resp = client.messages.create(
            model=CLAUDE_MODEL, max_tokens=600, system=SYSTEM,
            messages=[{"role": "user", "content": prompt}]
        )
        raw = re.sub(r"```json|```", "", resp.content[0].text).strip()
        result = json.loads(raw)
        result["_url"] = url
        result["_domain_score"] = domain_score(url)
        return result
    except Exception as e:
        log.debug(f"Stage1 failed {url}: {e}")
        return {}


def stage2_verify(name: str, position: str, county: str,
                  field: str, value: str, source_url: str, context: str) -> dict:
    if not value:
        return {"verified": False, "confidence": 0.0}
    try:
        resp = client.messages.create(
            model=CLAUDE_MODEL, max_tokens=200, system=SYSTEM,
            messages=[{"role": "user", "content": VERIFY_PROMPT.format(
                name=name, position=position, county=county,
                field=field, value=value, source_url=source_url,
                score=domain_score(source_url), context=context[:400]
            )}]
        )
        raw = re.sub(r"```json|```", "", resp.content[0].text).strip()
        return json.loads(raw)
    except Exception:
        return {"verified": True, "confidence": 0.6}


def stage3_reconcile(name: str, position: str, county: str, results: List[dict]) -> dict:
    if not results:
        return {}
    if len(results) == 1:
        r = results[0]
        c = r.get("confidence", 0.5)
        src = r.get("_url", "")
        return {
            **{f: {"value": r.get(f), "confidence": c, "source_url": src}
               for f in ["email","phone","office_address","website","instagram","twitter_x","facebook"]},
            "overall_confidence": c, "flags": []
        }
    enriched = [{
        **{k: v for k, v in r.items() if not k.startswith("_")},
        "source_url": r.get("_url",""),
        "domain_score": r.get("_domain_score", 30),
    } for r in results]
    try:
        resp = client.messages.create(
            model=CLAUDE_MODEL, max_tokens=700, system=SYSTEM,
            messages=[{"role": "user", "content": RECONCILE_PROMPT.format(
                name=name, position=position, county=county,
                sources=json.dumps(enriched, indent=2)
            )}]
        )
        raw = re.sub(r"```json|```", "", resp.content[0].text).strip()
        return json.loads(raw)
    except Exception as e:
        log.error(f"Reconcile failed: {e}")
        best = max(results, key=lambda r: r.get("_domain_score",0) * r.get("confidence",0))
        c = best.get("confidence", 0.4)
        return {
            **{f: {"value": best.get(f), "confidence": c}
               for f in ["email","phone","office_address","website","instagram","twitter_x","facebook"]},
            "overall_confidence": c, "flags": ["reconcile failed"]
        }

# ── Regex-only fallback for .gov pages ───────────────────────────────────────
def regex_only_fallback(name: str, text: str, url: str) -> dict:
    """
    If Claude finds nothing but the page is .gov and has the name,
    try pure regex extraction as last resort.
    """
    if not is_gov_url(url):
        return {}
    context = extract_name_context(text, name, window=1500)
    hints   = regex_preextract(context)
    result: Dict[str, Optional[str]] = {}

    # Take first email candidate
    if hints.get("email_candidates"):
        result["email"] = hints["email_candidates"][0]
    if hints.get("phone_candidates"):
        result["phone"] = hints["phone_candidates"][0]
    if hints.get("address_candidates"):
        result["office_address"] = hints["address_candidates"][0]

    if result:
        log.info(f"  Regex fallback found: {result}")
    return result


# ── Main scrape pipeline ──────────────────────────────────────────────────────
def scrape_official(name: str, position: str, county: str, existing: dict) -> dict:
    log.info(f"\n{'─'*60}")
    log.info(f"  {name} | {position} | {county}")

    missing = [f for f in ["Email","Phone","Website","Office_Address",
                            "Instagram","Twitter_X","Facebook"]
               if existing.get(f,"Not available") in ("Not available","",None)]
    if not missing:
        log.info("  All fields present — skipping")
        return existing
    log.info(f"  Missing: {missing}")

    # Pre-compute inferred email patterns
    inferred_emails = infer_email_candidates(name, county)
    if inferred_emails:
        log.debug(f"  Inferred email candidates: {inferred_emails[:3]}")

    # ── Phase 1: Direct county .gov URLs ────────────────────────────────────
    direct_urls = discover_county_urls(county, position)
    auth_urls   = AUTHORITATIVE_SOURCES.get(position,[]) + AUTHORITATIVE_SOURCES.get("_all",[])

    # ── Phase 2: Multi-pass search ──────────────────────────────────────────
    ranked = build_search_urls(name, position, county)
    ranked.sort(reverse=True)
    search_urls = [u for _, u in ranked[:MAX_SOURCES]]

    # Build final deduplicated URL list
    all_urls: List[str] = []
    seen_set: Set[str] = set()
    for u in direct_urls + auth_urls + search_urls:
        if u not in seen_set:
            all_urls.append(u)
            seen_set.add(u)
    all_urls = all_urls[:MAX_SOURCES + 3]   # allow a few extra

    # ── Phase 3: Fetch + extract ────────────────────────────────────────────
    source_results: List[dict] = []
    regex_fallback_result: dict = {}

    for url in all_urls:
        log.info(f"  Fetching: {url}")
        text, structured = fetch_page(url)
        if not text and not structured:
            continue

        pre = regex_preextract(text)
        if structured:
            pre["structured_data"] = structured
            if structured.get("email"):
                text = f"[STRUCTURED EMAIL: {structured['email']}]\n" + text
            if structured.get("phone"):
                text = f"[STRUCTURED PHONE: {structured['phone']}]\n" + text

        extracted = stage1_extract(name, position, county, url, text, pre, inferred_emails)

        if not extracted:
            # Try regex fallback on .gov pages even if Claude skipped it
            if is_gov_url(url) and name.split()[-1].lower() in text.lower():
                rb = regex_only_fallback(name, text, url)
                if rb and not regex_fallback_result:
                    regex_fallback_result = {**rb, "_url": url, "_domain_score": domain_score(url)}
            continue

        conf = extracted.get("confidence", 0)
        if not extracted.get("is_about_correct_person", True):
            log.info(f"    ✗ Wrong person (conf={conf:.2f})")
            continue
        if not extracted.get("is_current", True):
            log.warning(f"    ⚠ Possibly outdated: {url}")
            extracted["_outdated"] = True

        if conf >= 0.25:
            source_results.append(extracted)
            log.info(f"    ✓ conf={conf:.2f} | email={extracted.get('email')} | phone={extracted.get('phone')}")

            # Early exit on high-confidence .gov
            if conf >= 0.85 and domain_score(url) >= 85:
                log.info("    High-confidence .gov — stopping early")
                break

    # ── Phase 4: Stage 2 verify borderline values ────────────────────────────
    for result in source_results:
        url = result.get("_url","")
        if domain_score(url) < 60 or (0.4 < result.get("confidence",0) < 0.7):
            for fld in ["email","phone"]:
                val = result.get(fld)
                if val:
                    verify = stage2_verify(
                        name, position, county, fld, val, url,
                        f"'{val}' extracted from {url}"
                    )
                    if not verify.get("verified", True):
                        log.info(f"    Stage2 rejected {fld}={val}")
                        result[fld] = None
                    elif verify.get("corrected_value"):
                        result[fld] = verify["corrected_value"]

    # ── Phase 5: Reconcile ───────────────────────────────────────────────────
    if not source_results and not regex_fallback_result:
        log.warning(f"  Nothing found for {name}")
        return {**existing, "_flags": "Not found — manual review needed",
                "_overall_conf": "0", "_sources_used": ""}

    # Inject regex fallback as a low-confidence source if Claude found nothing
    if not source_results and regex_fallback_result:
        log.info(f"  Using regex fallback result")
        source_results = [{
            **regex_fallback_result,
            "confidence": 0.55,
            "is_about_correct_person": True,
            "is_current": True,
        }]

    final = stage3_reconcile(name, position, county, source_results)

    # ── Phase 6: Apply results ───────────────────────────────────────────────
    result = dict(existing)
    flags  = final.get("flags", [])

    for fld, csv_col in [
        ("email","Email"), ("phone","Phone"), ("website","Website"),
        ("office_address","Office_Address"), ("instagram","Instagram"),
        ("twitter_x","Twitter_X"), ("facebook","Facebook"),
    ]:
        fd      = final.get(fld, {})
        new_val = fd.get("value") if isinstance(fd, dict) else None
        conf    = fd.get("confidence", 0) if isinstance(fd, dict) else 0

        # Use split threshold: lower bar for .gov sources
        source_url = fd.get("source_url","") if isinstance(fd, dict) else ""
        threshold  = CONFIDENCE_GOV if is_gov_url(source_url) else CONFIDENCE_OTHER

        if result.get(csv_col,"") in ("Not available","",None) and new_val and conf >= threshold:
            if fld == "email":
                new_val = normalise_email(new_val)
            elif fld == "phone":
                new_val = normalise_phone(new_val)
            result[csv_col] = new_val
            log.info(f"  ✓ {csv_col}: {new_val} (conf={conf:.2f}, threshold={threshold})")
        elif new_val and conf < threshold:
            flags.append(f"Low-conf {fld}={new_val} ({conf:.2f}) — check manually")

    if any(r.get("_outdated") for r in source_results):
        flags.append("Some sources may be outdated — verify name is still current")

    result["_flags"]        = "; ".join(flags) if flags else ""
    result["_overall_conf"] = str(final.get("overall_confidence", 0))
    result["_sources_used"] = "; ".join(r.get("_url","")[:70] for r in source_results[:2])
    return result


# ── CSV runner ────────────────────────────────────────────────────────────────
def run(input_path: str, output_path: str):
    global client
    client = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY)
    load_county_sites()

    checkpoint_path = output_path.replace(".csv", "_checkpoint.json")
    checkpoint: dict = {}
    if Path(checkpoint_path).exists():
        with open(checkpoint_path) as f:
            checkpoint = json.load(f)
        log.info(f"Resuming — {len(checkpoint)} done")

    rows_in: List[dict] = []
    with open(input_path, newline="", encoding="utf-8") as f:
        reader = csv.DictReader(f)
        fieldnames = list(reader.fieldnames or [])
        rows_in = list(reader)

    for extra in ["_flags","_overall_conf","_sources_used"]:
        if extra not in fieldnames:
            fieldnames.append(extra)

    rows_out = []
    total = len(rows_in)

    for i, row in enumerate(rows_in):
        name     = row.get("Name","").strip()
        position = row.get("Position","").strip()
        county   = row.get("County","").strip()

        if not name:
            rows_out.append(row)
            continue

        key = f"{county}|{name}|{position}"
        if key in checkpoint:
            log.info(f"[{i+1}/{total}] Skipping (done): {name}")
            rows_out.append(checkpoint[key])
            continue

        log.info(f"[{i+1}/{total}] Processing: {name}")
        try:
            updated = scrape_official(name, position, county, dict(row))
            checkpoint[key] = dict(updated)
            with open(checkpoint_path, "w") as f:
                json.dump(checkpoint, f, indent=2)
            rows_out.append(updated)
        except KeyboardInterrupt:
            log.info("Interrupted — saving progress")
            with open(output_path, "w", newline="", encoding="utf-8") as f:
                writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
                writer.writeheader()
                writer.writerows(rows_out)
            raise
        except Exception as e:
            log.error(f"Failed on {name}: {e}")
            rows_out.append({**row, "_flags": f"Error: {e}",
                            "_overall_conf":"0", "_sources_used":""})

    with open(output_path, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
        writer.writeheader()
        writer.writerows(rows_out)

    log.info(f"\nDone. Output: {output_path}")


if __name__ == "__main__":
    import argparse
    p = argparse.ArgumentParser()
    p.add_argument("--input",  required=True)
    p.add_argument("--output", required=True)
    args = p.parse_args()
    run(args.input, args.output)