#!/usr/bin/env python3
"""Polite, robots.txt-respecting crawler. Saves raw HTML per page."""
import os, re, sys, time, urllib.robotparser
from collections import deque
from urllib.parse import urljoin, urlparse, urldefrag
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError

SEEDS = [
    "https://example.com/",
    "https://www.iana.org/",
    "https://books.toscrape.com/",
    "https://quotes.toscrape.com/",
]
UA = "ClaudeCrawledBS/1.0 (+respecting-robots; contact p2pdojo@gmail.com)"
MAX_PAGES = 50          # per site
MAX_DEPTH = 2           # levels of links to follow
DELAY = 1.0             # seconds between requests to same host
OUT = sys.argv[1] if len(sys.argv) > 1 else "./out"

LINK_RE = re.compile(rb'href=["\']([^"\'#]+)', re.I)

def safe_name(url):
    p = urlparse(url)
    path = p.path
    if path.endswith("/") or path == "":
        path += "index"
    name = (path.lstrip("/") + ("_" + p.query if p.query else ""))
    name = re.sub(r'[^A-Za-z0-9._/-]', '_', name)
    if not name.lower().endswith((".html", ".htm")):
        name += ".html"
    return name

def crawl_site(seed):
    host = urlparse(seed).netloc
    base = f"{urlparse(seed).scheme}://{host}"
    rp = urllib.robotparser.RobotFileParser()
    rp.set_url(urljoin(base, "/robots.txt"))
    try:
        rp.read()
    except Exception as e:
        print(f"  [robots] could not read for {host}: {e} — treating as allow-all")
    delay = rp.crawl_delay(UA) or DELAY

    site_dir = os.path.join(OUT, host)
    os.makedirs(site_dir, exist_ok=True)
    seen, saved = set(), 0
    q = deque([(seed, 0)])
    while q and saved < MAX_PAGES:
        url, depth = q.popleft()
        url, _ = urldefrag(url)
        if url in seen:
            continue
        seen.add(url)
        if not rp.can_fetch(UA, url):
            print(f"  [robots-block] {url}")
            continue
        try:
            req = Request(url, headers={"User-Agent": UA})
            with urlopen(req, timeout=15) as r:
                ctype = r.headers.get("Content-Type", "")
                if "html" not in ctype.lower():
                    continue
                body = r.read(3_000_000)
        except (HTTPError, URLError, Exception) as e:
            print(f"  [err] {url}: {e}")
            continue

        fn = os.path.join(site_dir, safe_name(url))
        os.makedirs(os.path.dirname(fn), exist_ok=True)
        with open(fn, "wb") as f:
            f.write(body)
        saved += 1
        print(f"  [saved {saved:>2}] {url}")

        if depth < MAX_DEPTH:
            for m in LINK_RE.findall(body):
                try:
                    link = urljoin(url, m.decode("utf-8", "ignore"))
                except Exception:
                    continue
                lp = urlparse(link)
                if lp.scheme in ("http", "https") and lp.netloc == host:
                    if link not in seen:
                        q.append((link, depth + 1))
        time.sleep(delay)
    print(f"  => {host}: {saved} pages saved (delay {delay}s)")
    return saved

def main():
    os.makedirs(OUT, exist_ok=True)
    total = 0
    for s in SEEDS:
        print(f"[site] {s}")
        total += crawl_site(s)
    print(f"\nTOTAL: {total} pages across {len(SEEDS)} sites -> {OUT}")

if __name__ == "__main__":
    main()