"""Fetch pages with a user agent that names you, and check robots.txt first.
Run with: uv run --with httpx python polite_fetch.py https://example.com/page"""
import sys
from urllib.parse import urlsplit
from urllib.robotparser import RobotFileParser
import httpx

UA = "ExampleBot/1.0 (+https://example.com/bot; bot@example.com)"  # who you are, and how to contact you
rules = {}  # one robots.txt per site


def allowed(client, url):
    site = "{0.scheme}://{0.netloc}".format(urlsplit(url))
    if site not in rules:
        rp = RobotFileParser()
        r = client.get(f"{site}/robots.txt")
        if r.status_code in (401, 403) or r.status_code >= 500:
            rp.disallow_all = True  # the same rules as Python's urllib.robotparser
        else:
            rp.parse(r.text.splitlines() if r.status_code == 200 else [])
        rules[site] = rp
    return rules[site].can_fetch(UA, url)


with httpx.Client(headers={"User-Agent": UA}, follow_redirects=True, timeout=20) as client:
    for url in sys.argv[1:]:
        if not allowed(client, url):
            print(f"skipped (robots.txt): {url}")
            continue
        print(f"{client.get(url).status_code} {url}")