import httpx
from protego import Protego

headers = {"User-Agent": "MyScraperBot/1.0 (https://example.com/bot; you@example.com) httpx/0.28"}
agents = ["Googlebot", "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "CCBot"]

for site in ["https://medium.com/", "https://www.nytimes.com/"]:
    robots = httpx.get(site + "robots.txt", headers=headers, follow_redirects=True).text
    rules = Protego.parse(robots)
    allowed = [a for a in agents if rules.can_fetch(site, a)]
    print(f"{site:26} allows: {', '.join(allowed)}")