"""JSON-LD and OpenGraph (read with extruct) against Shopify's product JSON on 8 product pages, run twice.
Each store's robots.txt allowed the product page and its .json route. October 3, 2026, from Mumbai."""
import json, time
import httpx, extruct
UA="MyScraperBot/1.0 (https://example.com/bot; you@example.com) httpx/0.28"
c=httpx.Client(headers={"User-Agent":UA},follow_redirects=True,timeout=20)
pages={"allbirds.com":"womens-allbirds-flip-flop-dune-white","colourpop.com":"pencil-sharpener","kyliecosmetics.com":"build-your-own-makeup-bundle",
       "rothys.com":"the-baby-moc","brooklinen.com":"marlow-mini-pillow-last-call","deathwishcoffee.com":"klean-kanteen-camp-mug",
       "beardbrand.com":"fox-hunt-mens-cologne","tentree.com":"morrell-sweater-warm-oak-nep"}
rows=[]
for run in (1,2):
    for s,h in pages.items():
        url=f"https://www.{s}/products/{h}"
        r=c.get(url)
        if r.status_code!=200:
            lst=c.get(f"https://www.{s}/products.json?limit=1").json()["products"][0]["handle"]; pages[s]=h=lst; url=f"https://www.{s}/products/{h}"; r=c.get(url)
        d=extruct.extract(r.text, base_url=str(r.url), syntaxes=["json-ld","opengraph"], uniform=True)
        prods=[x for x in d["json-ld"] for x in (x.get("@graph",[x]))] 
        prods=[x for x in prods if isinstance(x,dict) and x.get("@type") in ("Product","ProductGroup")]
        ld=[]
        for p in prods:
            o=p.get("offers") or []; o=o if isinstance(o,list) else [o]
            ld+=[str(x.get("price")) for x in o if isinstance(x,dict)]
        og=[x.get("product:price:amount") for x in d["opengraph"] if x.get("product:price:amount")]
        j=c.get(url+".json").json()["product"]
        row=(run,s,h,len(prods),sorted(set(ld))[:3],og[:1],len(j["variants"]),sorted({v["price"] for v in j["variants"]})[:3])
        rows.append(row); print(row, flush=True); time.sleep(1)
json.dump(rows,open("shop2.json","w"),indent=1)