[ { "rank": 1, "domain": "google.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.google.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 2, "domain": "cloudflare.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cloudflare.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ChatGPT-User", "Google-Extended", "PerplexityBot", "CCBot" ], "content_signal": [ "ai-train=yes, search=yes, ai-input=yes" ], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 3, "domain": "facebook.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://facebook.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": true, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "Googlebot", "PerplexityBot", "Applebot-Extended", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 4, "domain": "gstatic.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.gstatic.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 5, "domain": "googleapis.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://googleapis.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 6, "domain": "amazonaws.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://aws.amazon.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 7, "domain": "akamai.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 8, "domain": "microsoft.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.microsoft.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 9, "domain": "youtube.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.youtube.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 10, "domain": "apple.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.apple.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 11, "domain": "instagram.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://www.instagram.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": true, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "Googlebot", "PerplexityBot", "Applebot-Extended", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 12, "domain": "mail.ru", "group": "excluded: browser redirected elsewhere", "robots_status": 200, "robots_final": "https://mail.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 13, "domain": "fbcdn.net", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://fbcdn.net/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "Googlebot", "PerplexityBot", "Applebot-Extended", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 14, "domain": "ezviz7.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 15, "domain": "twitter.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://twitter.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "Google-Extended", "Googlebot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 16, "domain": "domaincontrol.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 17, "domain": "dzen.ru", "group": "partly in the HTML", "robots_status": 200, "robots_final": "https://dzen.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": null, "ai_txt": false, "tdmrep": false }, { "rank": 18, "domain": "linkedin.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://linkedin.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 19, "domain": "googlevideo.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 20, "domain": "googletagmanager.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://googletagmanager.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 21, "domain": "akamaiedge.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 22, "domain": "office.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=4765445b-32c6-49b0-83e6-1d93765276ca&redirect_uri=https%3A%2F%2Fwww.office.com%2Flandingv2&response_type=code%20id_token&scope=openid%20profile%20https%3A%2F%2Fwww.office.com%2Fv2%2FOfficeHome.All&response_mode=form_post&nonce=639274220414406345.ZjhiMmMwNDQtODIzYy00ZjIxLWI4NTMtOTUzZDE4MmE3OTNhZTkxZjQ3ZGMtOGEyMS00ZWQ5LWIyYmYtMjNhNjMwZjFkNjIz&ui_locales=en-US&mkt=en-US&client-request-id=54f104d6-fa78-4b16-8fea-14e4c37232b3&siwa=1&siwg=1&state=BC46S_KdWbVsthTa_h79A5ooZpTcLnJDSeuygh7LXtQr41bCFisqscyelTfeSy7UnsC0WxuK9xKINaH5XHRnHCj52GCGnAJHzyaPBRzEo_3rPqhJhOA_zzt-ebojzBMGA18RwAoqG6ykQTo6UcQr4hz6O1TO_cBeIU7iNqx8-_6VfTs-IjoQgL8kMxxBtGItuGZurYRr0ahLWvNj9czwEMEPMlV7OB_ppFXgNWoTlWTD-gSLGK-KPXaz7tBI470m6dwb2lkqZ2izK1tWEaxwCtoQ_Bkkdzxdgfskye9XtwRhisxAEaDoVsWmbG4-AbhKHJoRHFFjXMlEZBmhVZ-86vng5Q3prGefUXwcIyT84a406hf9iCmUs_aY_ja_NuUzMz2Lef_PjEuEkL833_2yb-V2mM8_zFzE7gM1F6pm0yyryBD6pzvocvNGxwJgLhB_41hScHrVApVMn3sqcaiyTG-PPfV-7cIZoMRLfYi3UIHcaa_Vp5v__NZLRVXC7ES2Cf3bxGmz-CxlXOCMABtmUZ8e9h4OkbtrBxdrr08UmXA&x-client-SKU=ID_NET8_0&x-client-ver=8.16.0.0", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 23, "domain": "live.com", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://outlook.live.com/mail/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 24, "domain": "hicloudcam.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 25, "domain": "akadns.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 26, "domain": "amazon.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 27, "domain": "gtld-servers.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 28, "domain": "azure.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://azure.microsoft.com/en-us", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 29, "domain": "github.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://github.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Bytespider" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 30, "domain": "wikipedia.org", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://en.wikipedia.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 31, "domain": "appsflyersdk.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://appsflyersdk.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 32, "domain": "whatsapp.net", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.whatsapp.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": false, "Bytespider": true, "Amazonbot": false, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "PerplexityBot", "Applebot-Extended", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 33, "domain": "bing.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.bing.com/robots.txt?toWww=1&redig=EE16826414F642C5BF33B55664D570DD", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 34, "domain": "apple-dns.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 35, "domain": "fastly.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.fastly.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 36, "domain": "aaplimg.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 37, "domain": "googleusercontent.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://googleusercontent.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 38, "domain": "doubleclick.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://doubleclick.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 39, "domain": "cloud.microsoft", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://m365.cloud.microsoft/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 40, "domain": "microsoftonline.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 41, "domain": "office.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 42, "domain": "trafficmanager.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 43, "domain": "netflix.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.netflix.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 44, "domain": "sharepoint.com", "group": "in the plain HTML", "robots_error": "ReadTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": false }, { "rank": 45, "domain": "gandi.net", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 46, "domain": "youtu.be", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://youtu.be/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 47, "domain": "cloudfront.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 48, "domain": "digicert.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.digicert.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 49, "domain": "wordpress.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://wordpress.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 50, "domain": "x.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://x.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "Google-Extended", "Googlebot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 51, "domain": "skype.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://teams.live.com/free?source=skype", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 52, "domain": "icloud.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.icloud.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 53, "domain": "pinterest.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.pinterest.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 54, "domain": "tiktok.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 55, "domain": "whatsapp.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.whatsapp.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": false, "Bytespider": true, "Amazonbot": false, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "PerplexityBot", "Applebot-Extended", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 56, "domain": "goo.gl", "group": "excluded: no real homepage", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 57, "domain": "cloudflare.net", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://cloudflare.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 58, "domain": "googledomains.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://domains.google/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 59, "domain": "yahoo.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.yahoo.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Google-Extended", "PerplexityBot", "CCBot", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 60, "domain": "googlesyndication.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://googlesyndication.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 61, "domain": "roblox.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.roblox.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 62, "domain": "gvt1.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 63, "domain": "pv-cdn.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 64, "domain": "endpoints.news", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://endpoints.news/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 65, "domain": "windows.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 66, "domain": "spotify.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.spotify.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 67, "domain": "windowsupdate.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 68, "domain": "adobe.com", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://www.adobe.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 69, "domain": "wa.me", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://wa.me/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 70, "domain": "myfritz.net", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://myfritz.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 71, "domain": "msn.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.msn.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 72, "domain": "ntp.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.ntp.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 73, "domain": "qq.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 74, "domain": "zoom.us", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://zoom.us/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 75, "domain": "root-servers.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 76, "domain": "nginx.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://nginx.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 77, "domain": "akam.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 78, "domain": "tiktokcdn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 79, "domain": "edgekey.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 80, "domain": "workers.dev", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cloudflare.com/products/workers/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 81, "domain": "baidu.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 82, "domain": "chatgpt.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://chatgpt.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Google-Extended", "PerplexityBot", "CCBot", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 83, "domain": "cloudflare-dns.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://one.one.one.one/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 84, "domain": "tiktokv.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://tiktokv.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 85, "domain": "yandex.net", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://yandex.net/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 86, "domain": "cdninstagram.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 87, "domain": "vimeo.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://vimeo.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 88, "domain": "openai.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://openai.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 89, "domain": "gvt2.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 90, "domain": "opera.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.opera.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 91, "domain": "f5.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.f5.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 92, "domain": "gwfb.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 93, "domain": "sentry.io", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://sentry.io/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [ "search=yes, ai-input=yes, ai-train=yes" ], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 94, "domain": "ytimg.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 95, "domain": "a2z.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 96, "domain": "nginx.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.f5.com/products/nginx", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 97, "domain": "windows.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.microsoft.com/en-in/windows/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 98, "domain": "samsung.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.samsung.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "Google-Extended", "Googlebot", "PerplexityBot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 99, "domain": "highwebmedia.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 100, "domain": "mozilla.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.mozilla.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 101, "domain": "yandex.ru", "group": "needs JavaScript", "robots_error": "ConnectError", "llms_txt": false, "ai_txt": null, "tdmrep": false }, { "rank": 102, "domain": "wordpress.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://wordpress.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 103, "domain": "t.me", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://t.me/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 104, "domain": "blogspot.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.blogger.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 105, "domain": "yuga.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://yuga.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 106, "domain": "dellemc.com", "group": "in the plain HTML", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 107, "domain": "actify.nl", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.zilverenkruis.nl/gezonder-leven/actify/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 108, "domain": "b-cdn.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://bunny.net", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 109, "domain": "reddit.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.reddit.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 110, "domain": "yccdn.ru", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 111, "domain": "edgesuite.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 112, "domain": "ui.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://ui.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 113, "domain": "nic.ru", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://nic.ru/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 114, "domain": "office365.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://office365.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 115, "domain": "aws.dev", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 116, "domain": "europa.eu", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://europa.eu/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 117, "domain": "bit.ly", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://bit.ly/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 118, "domain": "amazonvideo.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://www.primevideo.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "ChatGPT-User", "Claude-User", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 119, "domain": "google-analytics.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://google-analytics.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 120, "domain": "okcdn.ru", "group": "blocked for both", "robots_status": 404, "robots_final": "https://st.okcdn.ru/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 121, "domain": "outlook.com", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://outlook.live.com/mail/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 122, "domain": "registrar-servers.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://registrar-servers.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 123, "domain": "app-analytics-services.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://app-analytics-services.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 124, "domain": "trbcdn.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 125, "domain": "epicgames.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.epicgames.com/site/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 126, "domain": "cdn77.org", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 127, "domain": "github.io", "group": "excluded: browser redirected elsewhere", "robots_status": 200, "robots_final": "https://pages.github.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 128, "domain": "dnsowl.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 129, "domain": "app-measurement.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://app-measurement.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 130, "domain": "snapchat.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.snapchat.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 131, "domain": "triplinkintl.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.triplinkintl.com", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 132, "domain": "apache.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://apache.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 133, "domain": "reg.ru", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://reg.ru/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 134, "domain": "amazon.dev", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 135, "domain": "dropbox.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.dropbox.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 136, "domain": "vk.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://vk.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 137, "domain": "intuit.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://www.intuit.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 138, "domain": "amazon-adsystem.com", "group": "excluded: unreachable", "robots_error": "RemoteProtocolError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 139, "domain": "criteo.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://criteo.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 140, "domain": "unity3d.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://unity.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Bytespider" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 141, "domain": "nih.gov", "group": "blocked for both", "robots_status": 403, "robots_final": "https://www.nih.gov/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 142, "domain": "discord.gg", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://discord.gg/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 143, "domain": "kaspersky.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.kaspersky.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 144, "domain": "3gppnetwork.org", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 145, "domain": "static.microsoft", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 146, "domain": "forms.gle", "group": "excluded: no real homepage", "robots_status": 400, "robots_final": "https://forms.gle/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 147, "domain": "archive.org", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://archive.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 148, "domain": "aliyuncs.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://aliyuncs.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 149, "domain": "azurefd.net", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 150, "domain": "godaddy.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://www.godaddy.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 151, "domain": "afternic.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 152, "domain": "iiko.it", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 153, "domain": "msftncsi.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 154, "domain": "gravatar.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://gravatar.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": true }, "named": [ "GPTBot", "Google-Extended", "CCBot", "Bytespider", "Amazonbot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 155, "domain": "tumblr.com", "group": "blocked for plain HTTP only", "robots_error": "RemoteProtocolError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 156, "domain": "dns-parking.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 157, "domain": "nytimes.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.nytimes.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 158, "domain": "xiaomi.com", "group": "blocked for the browser only", "robots_status": 403, "robots_final": "https://www.mi.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 159, "domain": "jomodns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 160, "domain": "dns.google", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://dns.google/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 161, "domain": "tm-azurefd.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 162, "domain": "steamserver.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 163, "domain": "lencr.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://letsencrypt.org/docs/lencr.org/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 164, "domain": "mts.ru", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://moskva.mts.ru/robotstxt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 165, "domain": "adobe.io", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://developer.adobe.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 166, "domain": "hichina.com", "group": "in the plain HTML", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 167, "domain": "msftconnecttest.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 168, "domain": "webex.com", "group": "blocked for the browser only", "robots_status": 403, "robots_final": "https://www.webex.com/robots.txt", "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 169, "domain": "discord.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://discord.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 170, "domain": "example.com", "group": "needs JavaScript", "robots_status": 404, "robots_final": "https://example.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 171, "domain": "applovin.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://applovin.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 172, "domain": "adtrafficquality.google", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://adtrafficquality.google/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 173, "domain": "one.one", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.one.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 174, "domain": "paypal.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.paypal.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 175, "domain": "spectrum.com", "group": "excluded: browser failed to load", "robots_error": "ReadTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 176, "domain": "ax-msedge.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 177, "domain": "shopify.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://shopify.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 178, "domain": "vkuserphoto.ru", "group": "excluded: no real homepage", "robots_status": 418, "robots_final": "https://ps.userapi.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 179, "domain": "azurewebsites.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 180, "domain": "vk.ru", "group": "blocked for both", "robots_status": 200, "robots_final": "https://vk.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 181, "domain": "vedcdnlb.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 182, "domain": "base.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.base.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 183, "domain": "hosting24.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.hostinger.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 184, "domain": "macromedia.com", "group": "excluded: browser failed to load", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 185, "domain": "doctolib.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 186, "domain": "paytmfirstgames.com", "group": "in the plain HTML", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 187, "domain": "nflxso.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 188, "domain": "w3.org", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.w3.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 189, "domain": "yandexcloud.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 190, "domain": "flickr.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://flickr.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "OAI-SearchBot", "ChatGPT-User", "Claude-SearchBot", "Claude-User", "Googlebot", "PerplexityBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 191, "domain": "fastly-edge.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 192, "domain": "miit.gov.cn", "group": "in the plain HTML", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 193, "domain": "soundcloud.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://soundcloud.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 194, "domain": "googleadservices.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://googleadservices.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 195, "domain": "namebrightdns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 196, "domain": "ggpht.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 197, "domain": "shiabank.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 198, "domain": "duckdns.org", "group": "excluded: no real homepage", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 199, "domain": "telekom.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.telekom.com", "llms_txt": null, "ai_txt": false, "tdmrep": null }, { "rank": 200, "domain": "creativecommons.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://creativecommons.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 201, "domain": "cdn-apple.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 202, "domain": "lsrelayaccess.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 203, "domain": "medium.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://medium.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 204, "domain": "bytefcdn-oversea.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 205, "domain": "akamaized.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 206, "domain": "cnn.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cnn.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 207, "domain": "theguardian.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.theguardian.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 208, "domain": "ebay.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://www.ebay.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Googlebot", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 209, "domain": "t-online.de", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.t-online.de/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 210, "domain": "pki.goog", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://pki.goog/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 211, "domain": "ailawandorder.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 212, "domain": "adsafeprotected.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://adsafeprotected.com/skeleton.gif", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": true, "tdmrep": false }, { "rank": 213, "domain": "oracle.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://www.oracle.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 214, "domain": "youtube-nocookie.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 215, "domain": "shifen.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 216, "domain": "ttdns2.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 217, "domain": "ksyuncdn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 218, "domain": "doi.org", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://www.doi.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 219, "domain": "android.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.android.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "Claude-SearchBot", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 220, "domain": "rubiconproject.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.rubiconproject.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 221, "domain": "forbes.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.forbes.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 222, "domain": "bbc.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.bbc.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 223, "domain": "ubuntu.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://ubuntu.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Google-Extended", "PerplexityBot", "Perplexity-User", "Applebot-Extended", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 224, "domain": "2mdn.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 225, "domain": "vungle.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://vungle.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 226, "domain": "sciencedirect.com", "group": "excluded: no real homepage", "robots_status": 403, "robots_final": "https://www.sciencedirect.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 227, "domain": "cdngslb.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 228, "domain": "twitch.tv", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.twitch.tv/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 229, "domain": "sfx.ms", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 230, "domain": "researchgate.net", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.researchgate.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 231, "domain": "mit.edu", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://web.mit.edu/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 232, "domain": "taboola.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.taboola.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 233, "domain": "ozon.ru", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.ozon.ru/robots.txt?__rr=1", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 234, "domain": "pages.dev", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cloudflare.com/products/pages/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 235, "domain": "cisco.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cisco.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "Googlebot", "CCBot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 236, "domain": "bbc.co.uk", "group": "excluded: browser redirected elsewhere", "robots_status": 200, "robots_final": "https://www.bbc.co.uk/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 237, "domain": "azureedge.net", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 238, "domain": "comcast.net", "group": "blocked for the browser only", "robots_status": 403, "robots_final": "https://www.xfinity.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 239, "domain": "nist.gov", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.nist.gov/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 240, "domain": "userapi.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://vk.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 241, "domain": "stripe.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://stripe.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 242, "domain": "capcutapi.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 243, "domain": "telegram.org", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://telegram.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 244, "domain": "t.co", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://t.co/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 245, "domain": "rbxcdn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 246, "domain": "scorecardresearch.com", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://scorecardresearch.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 247, "domain": "canva.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.canva.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 248, "domain": "alidns.com", "group": "excluded: browser redirected elsewhere", "robots_status": 200, "robots_final": "https://alidns.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": true, "tdmrep": true }, { "rank": 249, "domain": "cpanel.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cpanel.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 250, "domain": "sourceforge.net", "group": "blocked for both", "robots_status": 200, "robots_final": "https://sourceforge.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 251, "domain": "meraki.com", "group": "excluded: robots.txt disallows", "robots_status": 401, "robots_final": "https://meraki.cisco.com/robots.txt", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 252, "domain": "wsdvs.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 253, "domain": "gmail.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://gmail.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 254, "domain": "wikimedia.org", "group": "blocked for plain HTTP only", "robots_status": 404, "robots_final": "https://www.wikimedia.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 255, "domain": "playstation.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 256, "domain": "ipify.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.ipify.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 257, "domain": "omtrdc.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 258, "domain": "media-amazon.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 259, "domain": "pubmatic.com", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://pubmatic.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 260, "domain": "roku.com", "group": "partly in the HTML", "robots_status": 200, "robots_final": "https://www.roku.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 261, "domain": "imcmdb.net", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 262, "domain": "miui.com", "group": "blocked for plain HTTP only", "robots_status": 404, "robots_final": "https://miui.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 263, "domain": "hubspot.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.hubspot.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 264, "domain": "spov-msedge.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 265, "domain": "nr-data.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 266, "domain": "who.int", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.who.int/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "CCBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 267, "domain": "adnxs.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://adnxs.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 268, "domain": "ripn.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 269, "domain": "imdb.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.imdb.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "Googlebot", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 270, "domain": "statsigapi.net", "group": "blocked for both", "robots_status": 403, "robots_final": "https://statsigapi.net/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 271, "domain": "alibabadns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 272, "domain": "linktr.ee", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://linktr.ee/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Amazonbot" ], "content_signal": [ "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes", "ai-train=no, search=yes" ], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 273, "domain": "hcaptcha.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.hcaptcha.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 274, "domain": "telecid.ru", "group": "blocked for both", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 275, "domain": "wixsite.com", "group": "in the plain HTML", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 276, "domain": "forter.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://forter.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [ "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes", "search=no, ai-input=no, ai-train=no", "search=yes, ai-input=yes, ai-train=no" ], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 277, "domain": "name-services.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 278, "domain": "adsrvr.org", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://adsrvr.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 279, "domain": "crashlytics.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://firebase.google.com/products/crashlytics/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 280, "domain": "ttvnw.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 281, "domain": "booking.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://booking.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 282, "domain": "samsungcloud.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.samsungcloud.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 283, "domain": "namecheap.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.namecheap.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 284, "domain": "launchpad.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://launchpad.net/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [ "ai-train=no, search=yes, ai-input=no" ], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 285, "domain": "amon.tech", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 286, "domain": "ibm.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.ibm.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 287, "domain": "aliyuncsslbintl.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 288, "domain": "msedge.net", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 289, "domain": "mtgglobals.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 290, "domain": "appsflyer.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.appsflyer.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 291, "domain": "mobile.de", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.mobile.de/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 292, "domain": "reuters.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.reuters.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "OAI-SearchBot", "ChatGPT-User", "Googlebot", "Applebot-Extended" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 293, "domain": "facebook.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 294, "domain": "mangosip.ru", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 295, "domain": "dnsmadeeasy.com", "group": "blocked for the browser only", "robots_status": 404, "robots_final": "https://dnsmadeeasy.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 296, "domain": "salesforce.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://www.salesforce.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 297, "domain": "mzstatic.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.apple.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 298, "domain": "springer.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.springer.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": true }, { "rank": 299, "domain": "telegram.me", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://telegram.org/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 300, "domain": "tinyurl.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://tinyurl.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 301, "domain": "hp.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.hp.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 302, "domain": "autodesk.com", "group": "blocked for the browser only", "robots_status": 403, "robots_final": "https://www.autodesk.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 303, "domain": "tiktokcdn-us.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 304, "domain": "nature.com", "group": "in the plain HTML", "robots_status": 406, "robots_final": "https://www.nature.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": true }, { "rank": 305, "domain": "googleblog.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://blog.google/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 306, "domain": "www.gov.uk", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.gov.uk/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 307, "domain": "steampowered.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://store.steampowered.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 308, "domain": "byteoversea.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 309, "domain": "inmobi.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://www.inmobi.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 310, "domain": "jsdelivr.net", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 311, "domain": "weibo.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 312, "domain": "amplitude.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://amplitude.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "Googlebot", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [ "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=yes, ai-input=yes, ai-train=yes", "search=no, ai-input=no, ai-train=no" ], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 313, "domain": "ok.ru", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 314, "domain": "debian.org", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://www.debian.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 315, "domain": "alibaba.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.alibaba.com/atlassitemapsitenet/static/www_alibaba_com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "Google-Extended", "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 316, "domain": "harvard.edu", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.harvard.edu/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 317, "domain": "casalemedia.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.indexexchange.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 318, "domain": "avast.com", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://www.avast.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 319, "domain": "clarity.ms", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 320, "domain": "wac-msedge.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 321, "domain": "weather.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://weather.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": true }, "named": [ "ChatGPT-User", "CCBot", "Bytespider", "Amazonbot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 322, "domain": "dell.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.dell.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Bytespider" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 323, "domain": "drom.ru", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.drom.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 324, "domain": "douyincdn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 325, "domain": "crpt.ru", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://crpt.ru/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 326, "domain": "spaceweb.pro", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 327, "domain": "awswaf.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 328, "domain": "heytapmobile.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 329, "domain": "wiley.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.wiley.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 330, "domain": "samsungqbe.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 331, "domain": "php.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.php.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 332, "domain": "shalltry.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://shalltry.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 333, "domain": "aliyun.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://www.aliyun.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 334, "domain": "etsy.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.etsy.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 335, "domain": "licdn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 336, "domain": "hostinger.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.hostinger.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [ "search=yes, ai-input=yes, ai-train=no", "search=yes, ai-input=yes, ai-train=no" ], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 337, "domain": "skybridge.click", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://skybridge.click/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 338, "domain": "amazon.co.uk", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.co.uk/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 339, "domain": "twc.com", "group": "excluded: browser failed to load", "robots_error": "ReadTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 340, "domain": "akamaihd.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 341, "domain": "trueconf.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 342, "domain": "mi.com", "group": "blocked for the browser only", "robots_status": 403, "robots_final": "https://www.mi.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 343, "domain": "media.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.media.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 344, "domain": "name.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.name.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": false }, "named": [ "meta-externalagent" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 345, "domain": "smartadserver.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.equativ.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 346, "domain": "bilibili.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.bilibili.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 347, "domain": "gnu.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.gnu.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 348, "domain": "weebly.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.weebly.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 349, "domain": "mozilla.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.mozilla.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 350, "domain": "tbcache.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 351, "domain": "ip-api.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://ip-api.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 352, "domain": "arubanetworks.com", "group": "blocked for the browser only", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 353, "domain": "doubleverify.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://doubleverify.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 354, "domain": "myshopify.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://myshopify.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 355, "domain": "ea.com", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://www.ea.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 356, "domain": "naver.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.naver.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 357, "domain": "slack.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://slack.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 358, "domain": "trustpilot.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.trustpilot.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": false, "CCBot": true, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 359, "domain": "checkpoint.com", "group": "in the plain HTML", "robots_status": 202, "robots_final": "https://checkpoint.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 360, "domain": "worldnic.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 361, "domain": "onelink.me", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.appsflyer.com/products/deep-linking/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 362, "domain": "dnspod.net", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 363, "domain": "selectel.ru", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://selectel.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 364, "domain": "cdc.gov", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cdc.gov/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 365, "domain": "un.org", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://maintenance.un.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 366, "domain": "nflximg.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 367, "domain": "washingtonpost.com", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://www.washingtonpost.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "ClaudeBot", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": null, "ai_txt": null, "tdmrep": false }, { "rank": 368, "domain": "inner-active.mobi", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 369, "domain": "opendns.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.opendns.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 370, "domain": "allawnos.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 371, "domain": "tradingview.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.tradingview.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 372, "domain": "gitlab.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://gitlab.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 373, "domain": "hostgator.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.hostgator.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 374, "domain": "zendesk.com", "group": "in the page source (JSON)", "robots_status": 403, "robots_final": "https://www.zendesk.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 375, "domain": "cookiedatabase.org", "group": "blocked for both", "robots_status": 403, "robots_final": "https://cookiedatabase.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 376, "domain": "nasa.gov", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.nasa.gov/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 377, "domain": "nvidia.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.nvidia.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [ "ai-train=yes, search=yes, ai-input=yes" ], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 378, "domain": "tp-link.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.tp-link.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 379, "domain": "live-video.net", "group": "excluded: no real homepage", "robots_status": 403, "robots_final": "https://live-video.net/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 380, "domain": "issuu.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://issuu.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 381, "domain": "volcfcdndvs.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 382, "domain": "cdnbuild.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 383, "domain": "vecdnlb.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 384, "domain": "netangels.ru", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.netangels.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 385, "domain": "bloomberg.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.bloomberg.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 386, "domain": "stanford.edu", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.stanford.edu/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 387, "domain": "wsj.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.wsj.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 388, "domain": "datadoghq.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.datadoghq.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 389, "domain": "samsungcloudsolution.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 390, "domain": "plesk.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.plesk.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 391, "domain": "xcal.tv", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 392, "domain": "calendly.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://calendly.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "CCBot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 393, "domain": "virginm.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 394, "domain": "vkuser.net", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 395, "domain": "tiktokv.us", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 396, "domain": "yandex.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://yandex.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 397, "domain": "amazontrust.com", "group": "excluded: no real homepage", "robots_error": "ReadTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 398, "domain": "gandi-ns.fr", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 399, "domain": "avcdn.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 400, "domain": "amazon.de", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.de/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 401, "domain": "nflxvideo.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 402, "domain": "bsky.app", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://bsky.app/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 403, "domain": "online-metrix.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 404, "domain": "espn.com", "group": "excluded: browser redirected elsewhere", "robots_status": 200, "robots_final": "https://www.espn.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ChatGPT-User", "Google-Extended", "CCBot", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 405, "domain": "ibyteimg.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 406, "domain": "shopifysvc.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 407, "domain": "expireddomains.com", "group": "blocked for plain HTTP only", "robots_status": 404, "robots_final": "https://expireddomains.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 408, "domain": "quickconnect.to", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://quickconnect.to/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": null }, { "rank": 409, "domain": "demdex.net", "group": "excluded: browser failed to load", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 410, "domain": "indeed.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.indeed.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 411, "domain": "kwai-pro.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 412, "domain": "globo.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://www.globo.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 413, "domain": "unsplash.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://unsplash.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 414, "domain": "163.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "http://www.163.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 415, "domain": "rocket-cdn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 416, "domain": "brave.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://brave.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 417, "domain": "timeweb.ru", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://timeweb.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 418, "domain": "line.me", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://line.me/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 419, "domain": "aliexpress.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 420, "domain": "heytapdl.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 421, "domain": "dropcatch.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.dropcatch.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 422, "domain": "gamepass.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 423, "domain": "beian.gov.cn", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 424, "domain": "go.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.go.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "Google-Extended" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 425, "domain": "netease.com", "group": "in the plain HTML", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 426, "domain": "vivoglobal.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://vivoglobal.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 427, "domain": "chinamobile.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 428, "domain": "claude.ai", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://claude.ai/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "Google-Extended" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 429, "domain": "dailymotion.com", "group": "needs JavaScript", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 430, "domain": "steamcommunity.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://steamcommunity.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 431, "domain": "tiktokv.eu", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 432, "domain": "google.cn", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://google.cn/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 433, "domain": "tiktokcdn-eu.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 434, "domain": "temu.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.temu.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 435, "domain": "skyhigh.cloud", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://www.skyhighsecurity.com", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 436, "domain": "huawei.com", "group": "in the plain HTML", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 437, "domain": "supercell.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://supercell.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 438, "domain": "kaspersky-labs.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 439, "domain": "intel.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.intel.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 440, "domain": "pangle.io", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 441, "domain": "taobao.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 442, "domain": "dotaplabs.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 443, "domain": "whecloud.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 444, "domain": "paloaltonetworks.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.paloaltonetworks.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "OAI-SearchBot", "ChatGPT-User", "Claude-SearchBot", "Claude-User", "PerplexityBot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 445, "domain": "tencent-cloud.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 446, "domain": "kwaipros.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 447, "domain": "awsglobalaccelerator.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 448, "domain": "duckduckgo.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://duckduckgo.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 449, "domain": "substack.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://substack.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 450, "domain": "amazonalexa.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.amazon.com/b?node=210710065011", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 451, "domain": "walmart.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.walmart.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 452, "domain": "myqcloud.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 453, "domain": "cloudns.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cloudns.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 454, "domain": "yahoo.co.jp", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://www.yahoo.co.jp/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "Google-Extended" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 455, "domain": "adobe.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 456, "domain": "businessinsider.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.businessinsider.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "ClaudeBot", "CCBot", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 457, "domain": "duolingo.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.duolingo.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 458, "domain": "aol.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.aol.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot", "CCBot", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 459, "domain": "ezvizlife.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.ezviz.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 460, "domain": "pixabay.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://pixabay.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 461, "domain": "g.page", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.google.com/search?q=robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 462, "domain": "edgecdn.ru", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 463, "domain": "cloudflareinsights.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://cloudflareinsights.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 464, "domain": "goodreads.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.goodreads.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "CCBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 465, "domain": "bluehost.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.bluehost.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 466, "domain": "vercel-dns-016.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 467, "domain": "gcdn.co", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 468, "domain": "speedtest.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.speedtest.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 469, "domain": "arxiv.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://arxiv.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 470, "domain": "byteglb.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 471, "domain": "firetvcaptiveportal.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 472, "domain": "edgcdn.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 473, "domain": "wp.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://wordpress.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 474, "domain": "eye4.cn", "group": "excluded: browser redirected elsewhere", "robots_status": 500, "robots_final": "https://eye4.cn/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 475, "domain": "ipinfo.io", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://ipinfo.io/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 476, "domain": "mikrotik.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://mikrotik.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 477, "domain": "prodregistryv2.org", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://prodregistryv2.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 478, "domain": "wbx2.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 479, "domain": "avsxappcaptiveportal.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 480, "domain": "sina.com.cn", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 481, "domain": "sohu.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.sohu.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 482, "domain": "enacdn.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 483, "domain": "dual-s-msedge.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 484, "domain": "cnbc.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cnbc.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 485, "domain": "scdn.co", "group": "blocked for both", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 486, "domain": "kwai.net", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 487, "domain": "statista.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.statista.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 488, "domain": "okta.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.okta.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 489, "domain": "bdydns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 490, "domain": "bitdefender.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.bitdefender.com/en-in/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 491, "domain": "markmonitor.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://markmonitor.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 492, "domain": "browser-intake-datadoghq.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://browser-intake-datadoghq.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 493, "domain": "ietf.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.ietf.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 494, "domain": "nease.net", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 495, "domain": "list-manage.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 496, "domain": "quora.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.quora.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Applebot-Extended", "Bytespider", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 497, "domain": "amazon.co.jp", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.co.jp/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 498, "domain": "addtoany.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.addtoany.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 499, "domain": "mcafee.com", "group": "excluded: browser failed to load", "robots_error": "ReadTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 500, "domain": "sc-gw.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 501, "domain": "launchdarkly.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://launchdarkly.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 502, "domain": "ivi.ru", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://www.ivi.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 503, "domain": "h264.io", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 504, "domain": "shein.com", "group": "excluded: unreachable", "robots_status": 200, "robots_final": "https://www.shein.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": null }, { "rank": 505, "domain": "impervadns.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 506, "domain": "npr.org", "group": "in the plain HTML", "robots_error": "ReadTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 507, "domain": "dynatrace.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.dynatrace.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 508, "domain": "run.app", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://run.app/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 509, "domain": "mediatek.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.mediatek.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 510, "domain": "gosuslugi.ru", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.gosuslugi.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 511, "domain": "spo-msedge.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 512, "domain": "newrelic.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://newrelic.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 513, "domain": "docker.com", "group": "excluded: browser failed to load", "robots_status": 403, "robots_final": "https://docker.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 514, "domain": "adriver.ru", "group": "excluded: browser failed to load", "robots_status": 404, "robots_final": "https://adriver.ru/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 515, "domain": "atlassian.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://atlassian.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 516, "domain": "scribd.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.scribd.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "ClaudeBot", "CCBot", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 517, "domain": "amzn.to", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://amzn.to/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 518, "domain": "behance.net", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.behance.net/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 519, "domain": "stackoverflow.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://stackoverflow.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 520, "domain": "indiatimes.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.indiatimes.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": false }, "named": [ "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 521, "domain": "google.de", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.google.de/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 522, "domain": "fandom.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.fandom.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 523, "domain": "foxnews.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.foxnews.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 524, "domain": "wildberries.ru", "group": "excluded: no real homepage", "robots_status": 498, "robots_final": "https://www.wildberries.ru/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 525, "domain": "amazon.co.za", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.co.za/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 526, "domain": "eventbrite.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://eventbrite.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "CCBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 527, "domain": "mysql.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.mysql.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 528, "domain": "noaa.gov", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://www.noaa.gov/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 529, "domain": "ui-dns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 530, "domain": "tplinkcloud.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.tplinkcloud.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 531, "domain": "elasticbeanstalk.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 532, "domain": "exp-tas.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 533, "domain": "netflix.net", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.netflix.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 534, "domain": "ipv4only.arpa", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 535, "domain": "on.aws", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 536, "domain": "ngenix.net", "group": "excluded: no real homepage", "robots_status": 503, "robots_final": "https://ngenix.net/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 537, "domain": "share.google", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://share.google/error", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 538, "domain": "slideshare.net", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.slideshare.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "ClaudeBot", "CCBot", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 539, "domain": "ampproject.org", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://amp.dev/robots.txt/?referrer=ampproject.org", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 540, "domain": "wswebcdn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 541, "domain": "squarespacedns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 542, "domain": "adobedtm.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 543, "domain": "capcut.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.capcut.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 544, "domain": "firefox-portal-detection.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://firefox-portal-detection.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 545, "domain": "bitrix24.ru", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.bitrix24.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 546, "domain": "oup.com", "group": "blocked for plain HTTP only", "robots_status": 202, "robots_final": "http://global.oup.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 547, "domain": "gotinder.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://www.gotinder.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 548, "domain": "trendmicro.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.trendmicro.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [ "search=yes, ai-input=yes, ai-train=no", "search=yes, ai-input=yes, ai-train=no" ], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 549, "domain": "azure-dns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 550, "domain": "squarespace.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.squarespace.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 551, "domain": "google.com.hk", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.google.com.hk/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 552, "domain": "libp2p.direct", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 553, "domain": "discord.media", "group": "excluded: no real homepage", "robots_status": 520, "robots_final": "https://discord.media/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 554, "domain": "ft.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.ft.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "ClaudeBot", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 555, "domain": "visualstudio.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://visualstudio.microsoft.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 556, "domain": "blogger.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.blogger.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 557, "domain": "firefox.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.firefox.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 558, "domain": "openx.net", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://openx.net/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 559, "domain": "adobedc.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 560, "domain": "icloud-content.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://developer.apple.com/icloud/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 561, "domain": "wired.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.wired.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 562, "domain": "fast.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://fast.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 563, "domain": "digitalocean.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.digitalocean.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 564, "domain": "jquery.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://jquery.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 565, "domain": "grammarly.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.grammarly.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 566, "domain": "nstld.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 567, "domain": "myhuaweicloud.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 568, "domain": "vidaahub.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 569, "domain": "rakuten.co.jp", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.rakuten.co.jp/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 570, "domain": "tandfonline.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.tandfonline.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 571, "domain": "live.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 572, "domain": "openstreetmap.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.openstreetmap.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 573, "domain": "time.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://time.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 574, "domain": "uol.com.br", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.uol.com.br/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 575, "domain": "translate.goog", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://translate.goog/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 576, "domain": "eset.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.eset.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": null, "ai_txt": false, "tdmrep": false }, { "rank": 577, "domain": "deviantart.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.deviantart.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 578, "domain": "shop.app", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://shop.app/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 579, "domain": "cornell.edu", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cornell.edu/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 580, "domain": "akamaitech.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 581, "domain": "oraclecloud.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://www.oracle.com/cloud/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 582, "domain": "360yield.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://ssp.improvedigital.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 583, "domain": "qlivecdn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 584, "domain": "vercel-dns-3.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 585, "domain": "free.fr", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://www.free.fr/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 586, "domain": "googletagservices.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 587, "domain": "palmplaystore.com", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://palmplaystore.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 588, "domain": "hicloud.com", "group": "in the plain HTML", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 589, "domain": "svc.ms", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 590, "domain": "aboutads.info", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://youradchoices.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 591, "domain": "share-dns.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 592, "domain": "teamviewer.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.teamviewer.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 593, "domain": "yahoodns.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 594, "domain": "dreamhost.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.dreamhost.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 595, "domain": "sberbank.ru", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 596, "domain": "t-msedge.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 597, "domain": "usatoday.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.usatoday.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 598, "domain": "bx-msedge.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 599, "domain": "tripadvisor.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.tripadvisor.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 600, "domain": "my.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 601, "domain": "ca.gov", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.ca.gov/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 602, "domain": "cdn20.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 603, "domain": "target.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.target.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 604, "domain": "braze.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://braze.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 605, "domain": "klaviyo.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.klaviyo.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User", "meta-externalagent" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 606, "domain": "3lift.com", "group": "excluded: browser redirected elsewhere", "robots_status": 403, "robots_final": "https://3lift.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 607, "domain": "e2ro.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 608, "domain": "playstation.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.playstation.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 609, "domain": "threads.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.threads.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "Googlebot", "PerplexityBot", "Applebot-Extended", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 610, "domain": "rlcdn.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 611, "domain": "hotjar.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.hotjar.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 612, "domain": "imgsmail.ru", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://imgsmail.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 613, "domain": "avito.ru", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://www.avito.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 614, "domain": "anthropic.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.anthropic.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 615, "domain": "branch.io", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.branch.io/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 616, "domain": "ieee.org", "group": "excluded: no real homepage", "robots_status": 202, "robots_final": "https://www.ieee.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 617, "domain": "comcast.com", "group": "blocked for the browser only", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 618, "domain": "xiaomi.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 619, "domain": "berkeley.edu", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.berkeley.edu/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 620, "domain": "mdpi.com", "group": "blocked for the browser only", "robots_status": 403, "robots_final": "https://mdpi.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 621, "domain": "vercel-dns-017.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 622, "domain": "ovh.net", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://proof.ovh.net", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 623, "domain": "atlassian.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.atlassian.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 624, "domain": "supertms.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 625, "domain": "telegraph.co.uk", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.telegraph.co.uk/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 626, "domain": "amazon.fr", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.fr/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 627, "domain": "onetrust.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.onetrust.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 628, "domain": "unesco.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.unesco.org/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [ "search=yes,ai-input=yes,ai-train=no,use=reference" ], "llms_txt": true, "ai_txt": false, "tdmrep": true }, { "rank": 629, "domain": "nintendo.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.nintendo.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 630, "domain": "wix.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.wix.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 631, "domain": "accuweather.com", "group": "excluded: browser failed to load", "robots_status": 403, "robots_final": "https://accuweather.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 632, "domain": "dyndns.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://account.dyn.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 633, "domain": "ikea.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.ikea.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 634, "domain": "alicdn.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 635, "domain": "bytedns1.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 636, "domain": "vercel.app", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://vercel.com", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 637, "domain": "ups.com", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://www.ups.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 638, "domain": "doctolib.fr", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.doctolib.fr/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 639, "domain": "force.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://www.force.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 640, "domain": "cdnvideo.ru", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://cdnvideo.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 641, "domain": "xhamster.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 642, "domain": "nike.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.nike.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 643, "domain": "tencent.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 644, "domain": "samsungapps.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 645, "domain": "amazon.ca", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.ca/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 646, "domain": "footprintdns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 647, "domain": "mega.co.nz", "group": "excluded: browser redirected elsewhere", "robots_status": 200, "robots_final": "https://mega.co.nz/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 648, "domain": "sharethrough.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://sharethrough.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "Googlebot", "PerplexityBot", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 649, "domain": "easebar.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 650, "domain": "msidentity.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 651, "domain": "resolver.arpa", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 652, "domain": "jotform.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.jotform.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 653, "domain": "techcrunch.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://techcrunch.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 654, "domain": "2gis.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://2gis.ae/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 655, "domain": "linode.com", "group": "partly in the HTML", "robots_status": 200, "robots_final": "https://www.linode.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 656, "domain": "publicnode.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://publicnode.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 657, "domain": "loc.gov", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.loc.gov/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 658, "domain": "sagepub.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.sagepub.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "Googlebot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": true }, { "rank": 659, "domain": "pushy.io", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://pushy.me/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 660, "domain": "intercom.io", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.intercom.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [ "ai-train=yes, ai-input=yes, search=yes" ], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 661, "domain": "withgoogle.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.google.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 662, "domain": "shopee.com.br", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://shopee.com.br/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 663, "domain": "bidswitch.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 664, "domain": "telephony.goog", "group": "excluded: unreachable", "robots_error": "RemoteProtocolError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 665, "domain": "ks-cdn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 666, "domain": "service.gov.uk", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.gov.uk", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 667, "domain": "redhat.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.redhat.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 668, "domain": "stackadapt.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.stackadapt.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 669, "domain": "britannica.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.britannica.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 670, "domain": "primevideo.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.primevideo.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "ChatGPT-User", "Claude-User", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 671, "domain": "elpais.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://elpais.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": true }, "named": [ "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 672, "domain": "usercontent.goog", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://usercontent.goog/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 673, "domain": "verisign.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.verisign.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 674, "domain": "360.cn", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://360.cn/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 675, "domain": "bunnyinfra.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 676, "domain": "yelp.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.yelp.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Perplexity-User", "CCBot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 677, "domain": "surveymonkey.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.surveymonkey.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 678, "domain": "anydesk.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://anydesk.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 679, "domain": "allegro.pl", "group": "blocked for both", "robots_status": 200, "robots_final": "https://allegro.pl/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 680, "domain": "perplexity.ai", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.perplexity.ai/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 681, "domain": "cambridge.org", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.cambridge.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "ChatGPT-User" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": true }, { "rank": 682, "domain": "vkontakte.ru", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://vkontakte.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 683, "domain": "hstgr.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 684, "domain": "optimizely.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.optimizely.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 685, "domain": "sophos.com", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://www.sophos.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "Googlebot", "PerplexityBot", "Amazonbot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 686, "domain": "uk.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://uk.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 687, "domain": "marriott.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.marriott.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 688, "domain": "wpguardian.io", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://wpguardian.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 689, "domain": "viber.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.viber.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 690, "domain": "amazon.in", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.in/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 691, "domain": "hihonorcloud.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://hihonorcloud.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 692, "domain": "vedsalb.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 693, "domain": "mhverifier.ru", "group": "blocked for both", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 694, "domain": "airbnb.com", "group": "excluded: browser redirected elsewhere", "robots_status": 200, "robots_final": "https://www.airbnb.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": false, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Googlebot", "PerplexityBot", "Applebot-Extended", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 695, "domain": "iso.org", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://iso.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 696, "domain": "cookielaw.org", "group": "in the plain HTML", "robots_status": 421, "robots_final": "https://www.cookielaw.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 697, "domain": "outbrain.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.outbrain.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 698, "domain": "dailymail.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://www.dailymail.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 699, "domain": "icanhazip.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://icanhazip.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": true, "tdmrep": false }, { "rank": 700, "domain": "dotomi.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.epsilon.com/us?redirect=dotomi", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 701, "domain": "kick.com", "group": "partly in the HTML", "robots_status": 200, "robots_final": "https://kick.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 702, "domain": "consultant.ru", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.consultant.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 703, "domain": "dailymail.co.uk", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://www.dailymail.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 704, "domain": "geobasket.ru", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 705, "domain": "cbsnews.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.cbsnews.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 706, "domain": "twimg.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 707, "domain": "lemonde.fr", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.lemonde.fr/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": true }, "named": [ "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 708, "domain": "wps.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.wps.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 709, "domain": "typekit.net", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 710, "domain": "fontawesome.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://fontawesome.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 711, "domain": "samsungacr.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 712, "domain": "ring.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://ring.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot", "Applebot-Extended" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 713, "domain": "seznam.cz", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.seznam.cz/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 714, "domain": "id5-sync.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://console.id5.io/login/auth", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 715, "domain": "cmediahub.ru", "group": "blocked for both", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 716, "domain": "fb.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://fb.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "Googlebot", "PerplexityBot", "Applebot-Extended", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 717, "domain": "porkbun.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://porkbun.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 718, "domain": "alipaydns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 719, "domain": "oxylabs.io", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://oxylabs.io/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [ "search=yes, ai-input=yes, ai-train=yes" ], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 720, "domain": "hostgator.com.br", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://hostgator.com.br/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 721, "domain": "wpguardian.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://wpguardian.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 722, "domain": "googlezip.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 723, "domain": "amazon.es", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.es/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 724, "domain": "pinimg.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.pinterest.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 725, "domain": "uber.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://www.uber.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 726, "domain": "ya.ru", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://ya.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 727, "domain": "nel.goog", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 728, "domain": "itch.io", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://itch.io/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 729, "domain": "cnet.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.cnet.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 730, "domain": "independent.co.uk", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.independent.co.uk/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 731, "domain": "bamgrid.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 732, "domain": "asus.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.asus.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 733, "domain": "amazon.it", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.it/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 734, "domain": "imgur.com", "group": "needs JavaScript", "robots_status": 429, "robots_final": "https://imgur.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 735, "domain": "nmrodam.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 736, "domain": "gafg.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 737, "domain": "bandcamp.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://bandcamp.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 738, "domain": "stripchat.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 739, "domain": "quantserve.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://quantserve.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 740, "domain": "globalsign.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.globalsign.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 741, "domain": "mailchimp.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://mailchimp.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 742, "domain": "shopee.co.id", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://shopee.co.id/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 743, "domain": "bugsnag.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.bugsnag.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 744, "domain": "grammarly.io", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 745, "domain": "githubusercontent.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 746, "domain": "dbankcloud.com", "group": "in the plain HTML", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 747, "domain": "wp.pl", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.wp.pl/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "PerplexityBot", "Perplexity-User", "CCBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 748, "domain": "python.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.python.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 749, "domain": "safebrowsing.apple", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 750, "domain": "creativecdn.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://creativecdn.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 751, "domain": "parklogic.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://parklogic.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 752, "domain": "cdnhwc1.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 753, "domain": "bidr.io", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 754, "domain": "1rx.io", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 755, "domain": "agkn.com", "group": "excluded: no real homepage", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 756, "domain": "imrworldwide.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 757, "domain": "webempresa.eu", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://webempresa.eu/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 758, "domain": "merriam-webster.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.merriam-webster.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 759, "domain": "ad-tech.ru", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 760, "domain": "g.co", "group": "excluded: no real homepage", "robots_status": 400, "robots_final": "https://g.co/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 761, "domain": "google.co.uk", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.google.co.uk/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 762, "domain": "ddnss.de", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 763, "domain": "fda.gov", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://www.fda.gov/apology_objects/abuse-detection-apology.html", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 764, "domain": "pvp.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 765, "domain": "homedepot.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.homedepot.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 766, "domain": "bild.de", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.bild.de/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": true }, "named": [ "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 767, "domain": "xboxlive.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 768, "domain": "smaato.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 769, "domain": "criteo.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 770, "domain": "mlb.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://www.mlb.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 771, "domain": "crwdcntrl.net", "group": "blocked for both", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 772, "domain": "lijit.com", "group": "excluded: no real homepage", "robots_status": 403, "robots_final": "https://lijit.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 773, "domain": "gumgum.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://gumgum.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "Googlebot", "PerplexityBot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 774, "domain": "patreon.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://patreon.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "PerplexityBot" ], "content_signal": [ "search=yes, ai-input=yes, ai-train=no", "search=yes, ai-input=yes, ai-train=no" ], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 775, "domain": "ozone.ru", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 776, "domain": "steamstatic.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 777, "domain": "nintendo.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 778, "domain": "mediafire.com", "group": "in the plain HTML", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 779, "domain": "buydomains.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://buydomains.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 780, "domain": "zillow.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://www.zillow.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": false, "Bytespider": true, "Amazonbot": false, "meta-externalagent": false }, "named": [ "Applebot-Extended", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 781, "domain": "keenetic.io", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 782, "domain": "box.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.box.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 783, "domain": "nikkei.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.nikkei.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 784, "domain": "curseforge.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.curseforge.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 785, "domain": "herokudns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 786, "domain": "google.com.br", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.google.com.br/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 787, "domain": "ovscdns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 788, "domain": "shopeemobile.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 789, "domain": "usgovcloudapi.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 790, "domain": "mckinsey.com", "group": "excluded: browser failed to load", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 791, "domain": "bestbuy.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.bestbuy.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 792, "domain": "sedo.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://sedo.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 793, "domain": "singular.net", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.singular.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 794, "domain": "meta.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.meta.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 795, "domain": "orderbox-dns.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 796, "domain": "warnerbros.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.warnerbros.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 797, "domain": "sc-cdn.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 798, "domain": "lenovo.com", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://www.lenovo.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 799, "domain": "craigslist.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://craigslist.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 800, "domain": "recaptcha.net", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://recaptcha.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 801, "domain": "wattpad.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.wattpad.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 802, "domain": "beget.ru", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://beget.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 803, "domain": "fiverr.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://www.fiverr.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 804, "domain": "allaboutcookies.org", "group": "blocked for both", "robots_status": 200, "robots_final": "https://allaboutcookies.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 805, "domain": "apnews.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://apnews.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": true, "Amazonbot": false, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "CCBot", "Applebot-Extended", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 806, "domain": "ted.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.ted.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 807, "domain": "ameblo.jp", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://ameblo.jp/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 808, "domain": "welt.de", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.welt.de/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": true }, "named": [ "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 809, "domain": "no-ip.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.noip.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 810, "domain": "liadm.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 811, "domain": "mixpanel.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://mixpanel.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 812, "domain": "rt.ru", "group": "excluded: no real homepage", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 813, "domain": "prnewswire.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.prnewswire.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 814, "domain": "moloco.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.moloco.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "CCBot", "Bytespider" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 815, "domain": "disneyplus.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.disneyplus.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": false, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 816, "domain": "xbox.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.xbox.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 817, "domain": "cursor.sh", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://cursor.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 818, "domain": "zoom.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://zoom.us/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 819, "domain": "rakuten.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.rakuten.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 820, "domain": "people.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://people.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 821, "domain": "eeroup.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://eeroup.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 822, "domain": "erome.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 823, "domain": "adform.net", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://adform.net/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 824, "domain": "33across.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://www.33across.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 825, "domain": "vivo.com.cn", "group": "partly in the HTML", "robots_status": 200, "robots_final": "https://www.vivo.com.cn/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 826, "domain": "adjust.com", "group": "in the plain HTML", "robots_status": 429, "robots_final": "https://www.adjust.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 827, "domain": "appcenter.ms", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://appcenter.ms/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 828, "domain": "liftoff.io", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://liftoff.ai/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 829, "domain": "zoho.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.zoho.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 830, "domain": "teads.tv", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://cdn.teads.tv/redirect.html", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 831, "domain": "wcdnga.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 832, "domain": "discordapp.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://discord.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 833, "domain": "networkadvertising.org", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://thenai.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 834, "domain": "odoo.com", "group": "excluded: unreachable", "robots_error": "ReadTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 835, "domain": "ryanair.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.ryanair.com/ie/en", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 836, "domain": "lefigaro.fr", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.lefigaro.fr/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": false, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "CCBot", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": true }, { "rank": 837, "domain": "grab.co", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 838, "domain": "deepl.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.deepl.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 839, "domain": "a-mo.net", "group": "excluded: no real homepage", "robots_status": 403, "robots_final": "https://a-mo.net/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 840, "domain": "me.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.icloud.com/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 841, "domain": "xvideos.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 842, "domain": "usps.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.usps.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 843, "domain": "turn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 844, "domain": "unpkg.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://unpkg.com/robots.txt@1.1.0/index.js", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 845, "domain": "blackberry.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.blackberry.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 846, "domain": "agoda.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://www.agoda.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 847, "domain": "usda.gov", "group": "blocked for the browser only", "robots_status": 403, "robots_final": "https://www.usda.gov/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 848, "domain": "goskope.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 849, "domain": "amagi.tv", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 850, "domain": "samsungosp.com", "group": "in the plain HTML", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 851, "domain": "att.com", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://www.att.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 852, "domain": "academia.edu", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://www.academia.edu/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "Claude-SearchBot", "Claude-User", "Google-Extended", "Googlebot", "PerplexityBot", "Applebot-Extended" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 853, "domain": "wb.ru", "group": "excluded: no real homepage", "robots_status": 498, "robots_final": "https://www.wildberries.ru/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 854, "domain": "theverge.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://theverge.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 855, "domain": "daum.net", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://daum.net/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 856, "domain": "pccc.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://pccc.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 857, "domain": "onlyfans.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://onlyfans.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 858, "domain": "ebay.de", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://www.ebay.de/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Googlebot", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 859, "domain": "pexels.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.pexels.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "CCBot", "Applebot-Extended", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 860, "domain": "cloudsink.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 861, "domain": "discogs.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.discogs.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 862, "domain": "onetag-sys.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://onetag-sys.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 863, "domain": "livejournal.com", "group": "blocked for plain HTTP only", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 864, "domain": "amazon.com.br", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.com.br/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 865, "domain": "amazon.com.au", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.amazon.com.au/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 866, "domain": "jetbrains.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.jetbrains.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 867, "domain": "ebay.co.uk", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://www.ebay.co.uk/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Googlebot", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 868, "domain": "tbank.ru", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 869, "domain": "life360.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.life360.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 870, "domain": "focus.de", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://focus.de/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 871, "domain": "adgrx.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 872, "domain": "figma.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.figma.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "CCBot", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 873, "domain": "fbsbx.com", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://fbsbx.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 874, "domain": "chess.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.chess.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 875, "domain": "frontiersin.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.frontiersin.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 876, "domain": "synology.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.synology.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 877, "domain": "cloudinary.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://cloudinary.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 878, "domain": "kleinanzeigen.de", "group": "excluded: no real homepage", "robots_status": 403, "robots_final": "https://kleinanzeigen.de/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 879, "domain": "dnse0.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 880, "domain": "a-msedge.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 881, "domain": "pikabu.ru", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://pikabu.ru/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 882, "domain": "yellowblue.io", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 883, "domain": "www.gov.br", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.gov.br/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 884, "domain": "kunluncan.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 885, "domain": "appspot.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://accounts.google.com/v3/signin/identifier?continue=https://console.cloud.google.com/appengine&followup=https://console.cloud.google.com/appengine&osid=1&passive=1209600&service=cloudconsole&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S857042002:1791825583218871", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 886, "domain": "hbr.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://hbr.org/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "Google-Extended", "CCBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 887, "domain": "biblegateway.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.biblegateway.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 888, "domain": "pendo.io", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.pendo.io/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 889, "domain": "disqus.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://disqus.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 890, "domain": "it.com", "group": "blocked for both", "robots_status": 404, "robots_final": "https://it.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 891, "domain": "myspace.com", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://myspace.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 892, "domain": "spotifycdn.com", "group": "blocked for both", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 893, "domain": "ssl-images-amazon.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 894, "domain": "nbcnews.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.nbcnews.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 895, "domain": "msftauth.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 896, "domain": "bytefcdn.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 897, "domain": "360safe.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 898, "domain": "azure-devices.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 899, "domain": "cdn-vk.ru", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 900, "domain": "dnse2.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 901, "domain": "pornhub.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 902, "domain": "lg.com", "group": "blocked for the browser only", "robots_status": 200, "robots_final": "https://www.lg.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 903, "domain": "agora.io", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.agora.io/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 904, "domain": "dv.tech", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 905, "domain": "worldbank.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.worldbank.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 906, "domain": "ys7.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.ys7.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 907, "domain": "immedia-semi.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 908, "domain": "mayoclinic.org", "group": "blocked for the browser only", "robots_status": 403, "robots_final": "https://www.mayoclinic.org/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 909, "domain": "chatgpt.site", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://chatgpt.com/features/sites/", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 910, "domain": "readthedocs.io", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://about.readthedocs.com/?ref=app.readthedocs.org", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 911, "domain": "princeton.edu", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.princeton.edu/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 912, "domain": "bankofamerica.com", "group": "partly in the HTML", "robots_status": 200, "robots_final": "https://www.bankofamerica.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 913, "domain": "coupang.com", "group": "excluded: robots.txt disallows", "robots_status": 403, "robots_final": "https://www.coupang.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 914, "domain": "fwmrm.net", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.freewheel.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 915, "domain": "arcgis.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.arcgis.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 916, "domain": "expedia.com", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.expedia.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "OAI-SearchBot", "ChatGPT-User", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 917, "domain": "lwsdns.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://lwsdns.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 918, "domain": "volcgslb.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 919, "domain": "netlify.app", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://netlify.app/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 920, "domain": "latimes.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.latimes.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "Bytespider" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 921, "domain": "wyzecam.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.wyze.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "Google-Extended", "PerplexityBot", "CCBot", "meta-externalagent" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 922, "domain": "thenai.org", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://thenai.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 923, "domain": "att.net", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.att.net/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 924, "domain": "starlink.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://starlink.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 925, "domain": "eu-1-id5-sync.com", "group": "excluded: no real homepage", "robots_status": 522, "robots_final": "https://eu-1-id5-sync.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 926, "domain": "stbid.ru", "group": "excluded: browser failed to load", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 927, "domain": "herokuapp.com", "group": "blocked for both", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 928, "domain": "epa.gov", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.epa.gov/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 929, "domain": "seedtag.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.seedtag.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 930, "domain": "abovedomains.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://abovedomains.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 931, "domain": "washington.edu", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.washington.edu/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 932, "domain": "qualtrics.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.qualtrics.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 933, "domain": "flashtalking.com", "group": "blocked for plain HTTP only", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 934, "domain": "queniuaa.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 935, "domain": "tiktokw.us", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 936, "domain": "mercadolivre.com.br", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.mercadolivre.com.br/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": false, "meta-externalagent": true }, "named": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "PerplexityBot", "Perplexity-User", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 937, "domain": "genius.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://genius.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "Bytespider", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 938, "domain": "trendyol.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.trendyol.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 939, "domain": "docker.io", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.docker.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 940, "domain": "deloitte.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.deloitte.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 941, "domain": "miwifi.com", "group": "partly in the HTML", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 942, "domain": "mercadolibre.com.ar", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.mercadolibre.com.ar/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": false, "Google-Extended": true, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": false, "meta-externalagent": true }, "named": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "PerplexityBot", "Perplexity-User", "Amazonbot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 943, "domain": "state.gov", "group": "in the plain HTML", "robots_status": 403, "robots_final": "https://www.state.gov/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 944, "domain": "smartthings.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://partners.smartthings.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 945, "domain": "lowes.com", "group": "blocked for the browser only", "robots_status": 403, "robots_final": "https://www.lowes.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 946, "domain": "yieldmo.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://yieldmo.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 947, "domain": "youku.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 948, "domain": "tiktokpangle.us", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 949, "domain": "fidelity.com", "group": "excluded: browser failed to load", "robots_status": 200, "robots_final": "https://www.fidelity.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 950, "domain": "tailscale.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://tailscale.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 951, "domain": "eporner.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 952, "domain": "zenecn.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 953, "domain": "px-cloud.net", "group": "excluded: no real homepage", "robots_status": 404, "robots_final": "https://px-cloud.net/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 954, "domain": "go-mpulse.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 955, "domain": "chaturbate.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 956, "domain": "leiniao.com", "group": "excluded: no real homepage", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 957, "domain": "binance.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.binance.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 958, "domain": "note.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://note.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 959, "domain": "dbankcloud.ru", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 960, "domain": "jd.com", "group": "in the page source (JSON)", "robots_status": 200, "robots_final": "https://jd.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "Googlebot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 961, "domain": "onet.pl", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://www.onet.pl/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 962, "domain": "character.ai", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://character.ai/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 963, "domain": "sentinelone.net", "group": "blocked for both", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 964, "domain": "t-s1-msedge.net", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 965, "domain": "samsungiotcloud.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 966, "domain": "mailchi.mp", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://mailchi.mp/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 967, "domain": "repubblica.it", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.repubblica.it/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 968, "domain": "hm.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://hm.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 969, "domain": "cdnhwc2.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 970, "domain": "webmd.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.webmd.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ClaudeBot", "CCBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 971, "domain": "auvik.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.auvik.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 972, "domain": "otto.de", "group": "partly in the HTML", "robots_status": 200, "robots_final": "https://www.otto.de/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "PerplexityBot", "Perplexity-User" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 973, "domain": "corriere.it", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.corriere.it/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "PerplexityBot", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [ "search=yes, ai-train=yes, ai-input=no" ], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 974, "domain": "4dex.io", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 975, "domain": "xhamster46.desi", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://xhamster46.desi/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 976, "domain": "hugedomains.com", "group": "blocked for plain HTTP only", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 977, "domain": "change.org", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.change.org/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 978, "domain": "xnxx.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 979, "domain": "caixa.gov.br", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.caixa.gov.br/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 980, "domain": "playrix.com", "group": "needs JavaScript", "robots_status": 200, "robots_final": "https://playrix.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 981, "domain": "nexusmods.com", "group": "blocked for plain HTTP only", "robots_status": 403, "robots_final": "https://www.nexusmods.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 982, "domain": "app-ads-services.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://app-ads-services.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 983, "domain": "xerox.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.xerox.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 984, "domain": "ancestry.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://ancestry.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 985, "domain": "presage.io", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 986, "domain": "kwai.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 987, "domain": "conviva.com", "group": "excluded: no real homepage", "robots_status": 200, "robots_final": "https://www.conviva.ai/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 988, "domain": "btloader.com", "group": "excluded: robots.txt disallows", "robots_status": 200, "robots_final": "https://btloader.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": false, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 989, "domain": "ubi.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.ubisoft.com/en-gb", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 990, "domain": "investopedia.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.investopedia.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": false, "Claude-User": false, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": false, "CCBot": false, "Applebot-Extended": false, "Bytespider": false, "Amazonbot": false, "meta-externalagent": false }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Applebot-Extended", "Bytespider", "Amazonbot", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 991, "domain": "sofascore.com", "group": "blocked for both", "robots_status": 403, "robots_final": "https://sofascore.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 992, "domain": "pixiv.net", "group": "blocked for both", "robots_status": 200, "robots_final": "https://www.pixiv.net/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": false, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": false, "Bytespider": true, "Amazonbot": true, "meta-externalagent": false }, "named": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Applebot-Extended", "meta-externalagent" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 993, "domain": "faphouse.com", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 994, "domain": "elmundo.es", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.elmundo.es/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": false, "ChatGPT-User": false, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": false, "Googlebot": true, "PerplexityBot": false, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "Google-Extended", "PerplexityBot", "CCBot" ], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 995, "domain": "ddns.net", "group": "excluded: unreachable", "robots_error": "ConnectTimeout", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 996, "domain": "transsion-os.com", "group": "excluded: unreachable", "robots_error": "ConnectError", "llms_txt": null, "ai_txt": null, "tdmrep": null }, { "rank": 997, "domain": "nationalgeographic.com", "group": "in the plain HTML", "robots_status": 200, "robots_final": "https://www.nationalgeographic.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 998, "domain": "bol.com", "group": "excluded: unreachable", "robots_status": 200, "robots_final": "https://www.bol.com/robots.txt", "allowed": { "GPTBot": true, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": true, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [], "content_signal": [], "llms_txt": false, "ai_txt": false, "tdmrep": false }, { "rank": 999, "domain": "flightradar24.com", "group": "blocked for plain HTTP only", "robots_status": 200, "robots_final": "https://www.flightradar24.com/robots.txt", "allowed": { "GPTBot": false, "OAI-SearchBot": true, "ChatGPT-User": true, "ClaudeBot": true, "Claude-SearchBot": true, "Claude-User": true, "Google-Extended": true, "Googlebot": true, "PerplexityBot": true, "Perplexity-User": true, "CCBot": false, "Applebot-Extended": true, "Bytespider": true, "Amazonbot": true, "meta-externalagent": true }, "named": [ "GPTBot", "ChatGPT-User", "CCBot" ], "content_signal": [], "llms_txt": true, "ai_txt": false, "tdmrep": false }, { "rank": 1000, "domain": "heytapmobi.com", "group": "in the plain HTML", "robots_status": 404, "robots_final": "https://heytapmobi.com/robots.txt", "llms_txt": false, "ai_txt": false, "tdmrep": false } ]