{"found":true,"method":"v0.2","host":"x.com","input_host":"x.com","name":"x.com","category":"Uncategorized","scanned_at":"2026-09-22T15:56:53.495Z","vantage":"cloudflare-edge","elapsed_ms":416,"score":14,"grade":"F","stance":"Walled","pillars":{"works":null,"welcome":4,"built":3},"pillar_max":{"works":50,"welcome":25,"built":25},"evidence":[{"t":"critical","text":"robots.txt disallows 17 of 19 user-directed agents via a default Disallow: / for unlisted bots: ChatGPT-User, Claude-User, Perplexity-User, GoogleAgent-Mariner, GoogleAgent-Shopping, Manus-User, MistralAI-User, Devin…"},{"t":"warn","text":"Homepage serves GoogleAgent-Mariner"},{"t":"warn","text":"robots.txt restricts some paths for Meta-ExternalFetcher, meta-externalfetcher"},{"t":"warn","text":"Also disallows 12 of 15 training/search crawlers (not scored; agents ≠ crawlers)"},{"t":"warn","text":"No Content-Signal in robots.txt"},{"t":"warn","text":"No llms.txt"},{"t":"warn","text":"No JSON-LD structured data on the homepage"},{"t":"warn","text":"Missing main/nav landmarks"},{"t":"warn","text":"No agent-protocol manifest (UCP, MCP, A2A, OpenAPI)"}],"matrix":{"Muse":"untested","ChatGPT Agent":"disallowed","Claude":"disallowed","Comet":"disallowed","Grok Bot":"disallowed","Gemini":"disallowed","Manus":"disallowed","Instinct":"untested"},"files":[],"jsonld":0,"semantic":{"main":0,"nav":1,"aria":13,"forms":1},"noscript_shell":false,"content_signal":false,"wildcard_block":true,"robots_status":200,"robots_ua_rules":11,"agents_disallowed":17,"agents_partial":2,"agents_total":19,"crawlers_blocked":12,"crawlers_total":15,"ua_policy":{"ChatGPT-User":"blocked","Claude-User":"blocked","Perplexity-User":"blocked","GoogleAgent-Mariner":"blocked","GoogleAgent-Shopping":"blocked","Manus-User":"blocked","MistralAI-User":"blocked","Devin":"blocked","Copilot":"blocked","GrokBot":"blocked","Grok-DeepSearch":"blocked","Meta-ExternalFetcher":"partial","meta-externalfetcher":"partial","Gemini-Deep-Research":"blocked","DuckAssistBot":"blocked","Operator":"blocked","Comet":"blocked","Muse":"blocked","Instinct":"blocked"},"crawler_policy":{"GPTBot":"blocked","ClaudeBot":"blocked","Claude-SearchBot":"blocked","OAI-SearchBot":"blocked","PerplexityBot":"blocked","Google-Extended":"partial","Applebot-Extended":"blocked","meta-externalagent":"partial","Meta-ExternalAgent":"partial","Bytespider":"blocked","CCBot":"blocked","Amazonbot":"blocked","xAI-Bot":"blocked","Anthropic-ai":"blocked","anthropic-ai":"blocked"},"live_served":1,"live_total":4,"agent_access":{"ChatGPT-User":{"status":402,"challenged":false,"title":""},"Claude-User":{"status":402,"challenged":false,"title":""},"Perplexity-User":{"status":402,"challenged":false,"title":""},"GoogleAgent-Mariner":{"status":200,"challenged":false,"title":"X. It’s what’s happening / X"}},"human_status":200,"human_title":"X. It’s what’s happening / X","human_challenged":false,"edge":"cloudflare envoy cloudflare","probe_dead":false,"seed":false,"detail":{"robots":0.5,"door":3,"signal":0,"surface":3,"protocols":0},"provisional":true,"url":"https://standardweb.xyz/site/x.com","recommendations":[{"k":"front-door","pillar":"Built","pts":10,"title":"Give agents a front door","body":"A Universal Commerce Protocol manifest for shopping, or an MCP server card for everything else. An agent that can call an API never needs to hold a customer's password.","code":"GET /.well-known/ucp\n{ \"ucp\": { \"version\": \"2026-01-23\",\n  \"services\": { \"dev.ucp.shopping\": [{ \"transport\": \"rest\", \"endpoint\": \"https://x.com/api/ucp/v1\" }] },\n  \"capabilities\": { \"dev.ucp.shopping.checkout\": [ … ] } } }\n\n# or\nGET /.well-known/mcp.json\n{ \"name\": \"x.com\", \"endpoint\": \"https://x.com/mcp\", \"transport\": \"streamable-http\" }"},{"k":"robots","pillar":"Welcome","pts":9,"title":"Stop disallowing identified user-directed agents in robots.txt","body":"You disallow 17 agents that act live for a person. Crawler rules (GPTBot, ClaudeBot) can stay; user-directed agents are the ones a customer sends.","code":"User-agent: ChatGPT-User\nUser-agent: Claude-User\nUser-agent: Perplexity-User\nUser-agent: GoogleAgent-Mariner\nUser-agent: GoogleAgent-Shopping\nUser-agent: Manus-User\nAllow: /"},{"k":"llms","pillar":"Built","pts":4,"title":"Publish llms.txt","body":"A short markdown map of what the site does, where the important pages are, and how to search or check out.","code":"/llms.txt\n# x.com\n> One paragraph on what this site is for.\n- [Products](/products): how to search and filter\n- [Checkout](/checkout): steps and constraints"},{"k":"signal","pillar":"Welcome","pts":3,"title":"Say your stance with a Content-Signal","body":"Two lines in robots.txt end the guessing game. ai-input covers agents reading pages on a person's behalf.","code":"User-Agent: *\nContent-Signal: search=yes, ai-input=yes, ai-train=no\nAllow: /"},{"k":"jsonld","pillar":"Built","pts":3,"title":"Add JSON-LD structured data","body":"schema.org Organization on the homepage, Product + Offer on product pages, with price and availability.","code":"<script type=\"application/ld+json\">\n{ \"@context\": \"https://schema.org\", \"@type\": \"Organization\", \"name\": \"x.com\", \"url\": \"https://x.com/\" }\n</script>"},{"k":"landmarks","pillar":"Built","pts":3,"title":"Use semantic landmarks","body":"main, nav, header, footer and labeled forms. Everything an accessibility audit already asks for.","code":"<header>…</header>\n<nav aria-label=\"Primary\">…</nav>\n<main>…</main>\n<form aria-label=\"Site search\"><label for=\"q\">Search</label><input id=\"q\" name=\"q\"></form>"}],"task_runs":[]}