{
  "@context": "https://schema.org",
  "@id": "https://webspecification.com/spec/agent-readiness/robots-for-ai-crawlers.jsonld",
  "@graph": [
    {
      "@type": "TechArticle",
      "@id": "https://webspecification.com/spec/agent-readiness/robots-for-ai-crawlers/#article",
      "headline": "robots.txt for AI crawlers",
      "description": "Major AI vendors publish named user-agents for their crawlers. Setting an explicit allow or disallow per agent is the clearest way to control how your content is used.",
      "url": "https://webspecification.com/spec/agent-readiness/robots-for-ai-crawlers/",
      "inLanguage": "en",
      "isPartOf": {
        "@type": "WebSite",
        "@id": "https://webspecification.com/#website",
        "name": "Web Specification Studio",
        "url": "https://webspecification.com"
      },
      "author": {
        "@type": "Person",
        "name": "Web Specification Studio",
        "url": "https://webspecification.com"
      },
      "publisher": {
        "@type": "Organization",
        "@id": "https://webspecification.com/#organization",
        "name": "Web Specification Studio",
        "url": "https://webspecification.com"
      },
      "license": "https://creativecommons.org/licenses/by/4.0/",
      "dateModified": "2026-05-29T12:14:17.000Z",
      "citation": [
        {
          "@type": "CreativeWork",
          "name": "OpenAI — GPTBot",
          "url": "https://platform.openai.com/docs/bots",
          "publisher": {
            "@type": "Organization",
            "name": "OpenAI"
          }
        },
        {
          "@type": "CreativeWork",
          "name": "Anthropic — Crawlers",
          "url": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
          "publisher": {
            "@type": "Organization",
            "name": "Anthropic"
          }
        },
        {
          "@type": "CreativeWork",
          "name": "Google — Google-Extended",
          "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers#google-extended",
          "publisher": {
            "@type": "Organization",
            "name": "Google Search Central"
          }
        },
        {
          "@type": "CreativeWork",
          "name": "Apple — Applebot-Extended",
          "url": "https://support.apple.com/en-us/119829",
          "publisher": {
            "@type": "Organization",
            "name": "Apple"
          }
        }
      ]
    },
    {
      "@type": "BreadcrumbList",
      "@id": "https://webspecification.com/spec/agent-readiness/robots-for-ai-crawlers/#breadcrumb",
      "itemListElement": [
        {
          "@type": "ListItem",
          "position": 1,
          "name": "Home",
          "item": "https://webspecification.com/"
        },
        {
          "@type": "ListItem",
          "position": 2,
          "name": "Spec",
          "item": "https://webspecification.com/spec/"
        },
        {
          "@type": "ListItem",
          "position": 3,
          "name": "Agent Readiness",
          "item": "https://webspecification.com/spec/agent-readiness/"
        },
        {
          "@type": "ListItem",
          "position": 4,
          "name": "robots.txt for AI crawlers",
          "item": "https://webspecification.com/spec/agent-readiness/robots-for-ai-crawlers/"
        }
      ]
    }
  ]
}