llms.txt

# Allow all AI systems by default User-agent: * Allow: / # OpenAI (ChatGPT, GPTBot) User-agent: GPTBot Allow: / # Anthropic (Claude) User-agent: ClaudeBot Allow: / # Perplexity AI User-agent: PerplexityBot Allow: / # Google AI (Gemini, Bard-related crawlers) User-agent: Google-Extended Allow: / # Common Crawl (used by many AI models) User-agent: CCBot Allow: / # Meta AI (Facebook / LLaMA) User-agent: Meta-ExternalAgent Allow: / # Amazon AI User-agent: Amazonbot Allow: / # Apple AI / Applebot User-agent: Applebot Allow: / # Bytespider (TikTok / ByteDance AI) User-agent: Bytespider Allow: / # You.com AI User-agent: YouBot Allow: / # Diffbot (AI extraction engine) User-agent: Diffbot Allow: / # Archive / dataset crawlers User-agent: ia_archiver Allow: /