User-agent: *
Allow: /

# Allow major AI bots (important for LLMs.txt usage)
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: meta-externalagent
Allow: /

# Optional: allow other AI crawlers
User-agent: CCBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Block sensitive/private areas only
Disallow: /admin/
Disallow: /staging/
Disallow: /filters/
Disallow: /cdn-cgi/

# Sitemap
Sitemap: https://nobukojapan.com/sitemap.xml

# LLM file
LLM: https://nobukojapan.com/llms.txt