# LLM and AI Crawlers - Full Access
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Google Bard/Gemini
User-agent: Google-Extended
Allow: /

# General crawlers
User-agent: *
Allow: /

# Disallow admin and private areas for all agents
User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /storage/
Disallow: /login
Disallow: /register
Disallow: /password/
Disallow: /livewire/
Disallow: /cdn-cgi/
Disallow: /auth/google
Disallow: /api/auth/google
Disallow: /api/auth/github

# LLM-specific resources
User-agent: *
Allow: /llms.txt
Allow: /llms-full.txt

# Sitemap location
Sitemap: https://converthub.com/sitemap.xml
Sitemap: https://converthub.com/blog/sitemap.xml
Sitemap: https://converthub.com/sitemap-llm.xml