# Marfa AI - robots.txt
# This file tells search engines which pages to crawl

# Allow all crawlers
User-agent: *

# Allow all pages
Allow: /

# Specifically allow blog pages (for clarity)
Allow: /blog
Allow: /blog/*/

# Block authentication pages from indexing
Disallow: /login
Disallow: /register
Disallow: /auth/
Disallow: /complete-profile

# Block dashboard pages from indexing (requires login)
Disallow: /dashboard
Disallow: /assistants
Disallow: /notes
Disallow: /conversations
Disallow: /phone
Disallow: /settings
Disallow: /subscription
Disallow: /apps

# Block admin pages
Disallow: /api/

# Block Cloudflare internal paths (email protection, etc.)
Disallow: /cdn-cgi/

# Sitemap location
Sitemap: https://heymarfa.com/sitemap.xml

# Crawl-delay (optional, adjust based on server capacity)
# Crawl-delay: 1