User-agent: *
Allow: /

# Block private/sensitive areas
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /admin/
Disallow: /api/
Disallow: /_next/
Disallow: /replay/
Disallow: /subscription/

# Block development and system files
Disallow: /_vercel/
Disallow: /uploads/
Disallow: /.well-known/
Disallow: /monitoring/

# Allow important public pages
Allow: /pricing
Allow: /how-it-works
Allow: /login
Allow: /register
Allow: /dashboard
Allow: /features
Allow: /contact
Allow: /support
Allow: /about
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /cookie-policy

# Allow static assets
Allow: /assets/
Allow: /images/
Allow: /icons/

# Block specific bots that might cause issues
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

# Allow major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# Crawl delay for all bots
Crawl-delay: 1

# Sitemap location
Sitemap: https://trade-bench.com/sitemap.xml