User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Allow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Allow: /

User-agent: meta-externalagent
Disallow: /

User-agent: *
Allow: /

# Block API endpoints — no SEO value, save crawl budget
Disallow: /api/

# Block utility & legal pages from being crawled heavily
Disallow: /404/

# Allow all brand, category, origin pages
Allow: /brand/
Allow: /brands/
Allow: /category/
Allow: /origin/

# Sitemap
Sitemap: https://batchfirst.com/sitemap-index.xml