# LocallyGrown.net Production Robots.txt
# Allow search engines to index public content
User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /cart/
Disallow: /orders/
Disallow: /checkout/
Disallow: /account/
Disallow: /growers/admin/
Disallow: /markets/edit
Disallow: /signin
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password

# Block old Rails routes that no longer exist
Disallow: /market/index/
Disallow: /growers/show/
Disallow: /photos/show/
Disallow: /products/show/
Disallow: /users/show/
Disallow: /categories/show/
Disallow: /markets/show/
Disallow: /news/show/
Disallow: /events/show/
Disallow: /orders/show/
Disallow: /weblog/show/
Disallow: /recipes/show/
Disallow: /user/signin
Disallow: /user/signup
Disallow: /user/register
Disallow: /user/login

Allow: /

# Aggressive bot blocking
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BlexBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: ByteSpider
Disallow: /

# Slow down aggressive bots
User-agent: *
Crawl-delay: 2

# Sitemap location
Sitemap: https://locallygrown.net/sitemap.xml