# Sitemap location
Sitemap: https://www.autosolid.nl/sitemap.xml

# ---------------- BLOCKED CRAWLERS ----------------

# Block AhrefsBot
User-agent: AhrefsBot
Disallow: /

# Block SemrushBot
User-agent: SemrushBot
Disallow: /

# Block MJ12bot
User-agent: MJ12bot
Disallow: /

# ---------------- CRAWL-RATE LIMITED ----------------

# FIX: Crawl-delay directives were listed in comments but never actually written.
#      Added the missing directive lines so they take effect.

# Yandex: slow down crawling
User-agent: Yandex
Crawl-delay: 10

# Baiduspider: slow down crawling
User-agent: Baiduspider
Crawl-delay: 15

# ---------------- ALLOWED CRAWLERS ----------------

# All other bots have full access
User-agent: *
Disallow: