# Block webservices, allow all others.
User-agent: *
Disallow: /svc/

# Block known spiders causing issues on the website
User-agent: AhrefsBot
User-agent: Fatbot
User-agent: MotorcycleGold
User-agent: Pimonster
User-agent: NetcraftSurveyAgent
User-agent: Halebot
User-Agent: Halebot (Mozilla/5.0 compatible; Halebot/2.1) 
User-agent: ScoutJet
Disallow: / 

# Issue a 2 second crawl delay on possibly friendly bots
# Block webservices, allow all others.
User-agent: Yandex
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Mail.RU_Bot
User-agent: GrapeshotCrawler
User-agent: CCBot
Crawl-delay: 5
Disallow: /svc/

Sitemap: http://www.bikebandit.com/sitemap_index.xml