User-agent: MJ12bot
Disallow: /
User-agent: Googlebot
Allow: /
Crawl-delay: 1
User-agent: msnbot
Allow: /
Crawl-delay: 5
User-agent: bingbot
Allow: /
Crawl-delay: 5
#Baiduspider
User-agent: Baiduspider
Allow: /
Crawl-delay: 10
User-agent: Slurp
Allow: / Crawl-delay: 30
User-agent: Yandex
Allow: /
Crawl-delay: 10
User-agent: TwengaBot
Allow: / Crawl-delay: 10
User-agent: *
Allow: /
Crawl-delay: 20