User-agent: *
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: Adsbot-Google
Allow: /

User-agent: Baiduspider
Disallow: /

User-agent: Yeti
Disallow: /


User-agent: dotbot
Disallow: / 
User-agent: ilse
Disallow: /
User-agent: BDFetch
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: FairShare
Disallow: /
User-agent: yahooysmcm
Disallow: /