# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file # # To ban all spiders from the entire site uncomment the next two lines: # User-Agent: * # Disallow: / User-Agent: * Disallow: */paylinks/* Disallow: */card_update_links/* Disallow: */print* Disallow: /*? #Baiduspider China and Japan User-agent: Baiduspider Disallow: / User-agent: Baiduspider+ Disallow: / User-agent: Baiduspider+(+http://www.baidu.com/search/spider.htm) Disallow: / User-agent: Baiduspider/2.0;+http://www.baidu.com/search/spider.html Disallow: / User-agent: Baiduspider/2.0 Disallow: / User-agent: +Baiduspider Disallow: / User-agent: +Baiduspider/2.0 Disallow: / User-agent: +Baiduspider/2.0;++http://www.baidu.com/search/spider.html Disallow: / #Mail.RU_Bot/2.0 Russia User-agent: Mail.RU_Bot/2.0 Disallow: / #Mail.RU Russia User-agent: Mail.RU Disallow: / #Mail.Ru Russia User-agent: Mail.Ru Disallow: / User-agent: Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots Disallow: / User-agent: FacebookBot Disallow: / User-agent: FaceBot Disallow: /