User-agent: Googlebot
Disallow: /admin/
User-agent:
BizBot04 kirk.overleaf.com
User-agent:HappyBot (gserver.kw.net)
User-agent:CaliforniaBrownSpider
User-agent:EI*Net/0.1 libwww/0.1
User-agent:Ibot/1.0 libwww-perl/0.40
User-agent:Merritt/1.0
User-agent:StatFetcher/1.0
User-agent:TeacherSoft/1.0 libwww/2.17
User-agent:WWW Collector
User-agent:processor/0.0ALPHA libwww-perl/0.20
User-agent:wobot/1.0 from 206.214.202.45
User-agent:Libertech-Rover www.libertech.com?
User-agent:WhoWhere Robot
User-agent:ITI Spider
User-agent:w3index
User-agent:MyCNNSpider
User-agent:SummyCrawler
User-agent:OGspider
User-agent:linklooker
User-agent:CyberSpyder (amant@www.cyberspyder.com)
User-agent:SlowBot
User-agent:heraSpider
User-agent:Surfbot
User-agent:Bizbot003
User-agent:WebWalker
User-agent:SandBot
User-agent:EnigmaBot
User-agent:spyder3.microsys.com
User-agent:www.freeloader.com.
|
