# robots.txt Sep 4 2007 - DWM
User-agent: *
Disallow: /cgi-bin/
Disallow: /classifieds/
Disallow: /search/
Disallow: /events/
Disallow: /dev/
Disallow: /presspass/
Disallow: /shopping/
Disallow: /txt/
Disallow: /textalerts/
Disallow: /PressPass/
Disallow: /assets/
Disallow: /client/
Disallow: /admin/
Disallow: /mailfriend
Disallow: /comments/flag
Disallow: /users/
Disallow: /r/
Disallow: /preview/dev/
Disallow: /photos/


User-agent: Googlebot
Disallow: /cgi-bin/
Disallow: /classifieds/
Disallow: /search/
Disallow: /events/
Disallow: /dev/
Disallow: /presspass/
Disallow: /shopping/
Disallow: /txt/
Disallow: /textalerts/
Disallow: /PressPass/
Disallow: /assets/
Disallow: /client/
Disallow: /admin/
Disallow: /mailfriend
Disallow: /comments/flag
Disallow: /users/
Disallow: /r/
Disallow: /preview/dev/


User-agent: Googlebot-Image
Disallow: /cgi-bin/
Disallow: /classifieds/
Disallow: /search/
Disallow: /events/
Disallow: /dev/
Disallow: /presspass/
Disallow: /shopping/
Disallow: /txt/
Disallow: /textalerts/
Disallow: /PressPass/
Disallow: /assets/
Disallow: /client/
Disallow: /admin/
Disallow: /mailfriend
Disallow: /comments/flag
Disallow: /users/
Disallow: /r/


User-agent: MSNBot
Disallow: /cgi-bin/
Disallow: /classifieds/
Disallow: /search/
Disallow: /events/
Disallow: /dev/
Disallow: /presspass/
Disallow: /shopping/
Disallow: /txt/
Disallow: /textalerts/
Disallow: /PressPass/
Disallow: /assets/
Disallow: /client/
Disallow: /admin/
Disallow: /mailfriend
Disallow: /comments/flag
Disallow: /users/
Disallow: /r/


User-agent: Slurp
Crawl-delay: 20
Disallow: /cgi-bin/
Disallow: /classifieds/
Disallow: /search/
Disallow: /events/
Disallow: /dev/
Disallow: /presspass/
Disallow: /shopping/
Disallow: /txt/
Disallow: /textalerts/
Disallow: /PressPass/
Disallow: /assets/
Disallow: /client/
Disallow: /admin/
Disallow: /mailfriend
Disallow: /comments/flag
Disallow: /users/
Disallow: /r/


User-agent: Robozilla
Crawl-delay: 60
Disallow: /cgi-bin/
Disallow: /classifieds/
Disallow: /search/
Disallow: /events/
Disallow: /dev/
Disallow: /presspass/
Disallow: /shopping/
Disallow: /txt/
Disallow: /textalerts/
Disallow: /PressPass/
Disallow: /assets/
Disallow: /client/
Disallow: /admin/
Disallow: /mailfriend
Disallow: /comments/flag
Disallow: /users/
Disallow: /r/


User-agent: yahoo-mmcrawler
Disallow: /cgi-bin/
Disallow: /classifieds/
Disallow: /search/
Disallow: /events/
Disallow: /dev/
Disallow: /presspass/
Disallow: /shopping/
Disallow: /txt/
Disallow: /textalerts/
Disallow: /PressPass/
Disallow: /assets/
Disallow: /client/
Disallow: /admin/
Disallow: /mailfriend
Disallow: /comments/flag
Disallow: /users/
Disallow: /r/


User-agent: psbot
Crawl-delay: 20
Disallow: /cgi-bin/
Disallow: /classifieds/
Disallow: /search/
Disallow: /events/
Disallow: /dev/
Disallow: /presspass/
Disallow: /shopping/
Disallow: /txt/
Disallow: /textalerts/
Disallow: /PressPass/
Disallow: /assets/
Disallow: /client/
Disallow: /admin/
Disallow: /mailfriend
Disallow: /comments/flag
Disallow: /users/
Disallow: /r/

User-agent: twiceler
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: Scrubby
Disallow: /

User-agent: asterias
Disallow: /

User-agent: yahoo-blogs/v3.9
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: yeti*
Disallow: /

User-agent: GammaSpider/1.0
Disallow: /

User-agent: GammaSpider
Disallow: /

User-agent: Pompos
Disallow: /

User-agent: aipbot
Disallow: /

User-agent: Topix.net
Disallow: /

User-agent: SpiderMan
Disallow: /

User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 5.0 Robot)
Disallow: /

User-agent: MSIE 6.0
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: Yoriwa/0.1
Disallow: /

User-agent: LeapTag
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: oodlebot
Disallow: /

User-agent: Atomic_Email_Hunter/4.0
Disallow: /