# robots.txt for tomw.net.au

# Let the Internet Archive have almost everything.

User-agent: ia_archiver
Crawl-delay: 0
Disallow: /research/ 

# Stop some bots completely.

User-agent: Sensis.au Web Crawler
Disallow: /

User-agent: Sensis.au
Disallow: /

User-agent: Sensis
Disallow: /

User-agent: ShopWiki
Disallow: /

User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: wget
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: k2spider
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: WebReaper
Disallow: /

# Ban some image crawlers.

User-agent: MSNBot-Media
Disallow: /

User-agent: yahoo-mmcrawler
Disallow: /

User-agent: psbot
Disallow: /

User-agent: asterias
Disallow: /

# Ban Microsoft product crawler and slow down text crawler.

User-agent: MSNBot-Products
Disallow: /

User-agent: msnbot
Crawl-delay: 300
Disallow: /research/ 
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /blog/archive/
Disallow: /anucecs/archive/
Disallow: /screenact/archive/
Disallow: /*.ppt$
Disallow: /*.rtf$
Disallow: /*.rm$
Disallow: /*.ra$
Disallow: /*.ram$
Disallow: /*.dxf$
Disallow: /*.au$
Disallow: /*.jpg$
Disallow: /*.jpeg$
Disallow: /*.gif$
Disallow: /*.png$

# Slow down Yahoo and ban non-HTML files and images.

User-agent: Yahoo! Slurp
Crawl-delay: 600
Disallow: /research/
Disallow: /images/
Disallow: /cgi-bin/
Disallow: /blog/archive/
Disallow: /anucecs/archive/
Disallow: /screenact/archive/
Disallow: /*.ppt$
Disallow: /*.rtf$
Disallow: /*.rm$
Disallow: /*.ra$
Disallow: /*.ram$
Disallow: /*.dxf$
Disallow: /*.au$
Disallow: /*.jpg$
Disallow: /*.jpeg$
Disallow: /*.gif$
Disallow: /*.png$

# Allow Google to crawl images but not non-HTML files.

User-agent: Googlebot-Image
Crawl-delay: 0
Disallow: /research/ 

User-agent: Googlebot
Crawl-delay: 0
Disallow: /research/ 
Disallow: /cgi-bin/
Disallow: /blog/archive/
Disallow: /anucecs/archive/
Disallow: /screenact/archive/
Disallow: /*.ppt$
Disallow: /*.rtf$
Disallow: /*.rm$
Disallow: /*.ra$
Disallow: /*.ram$
Disallow: /*.dxf$
Disallow: /*.au$
Disallow: /*.pdf$

# Other crawlers ban non-HTML files and images. Slow down a little.

User-agent: *
Crawl-delay: 10
Disallow: /research/ 
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /blog/archive/
Disallow: /anucecs/archive/
Disallow: /screenact/archive/
Disallow: /2000/bat.ppt
Disallow: /2000/icarba.ppt
Disallow: /2000/iim2000.ppt
Disallow: /2000/iim2000.rtf
Disallow: /2000/ipub.ppt
Disallow: /2000/isup.ppt
Disallow: /2000/pt.ppt
Disallow: /2000/pt/pt.rm
Disallow: /2000/scsp.ppt
Disallow: /2000/yxml.ppt
Disallow: /2000/yxml2.ppt
Disallow: /2001/bat2001.ppt
Disallow: /2001/eal/keynote.ppt
Disallow: /2001/eal/notb2b.ppt
Disallow: /2001/esb/invoice.rtf
Disallow: /2001/itv.ppt
Disallow: /2001/wf.ppt
Disallow: /2001/wwgw/wireless.ppt
Disallow: /2002/acra.ppt
Disallow: /2002/atoxml.ppt
Disallow: /2002/ebcwxml.ppt
Disallow: /2002/edm.ppt
Disallow: /2002/fit.ppt
Disallow: /2002/maa.ppt
Disallow: /2002/mka.ppt
Disallow: /2002/nbt.ppt
Disallow: /2003/bws.ppt
Disallow: /2003/xmlstd.ppt
Disallow: /2003/xmltech.ppt
Disallow: /as/apan.ppt
Disallow: /gsr/gsrit.ppt
Disallow: /intrnt.rtf
Disallow: /irc/irc1intr.au
Disallow: /irc/irc1rm.au
Disallow: /irc/irc5.au
Disallow: /nt/nt.pdf
Disallow: /nt/ntad.rtf
Disallow: /nt/ntflyr.rtf
Disallow: /nt/ntposter.doc
Disallow: /nt/nts.ppt
Disallow: /papers/ccit99.rtf
Disallow: /papers/outint.rtf
Disallow: /tomwlcme.au
Disallow: /travel/ausalps1.au
Disallow: /travel/balloon1.au
Disallow: /uso/uso.pdf
Disallow: /uso/usocvr.rtf
Disallow: /uso/usoflyr.rtf
Disallow: /uso/w3d.ppt
Disallow: /uso/w3d.rtf