# robots.txt for http://www.spcsf.org
# Last modified: 2009-6-5T07:50:00-0800

User-agent: * 
Disallow: /administrator/
Disallow: /_private/
Disallow: /_vti_bin/
Disallow: /_vti_cnf/
Disallow: /_vti_log/
Disallow: /_vti_pvt/
Disallow: /_vti_txt/
Disallow: /cache/
Disallow: /components/
Disallow: /editor/
Disallow: /help/
Disallow: /images/
Disallow: /includes/
Disallow: /language/
Disallow: /mambots/
Disallow: /media/
Disallow: /modules/
Disallow: /templates/
Disallow: /installation/
Disallow: /configuration.php 
# Bot Blocks
User-agent: Orthogaffe
# advertising-related bots:
User-agent: grub-client 	
User-agent: grub 
User-agent: looksmart 
User-agent: WebZip 
User-agent: larbin 
User-agent: b2w/0.1 
User-agent: Copernic 
User-agent: NutchOrg
User-agent: Nutch
# Crawlers that are kind enough to obey, but which we'd rather not have
# unless they're feeding search engines.
User-agent: UbiCrawler
User-agent: Zao
# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.
User-agent: Zealbot	
User-agent: MSIECrawler
User-agent: SiteSnagger
User-agent: WebStripper
User-agent: WebCopier
User-agent: Fetch
User-agent: Offline Explorer
User-agent: sitecheck.internetseer.com
User-agent: Teleport
User-agent: TeleportPro
User-agent: linko
User-agent: WebZIP
User-agent: Microsoft.URL.Control
User-agent: HTTrack
User-agent: Xenu
User-agent: libwww
User-agent: larbin
User-agent: ZyBORG
User-agent: Download Ninja
User-agent: NPBot
User-agent: ia_archiver-web.archive.org
User-agent: duggmirror
# Sorry, wget in its recursive mode is a frequent problem.
# Please read the man page and use it properly; there is a
# --wait option you can use to set the delay between hits,
User-agent: wget
User-agent: grub-client
User-agent: k2spider
# A capture bot, downloads gazillions of pages with no public benefit
User-agent: WebReaper
Disallow: /

# Allows
User-agent: Mediapartners-Google
User-agent: Adsbot-Google
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
Allow: /
# Sitemap
Sitemap: http://www.askapache.com/sitemap.xml