# WSN: World Sports Network
# http://www.wsn.com/robots.txt

# Please use Sitemap
Sitemap: http://www.wsn.com/sitemap_index.xml

# Explicitly allow AdSense Bot:

User-agent: Mediapartners-Google*
Disallow: 

# Crawlers that are kind enough to obey, but which we'd rather not have
# unless they're feeding search engines.

User-agent: UbiCrawler
Disallow: /
User-agent: DOC
Disallow: /
User-agent: Zao
Disallow: /

# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.

User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: linko
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Microsoft.URL.Control
Disallow: /
User-agent: larbin
Disallow: /
User-agent: libwww
Disallow: /
User-agent: ZyBORG
Disallow: /
User-agent: Download Ninja
Disallow: /
User-agent: DotSpotsBot
Disallow: /
User-agent: LinksManager.com_bot 
Disallow: /
User-agent: Gaisbot
Disallow: /
User-agent: CollapsarWEB
Disallow: /
User-agent: WebCensusProjectBot
Disallow: /
User-agent: Mp3Bot
Disallow: /
User-agent: yacybot 
Disallow: /

# Sorry, wget in its recursive mode is a frequent problem.

User-agent: wget
Disallow: /

# The 'grub' distributed client has been *very* poorly behaved.

User-agent: grub-client
Disallow: /

# Doesn't follow robots.txt anyway, but...

User-agent: k2spider
Disallow: /

# Hits many times per second, not acceptable
# http://www.nameprotect.com/botinfo.html

User-agent: NPBot
Disallow: /

# A capture bot, downloads gazillions of pages with no public benefit
# http://www.webreaper.net/

User-agent: WebReaper
Disallow: /

#
# All the rest are welcome to crawl responsibly
#

User-agent: *

# Advertising
Disallow: /content/
Disallow: /bet-now/

# Security:
Disallow: /adodb/
Disallow: /cache/
Disallow: /cgi-bin/
Disallow: /email/
Disallow: /wsn_cms/

# SVN
Disallow: /branches/
Disallow: /tags/
Disallow: /trunk/

# Duplication / Optimization:
Disallow: /search/

# Wordpress
Disallow: /blog/private
Disallow: /blog/wp-admin
Disallow: /blog/wp-includes
Disallow: /blog/wp-content
Disallow: /blog/wt-js