# My robots.txt file
# 	List directories here that should not be followed by the web-crawlers

User-agent: *
Disallow: /customer_pages/
Disallow: /dataobjects/
Disallow: /includes/
Disallow: /javascript/
Disallow: /Scripts/
Disallow: /test/
Disallow: /webservices/
Disallow: /wsphp/

# Slow down the crawl speed of MSN and Yahoo, so they don't suck down so much bandwidth
User-Agent: msnbot
Crawl-Delay: 10

User-Agent: Slurp
Crawl-Delay: 10