# All robots will spider the domain
User-agent: *
# Disallow directory /cgi-bin/, Themes and Sources.
Disallow: /cgi-bin/
Disallow: /Themes/
Disallow: /Sources/
# Set a time delay for the crawlers - some, anyway.
Crawl-delay: 10