# To exclude all robots from the entire server
#     User-agent: *
#     Disallow: /
# To exclude all robots from part of the server
# To exclude a single robot
#     User-agent: ia_archiver
#     Disallow: /
#     User-agent: Googlebot
#     Disallow: /

#     User-agent: *
#     Disallow: /
#     Disallow: /proxy/

# To allow a single robot
#      User-agent: WebCrawler
#      Disallow:

# To exclude all files except one

# This is currently a bit awkward, as there is no "Allow" field. The easy way is to put all # files to be disallowed into a
# separate directory, say "docs", and leave the one file in the level above this directory: 

#      User-agent: *
#      Disallow: /~joe/docs/

# Alternatively you can explicitly disallow all disallowed pages: 

#      User-agent: *
#      Disallow: /~joe/private.html
#      Disallow: /~joe/foo.html
#      Disallow: /~joe/bar.html