# For more details, see : # http://www.robotstxt.org/ # http://en.wikipedia.org/wiki/Robots.txt # http://www.conman.org/people/spc/robots2.html # # Comments appear after the "#" symbol at the start of a line, or after a directive # # The wildcard "*" specifies that all robots/crawlers are concerned # User-agent: * # # Tells the specified robots/crawlers to crawl every file(s) # Allow everything and Disallow nothing (Allow is a nonstandard extension) # Allow: / Disallow: # # Tells the specified robots/crawlers not to enter specific file(s) # # Disallow: /a-page.html # # Nonstandard extensions : # # Set to 10 seconds to wait between successive requests to the same server # Crawl-delay: 10 # # Maximum rate is one page every 10 seconds # Request-rate: 1/10 # # URI of the xml version of the sitemap # Sitemap: http://www.benoitbeauchaine.com/sitemap.xml