# Instructs scanning robots where they are and are not welcome # # This file for: kms.djxmmx.net # # Usage: # # User-agent: Specify robot by name. # Data format: "robot-name" # Also accepts the wildcard "*" which applies to all robots. # # Crawl-delay: Set the number of seconds between sucessive crawl requests. # Data format: "10" # # Request-rate: Set the maximum request rate in pages every amount of seconds. # Data format: "1/5" # # Visit-time: Set the time period robots are allow to vist. # Data format: "0600-1200" # Times are in UTC 24-hour format. # # Allow: Allow specified directory or file. # Data format: "/directory/" (must end with trailing slash) or "/directory/file.htm" # These entries must be first, if used together with "Disallow". # # Disallow: Disallow specified directory or file. # Data format: "/directory/" (must end with trailing slash) or "/directory/file.htm" # These entries must be second, if used together with "Allow". # User-agent: * Crawl-delay: 5 Request-rate: 1/1 # # Last updated: October 20, 2009 (10/20/09) #