User-agent: * Crawl-delay: 10 # Directories # Files # Paths (clean URLs) Disallow: /usage/ Disallow: /cgi-bin/ Disallow: /domain/ # Paths (no clean URLs)