User-agent: * # applies to all robots Disallow: /shop/checkout # disallow shop checkout indexing Disallow: /estimate # disallow shop checkout indexing User-agent: Slurp # throttle Yahoo bot Crawl-delay: 10