User-agent: * Disallow: /crawl Disallow: /ftp # I do not have a sitemap, I do not want a sitemap, and I'm not # interested in traffic from anyone who blindly assumes I do. Sitemap: http://127.0.0.1/dev/null