# see http://tool.motoricerca.info/robots-checker.phtml validator # remove from archive.org (wayback machine) User-agent: ia_archiver Disallow: / # Allow all robots in User-agent: * Disallow: Crawl-delay: 30 # remove from aboutUs: User-agent: AboutUsBot Disallow: /