User-agent: * Disallow: /git Disallow: /gitweb Disallow: /static Disallow: /admin Disallow: /attic Disallow: /debian/pool User-Agent: AhrefsBot Crawl-Delay: 35 User-agent: SemrushBot Crawl-delay: 40