User-agent: * Disallow: /download.php #no bot hits to downloads Disallow: /404.php Disallow: /index.php?page=/404.php Disallow: /downloads/index.php Disallow: /images/index.php Disallow: /cgi-bin/ #User-agent: Slurp #Crawl-delay: 10