# robots.txt for doi.pangaea.de User-agent: * Disallow: User-agent: Slurp Disallow: /10.1594/PANGAEA.*? Allow: / User-agent: Googlebot Disallow: /10.1594/PANGAEA.*? Allow: / Sitemap: http://doi.pangaea.de/sitemap.xml