User-agent: *

Disallow: /user/
Disallow: /*/user/
Disallow: /search/
Disallow: /*/search/
Disallow: /occurrence/
Disallow: /*/occurrence/
Allow: /occurrence/download/*
Allow: /*/occurrence/download/*
Disallow: /species/search
Disallow: /*/species/search
Disallow: /dataset/search
Disallow: /*/dataset/search
Disallow: /publisher/search
Disallow: /*/publisher/search
Disallow: /grscicoll/collection/search
Disallow: /*/grscicoll/collection/search
Disallow: /grscicoll/institution/search
Disallow: /*/grscicoll/institution/search
Disallow: /grscicoll/person/search
Disallow: /*/grscicoll/person/search
crawl-delay: 0.1

User-agent: Twitterbot
Allow: /occurrence/
Disallow: /*/occurrence/
Disallow: /*/occurrence/search

Sitemap: https://www.gbif.org/sitemap.xml
Sitemap: https://www.gbif.org/sitemap-dataset.xml
Sitemap: https://www.gbif.org/sitemap-publisher.xml
Sitemap: https://www.gbif.org/sitemap-node.xml
Sitemap: https://www.gbif.org/sitemap-network.xml
Sitemap: https://www.gbif.org/sitemap-installation.xml
Sitemap: https://www.gbif.org/sitemap-species.xml