Sitemap: http://dash.harvard.edu/sitemap

User-agent: *
Disallow: /admin
Disallow: /advanced-search
Disallow: /browse
Disallow: /community-list
Disallow: /feed
Disallow: /forgot
Disallow: /login
Disallow: /person
Disallow: /register
Disallow: /search
Disallow: /*/*.pdf.txt*
Disallow: /*/*.doc.txt*
Disallow: /*/*.html.txt*
Disallow: /sword
Disallow: /swordv2

User-agent: Googlebot
Disallow: /admin
Disallow: /advanced-search
Disallow: /browse
Disallow: /community-list
Disallow: /feed
Disallow: /forgot
Disallow: /login
Disallow: /register
Disallow: /search
Disallow: /*/advanced-search
Disallow: /*/browse?
Disallow: /*/*.pdf.txt*
Disallow: /*/*.doc.txt*
Disallow: /*/*.html.txt*

User-agent: msnbot
Disallow: /admin
Disallow: /advanced-search
Disallow: /browse
Disallow: /community-list
Disallow: /feed
Disallow: /forgot
Disallow: /login
Disallow: /register
Disallow: /search
Disallow: /*/advanced-search
Disallow: /*/browse?
Disallow: /*/*.pdf.txt*
Disallow: /*/*.doc.txt*
Disallow: /*/*.html.txt*

User-agent: Slurp
Disallow: /admin
Disallow: /advanced-search
Disallow: /browse
Disallow: /community-list
Disallow: /feed
Disallow: /forgot
Disallow: /login
Disallow: /register
Disallow: /search
Disallow: /*/advanced-search
Disallow: /*/browse?
Disallow: /*/*.pdf.txt*
Disallow: /*/*.doc.txt*
Disallow: /*/*.html.txt*