clone of github.com/decent-newsroom/newsroom
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
|
# Robots.txt |
|
|
|
# Disallow all by default |
|
User-agent: * |
|
Disallow: / |
|
|
|
# Allow only specific public content |
|
Allow: /assets/ |
|
Allow: /icons/ |
|
Allow: /$ |
|
Allow: /unfold |
|
Allow: /about |
|
Allow: /mag/ |
|
Allow: /p/ |
|
Allow: /list/ |
|
|
|
# Crawl delay |
|
Crawl-delay: 1
|
|
|
