Merge pull request #6469 from chrisrueger/exclude-docs-releases-folder-from-google

docs: exclude /releases urls from search engines

Author: chrisrueger

docs/_config.yml

@@ -32,6 +32,11 @@ defaults:
       path: ""
     values:
       layout: default
+  # exclude all the files from `/releases/` in the sitemap
+  - scope:
+      path:            releases/**
+    values:
+      sitemap:         false
 
 exclude:
   - 'ADDING_RELEASE_DOCS.md'

docs/robots.txt

@@ -0,0 +1,7 @@
+# This robots.txt file controls crawling of URLs under https://bnd.bndtools.org.
+# All crawlers are disallowed to crawl files in the "releases" directory, 
+# because they dilute search results, because they are outdated.
+User-agent: *
+Disallow: /releases/
+
+Sitemap: https://bnd.bndtools.org/sitemap.xml