diff --git a/docs/_config.yml b/docs/_config.yml index 0b28991a6b..8cebbba0a3 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -32,6 +32,11 @@ defaults: path: "" values: layout: default + # exclude all the files from `/releases/` in the sitemap + - scope: + path: releases/** + values: + sitemap: false exclude: - 'ADDING_RELEASE_DOCS.md' diff --git a/docs/robots.txt b/docs/robots.txt new file mode 100644 index 0000000000..10ba859b73 --- /dev/null +++ b/docs/robots.txt @@ -0,0 +1,7 @@ +# This robots.txt file controls crawling of URLs under https://bnd.bndtools.org. +# All crawlers are disallowed to crawl files in the "releases" directory, +# because they dilute search results, because they are outdated. +User-agent: * +Disallow: /releases/ + +Sitemap: https://bnd.bndtools.org/sitemap.xml \ No newline at end of file