diff --git a/lib/robots.rb b/lib/robots.rb index 2ea334e..c72d734 100644 --- a/lib/robots.rb +++ b/lib/robots.rb @@ -26,19 +26,19 @@ def initialize(uri, user_agent) io.each do |line| next if line =~ /^\s*(#.*|$)/ arr = line.split(":") - key = arr.shift + key = arr.shift.to_s.downcase value = arr.join(":").strip value.strip! case key - when "User-agent" + when "user-agent" agent = to_regex(value) - when "Allow" + when "allow" @allows[agent] ||= [] @allows[agent] << to_regex(value) - when "Disallow" + when "disallow" @disallows[agent] ||= [] @disallows[agent] << to_regex(value) - when "Crawl-delay" + when "crawl-delay" @delays[agent] = value.to_i else @other[key] ||= [] diff --git a/test/fixtures/mixcase.txt b/test/fixtures/mixcase.txt new file mode 100644 index 0000000..4ad73fb --- /dev/null +++ b/test/fixtures/mixcase.txt @@ -0,0 +1,4 @@ +User-agent: * +Disallow: /test +User-Agent: Another +Disallow: / \ No newline at end of file diff --git a/test/test_robots.rb b/test/test_robots.rb index a86baae..7c05961 100644 --- a/test/test_robots.rb +++ b/test/test_robots.rb @@ -49,10 +49,15 @@ def test_site_with_disallowed end def test_other_values - sitemap = {"Sitemap" => ["http://www.eventbrite.com/sitemap_index.xml", "http://www.eventbrite.com/sitemap_index.xml"]} + sitemap = {"sitemap" => ["http://www.eventbrite.com/sitemap_index.xml", "http://www.eventbrite.com/sitemap_index.xml"]} assert_other_equals("eventbrite", sitemap) end + def test_mix_case_user_agent + assert_allowed("mixcase", "/") + assert_disallowed("mixcase", "/test") + end + def assert_other_equals(name, value) assert_equal(value, @robots.other_values(uri_for_name(name, "/"))) end