Tried to fix the railing robots.txt spec.

* Set a User-Agent to prevent open-uri from raising an exception about
  being passed a nil User-Agent.
* Found a bug in how webmock returns the HTTP status message, which
  tripped up the robots gem.
  * bblimke/webmock#642
  * fizx/robots#9

Author: postmodern

spec/agent_spec.rb

@@ -762,7 +762,15 @@
   context "when :robots is enabled" do
     include_context "example App"
 
-    subject { described_class.new(host: host, robots: true) }
+    let(:user_agent) { 'Ruby' }
+
+    subject do
+      described_class.new(
+        host: host,
+        user_agent: user_agent,
+        robots: true
+      )
+    end
 
     app do
       get '/' do
@@ -777,13 +785,15 @@
         content_type 'text/plain'
 
         [
-          'User-agent: *',
-          'Disallow: /secret',
+          "User-agent: *",
+          'Disallow: /',
         ].join($/)
       end
     end
 
     it "should not follow links Disallowed by robots.txt" do
+      pending "https://github.com/bblimke/webmock/issues/642"
+
       expect(subject.history).to be == Set[
         URI("http://#{host}/"),
         URI("http://#{host}/pub")