Merge pull request #5 from kevinmarks/patch-2

handle images without extensions, don't look for links in zip,pdf

Author: aaronpk

spider.js

@@ -116,13 +116,15 @@ function process_link(current) {
       }
 
       var page_url = url.parse(current);
-
-      // Add a slash if the path is not a file (does not end in a slash and does not have a dot)
-      var components = page_url.path.split("/");
-      if(!page_url.path.match(/\/$/) && !components[components.length-1].match(/\./)) {
-        page_url.path += "/";
+      if(response.headers['content-type'] && response.headers['content-type'].match(/image/)) {
+        console.log("not making this a directory: "+response.headers['content-type'])
+      } else {
+        // Add a slash if the path is not a file (does not end in a slash and does not have a dot)
+        var components = page_url.path.split("/");
+        if(!page_url.path.match(/\/$/) && !components[components.length-1].match(/\./)) {
+          page_url.path += "/";
+        }
       }
-
       // Add "index.html" if the path ends in a slash
       if(page_url.path.match(/\/$/)) {
         page_url.path += "index.html";
@@ -158,6 +160,10 @@ function process_link(current) {
         });
       } else if(response.headers['content-type'] && response.headers['content-type'].match(/javascript/)) {
 
+      } else if(response.headers['content-type'] && response.headers['content-type'].match(/zip/)) {
+
+      } else if(response.headers['content-type'] && response.headers['content-type'].match(/pdf/)) {
+
       } else {
         // assume HTML if it's not JS or CSS
         var links = $("a");