Build 242 - Updated config to call gobuster with slash and without slash for best results.

Author: sethsec

celerystalk

@@ -19,6 +19,7 @@ Usage:
     celerystalk db ([workspaces]|[workspace]|[services]|[ports]|[hosts]|[vhosts]|[paths]|[paths_only]|[tasks]) [-h]
     celerystalk db export [-h]
     celerystalk db paths_only limit [-h]
+    celerystalk db paths_include_404s [-h]
     celerystalk admin ([start]|[stop]|[restart]|[reset]|[backup]|[restore]) [-f <restore_file>] [-h]
     celerystalk interactive [-h]
     celerystalk (help | -h | --help)
@@ -89,7 +90,7 @@ import csv
 
 from lib.nmap import nmapcommand
 
-build=str(243)
+build=str(244)
 
 
 def print_banner():
@@ -611,7 +612,8 @@ def main(arguments):
                                 ./celerystalk db ports
     Show hosts:                 ./celerystalk db hosts
     Show vhosts only            ./celerystalk db vhosts
-    Show paths:                 ./celerystalk db paths
+    Show paths (404s excluded): ./celerystalk db paths
+    Show paths & include 404s   ./celerystalk db paths_include_404s
     Show paths (no table)       ./celerystalk db paths_only     
     Show tasks:                 ./celerystalk db tasks
     Export tables to csv        ./celerystalk db export
@@ -679,15 +681,29 @@ def main(arguments):
                 print(services_table)
                 print("\n\n")
             elif arguments["paths"]:
-                print("[+] Showing paths for the [{0}] workspace\n".format(workspace))
-                columns = ["IP", "Port", "Path"]
+                print("[+] Showing paths for the [{0}] workspace (excluding 404s)\n    To show table with 404s use:\n        ./celerystalk db paths_include_404s\n".format(workspace))
+                columns = ["IP", "Port", "Path","Status"]
+                paths_rows = lib.db.get_all_paths_exclude_404(workspace)
+                paths_table = PrettyTable(columns)
+                paths_table.align[columns[0]] = "l"
+                paths_table.align[columns[1]] = "l"
+                paths_table.align[columns[2]] = "l"
+                paths_table.align[columns[3]] = "l"
+                for row in paths_rows:
+                    paths_table.add_row(row[1:5])
+                print(paths_table)
+                print("\n\n")
+            elif arguments["paths_include_404s"]:
+                print("[+] Showing all paths for the [{0}] workspace\n".format(workspace))
+                columns = ["IP", "Port", "Path","Status"]
                 paths_rows = lib.db.get_all_paths(workspace)
                 paths_table = PrettyTable(columns)
                 paths_table.align[columns[0]] = "l"
                 paths_table.align[columns[1]] = "l"
                 paths_table.align[columns[2]] = "l"
+                paths_table.align[columns[3]] = "l"
                 for row in paths_rows:
-                    paths_table.add_row(row[1:4])
+                    paths_table.add_row(row[1:5])
                 print(paths_table)
                 print("\n\n")
             elif arguments["paths_only"]:
@@ -698,7 +714,7 @@ def main(arguments):
                     for path in paths:
                         print path
                 else:
-                    paths_rows = lib.db.get_all_paths(workspace)
+                    paths_rows = lib.db.get_all_paths_exclude_404(workspace)
                     for row in paths_rows:
                         sys.stdout.write(row[3]+"\n")
                 print("\n")
@@ -752,6 +768,7 @@ def main(arguments):
                 print("[+] Saved all paths in the [{0}] workspace to {1}".format(workspace,paths_output_file))
                 paths_filename = workspace + "_paths.txt"
                 paths_output_file = os.path.join(output_dir, paths_filename)
+                paths_rows = lib.db.get_all_paths_exclude_404(workspace)
                 with open(paths_output_file, 'wb') as f:
                     for path in paths_rows:
                         f.write(path[3] + "\n")

lib/csimport.py

@@ -237,7 +237,7 @@ def import_url(url,workspace,output_base_dir):
                     url_path = ''
 
                 url_screenshot_filename = scan_output_base_file_dir + url_path.replace("/", "_") + ".png"
-                db_path = (vhost, port, url.rstrip("/"), 0, url_screenshot_filename, workspace)
+                db_path = (vhost, port, url.rstrip("/"), 0, 0, url_screenshot_filename,workspace)
                 db.insert_new_path(db_path)
                 # print("Found Url: " + str(url))
                 #urls_to_screenshot.append((url, url_screenshot_filename))
@@ -246,7 +246,7 @@ def import_url(url,workspace,output_base_dir):
                 # print(result)
 
 
-            db_path = (vhost, port, url.rstrip("/"), 0, url_screenshot_filename, workspace)
+            db_path = (vhost, port, url.rstrip("/"), 0, 0, url_screenshot_filename, workspace)
             lib.db.insert_new_path(db_path)
     else:
         print("[!] {0} is explicitly marked as out of scope. Skipping...".format(vhost))
@@ -527,7 +527,7 @@ def process_nmap_data(nmap_report,workspace, target=None):
                     db_path = db.get_path(path, workspace)
                     if not db_path:
                         url_screenshot_filename = scan_output_base_file_dir + ".png"
-                        db_path = (ip, scanned_service_port, path, 0, url_screenshot_filename, workspace)
+                        db_path = (ip, scanned_service_port, path, 0, 0, url_screenshot_filename, workspace)
                         db.insert_new_path(db_path)
 
 
@@ -558,7 +558,7 @@ def process_nmap_data(nmap_report,workspace, target=None):
                         db_path = db.get_path(path, workspace)
                         if not db_path:
                             url_screenshot_filename = scan_output_base_file_dir + ".png"
-                            db_path = (vhost, scanned_service_port, path, 0, url_screenshot_filename, workspace)
+                            db_path = (vhost, scanned_service_port, path, 0, 0, url_screenshot_filename, workspace)
                             db.insert_new_path(db_path)
 
 

lib/db.py

@@ -97,8 +97,9 @@ def create_path_table():
                                         ip text NOT NULL,
                                         port int NOT NULL,
                                         path text NOT NULL UNIQUE,
+                                        url_status int,
                                         submitted int,
-                                        url_screenshot_filename text,
+                                        url_screenshot_filename text,                                        
                                         workspace text NOT NULL
                                     ); """
 
@@ -639,31 +640,37 @@ def insert_new_path(db_path):
     :param db_path:
     :return:
     """
-    sql = '''INSERT OR IGNORE INTO paths(ip,port,path,submitted,url_screenshot_filename,workspace)
-              VALUES(?,?,?,?,?,?)  '''
+    sql = '''INSERT OR IGNORE INTO paths(ip,port,path,url_status,submitted,url_screenshot_filename,workspace)
+              VALUES(?,?,?,?,?,?,?)  '''
     CUR.execute(sql, db_path)
     CONNECTION.commit()
 
 def get_all_paths(workspace):
-    CUR.execute("SELECT * FROM paths WHERE workspace = ? ORDER BY ip,port,path", (workspace,))
+    CUR.execute("SELECT * FROM paths WHERE workspace = ? ORDER BY ip,port,path,url_status", (workspace,))
     all_paths = CUR.fetchall()
     CONNECTION.commit()
     return all_paths
 
-def get_all_paths_for_host(ip):
-    CUR.execute("SELECT ip,port,path,url_screenshot_filename,workspace FROM paths WHERE ip = ? ORDER BY port,path", (ip,))
+def get_all_paths_exclude_404(workspace):
+    CUR.execute("SELECT * FROM paths WHERE workspace = ? AND url_status != 404 ORDER BY ip,port,path,url_status", (workspace,))
+    all_paths = CUR.fetchall()
+    CONNECTION.commit()
+    return all_paths
+
+def get_all_paths_for_host_exclude_404(ip):
+    CUR.execute("SELECT ip,port,path,url_screenshot_filename,workspace FROM paths WHERE ip = ? AND url_status != 404 ORDER BY port,path", (ip,))
     all_paths_for_host = CUR.fetchall()
     CONNECTION.commit()
     return all_paths_for_host
 
 def get_all_paths_for_host_path_only(ip,workspace):
-    CUR.execute("SELECT path FROM paths WHERE ip = ? AND workspace = ?", (ip,workspace))
+    CUR.execute("SELECT path FROM paths WHERE ip = ? AND workspace = ? AND url_status != 404", (ip,workspace))
     all_paths_for_host = CUR.fetchall()
     CONNECTION.commit()
     return all_paths_for_host
 
 def get_x_paths_for_host_path_only(ip,workspace,config_max):
-    CUR.execute("SELECT path FROM paths WHERE ip = ? AND workspace = ? LIMIT ?", (ip,workspace,config_max))
+    CUR.execute("SELECT path FROM paths WHERE ip = ? AND workspace = ? AND AND url_status != 404 LIMIT ?", (ip,workspace,config_max))
     all_paths_for_host = CUR.fetchall()
     CONNECTION.commit()
     return all_paths_for_host

lib/report.py

@@ -381,7 +381,7 @@ def report(workspace,config_file,target_list=None):
         combined_report_file.write("\n<br>" + services_table_html + "\n<br>")
         combined_report_file.write("\n</div>")
 
-        all_paths = lib.db.get_all_paths_for_host(vhost)
+        all_paths = lib.db.get_all_paths_for_host_exclude_404(vhost)
         #print(str(all_paths))
         #print(len(all_paths))
         if len(all_paths) > 0:

lib/screenshot.py

@@ -41,7 +41,7 @@ def screenshot_command(arguments):
 
     # lib.screenshot.screenshot_all_paths(workspace)
     #TODO: change this to reflect number of screenshots taken based on config.ini max
-    paths_len = len(lib.db.get_all_paths(workspace))
+    paths_len = len(lib.db.get_all_paths_exclude_404(workspace))
     max_paths_len = len(get_max_screenshots(workspace,config_file))
     max = lib.config_parser.get_screenshot_max(config_file)
     print("[+]\n[+] There are [{0}] paths in the DB").format(str(paths_len))
@@ -65,7 +65,7 @@ def aquatone_all_paths(workspace,simulation=None,config_file=None):
     #print("in aquatone all_paths")
     urls_to_screenshot = []
     #TODO: Instead of just grabbing all paths here, maybe add some logic to see if only new paths should be scanned or something. at a minimum, as they are grabbed, we need to update the "screenshot taken" column and put the auatone directory or something like that.
-    paths = lib.db.get_all_paths(workspace)
+    paths = lib.db.get_all_paths_exclude_404(workspace)
     celery_path = lib.db.get_current_install_path()[0][0]
     outdir = lib.db.get_output_dir_for_workspace(workspace)[0][0]
     outdir = os.path.join(outdir,'celerystalkReports/aquatone/')

parsers/generic_urlextract.py

@@ -63,29 +63,29 @@ def is_url_in_scope(url):
     else:
         return str(False)
 
-def insert_url_into_db(vhost,port,url,workspace):
-    db_path = (vhost, port, url, 0, "", workspace)
+def insert_url_into_db(vhost,port,url,url_status, workspace):
+    db_path = (vhost, port, url, url_status, 0, "", workspace)
     lib.db.insert_new_path(db_path)
-    print("Found Url: " + str(url))
 
 def extract_in_scope_urls_from_task_output(tool_output):
     urls = extract_urls(tool_output)
+    valid_url_count = 0
     for url in urls:
-        exists,vhost,port,url,workspace = is_url_in_scope(url)
-        if exists == "True":
+        is_in_scope,vhost,port,url,workspace = is_url_in_scope(url)
+        if is_in_scope == "True":
             url_status = check_if_page_exists(url)
-            if url_status:
-                insert_url_into_db(vhost, port, url, workspace)
+            print(url,url_status)
+            if url_status != 999:
+                insert_url_into_db(vhost, port, url, url_status, workspace)
+                valid_url_count += 1
+    return valid_url_count
+
+
 
 def check_if_page_exists(url):
     try:
         response = requests.head(url, timeout=5, verify=False)
         status_code = response.status_code
-        reason = response.reason
     except requests.exceptions.ConnectionError:
         status_code = 999
-        reason = 'ConnectionError'
-    if status_code == 200:
-        return status_code
-    else:
-        print("Skipping Url (not found): " + str(url))
+    return status_code