Refactor last scripts, logging, central directory for downloads

* Refactored generate_moz-top50.py * Download all file to new /tmp file to centralize all downloads * Add central logging to generators.log file * Create Bash script that generates all warninglists * Add /tmp folder and extra files to .gitignore * Start adding exception handling in download_to_file and write_to_file
MISP · Jul 27, 2020 · a9c163e · a9c163e
1 parent 610292e
commit a9c163e
Show file tree

Hide file tree

Showing 28 changed files with 24,766 additions and 25,065 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 # files download by generators
+tmp/*
 alexa_top-1m.csv.zip
 amazon_ip-ranges.json
 cisco_top-1m.csv.zip
@@ -18,3 +19,5 @@ top500.pages.csv
 top-1m.csv.zip
 public-dns-nameservers.csv
 ms-azure.json
+moz-top500.pages.csv
+moz-top500.domains.csv
diff --git a/generate_all.sh b/generate_all.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+set -x
+
+pushd tools
+python3 generate_alexa.py
+python3 generate-amazon-aws.py
+python3 generate-cisco.py
+python3 generate-cloudflare.py
+python3 generate-covid.py
+python3 generate-crl-ip-list.py
+python3 generate-disposal.py
+# TODO: Google page on Wikipedia does not exist anymore
+# Suggestion came to use a passivetotal whois search for org:Google LLC
+#python3 generate-google.py > lists/google/list.json
+python3 generate_majestic-million.py
+python3 generate-microsoft-azure.py
+python3 generate_mozilla_certificates.py
+python3 generate_moz-top500.py
+# Deprecate?
+#python3 generate-office365-cn.py > lists/microsoft-office365-cn/list.json
+python3 generate-office365.py
+python3 generate-publicdns.py
+python3 generate-tlds.py
+python3 generate_tranco.py
+python3 generate-university-domain-list.py
+python3 generate-vpn.py
+python3 generate-wikimedia.py
+popd
+
+./jq_all_the_things.sh
diff --git a/lists/amazon-aws/list.json b/lists/amazon-aws/list.json
@@ -134,6 +134,10 @@
     "13.34.23.32/27",
     "13.34.23.64/27",
     "13.34.23.96/27",
+    "13.34.24.128/27",
+    "13.34.24.160/27",
+    "13.34.24.192/27",
+    "13.34.24.96/27",
     "13.35.0.0/16",
     "13.48.0.0/15",
     "13.48.186.128/27",
@@ -164,6 +168,7 @@
     "140.179.0.0/16",
     "143.204.0.0/16",
     "144.220.0.0/16",
+    "15.152.0.0/16",
     "15.161.0.0/16",
     "15.161.135.164/30",
     "15.164.0.0/15",
@@ -226,6 +231,9 @@
     "15.221.20.0/22",
     "15.221.24.0/21",
     "15.221.3.0/24",
+    "15.221.33.0/24",
+    "15.221.34.0/24",
+    "15.221.35.0/24",
     "15.221.4.0/23",
     "15.221.8.0/21",
     "15.222.0.0/15",
@@ -382,6 +390,7 @@
     "15.230.40.0/24",
     "15.230.41.0/24",
     "15.230.42.0/24",
+    "15.230.43.0/24",
     "15.230.5.0/24",
     "15.230.6.0/24",
     "15.230.64.0/26",
@@ -414,6 +423,22 @@
     "150.222.118.0/24",
     "150.222.119.0/24",
     "150.222.12.0/24",
+    "150.222.120.20/31",
+    "150.222.120.224/31",
+    "150.222.120.226/31",
+    "150.222.120.228/31",
+    "150.222.120.230/31",
+    "150.222.120.232/31",
+    "150.222.120.234/31",
+    "150.222.120.240/31",
+    "150.222.120.242/31",
+    "150.222.120.244/31",
+    "150.222.120.246/31",
+    "150.222.120.248/31",
+    "150.222.120.250/31",
+    "150.222.120.252/32",
+    "150.222.120.255/32",
+    "150.222.120.62/31",
     "150.222.121.0/24",
     "150.222.122.100/31",
     "150.222.122.102/31",
@@ -494,6 +519,7 @@
     "150.222.222.0/24",
     "150.222.223.0/24",
     "150.222.224.0/24",
+    "150.222.226.0/24",
     "150.222.227.0/24",
     "150.222.228.0/24",
     "150.222.229.0/24",
@@ -545,6 +571,10 @@
     "150.222.244.35/32",
     "150.222.244.37/32",
     "150.222.245.122/31",
+    "150.222.3.176/32",
+    "150.222.3.177/32",
+    "150.222.3.178/32",
+    "150.222.3.179/32",
     "150.222.5.0/24",
     "150.222.6.0/24",
     "150.222.66.0/24",
@@ -623,7 +653,6 @@
     "176.34.64.0/18",
     "177.71.128.0/17",
     "177.71.207.128/26",
-    "177.71.207.16/29",
     "177.72.240.0/21",
     "178.236.0.0/20",
     "18.130.0.0/16",
@@ -1009,11 +1038,42 @@
     "2600:1ffc:8000::/40",
     "2600:1ffc:c000::/40",
     "2600:1ffc:e000::/40",
+    "2600:1ffd:807b::/48",
     "2600:1ffd:807f::/48",
+    "2600:1ffd:80c8::/48",
     "2600:1ffd:80cb::/48",
+    "2600:1ffd:80d0::/48",
     "2600:1ffd:80e1::/48",
+    "2600:1ffd:80e8::/48",
+    "2600:1ffd:80f0::/48",
+    "2600:1ffd:8143::/48",
+    "2600:1ffd:8149::/48",
+    "2600:1ffd:814b::/48",
+    "2600:1ffd:8152::/48",
+    "2600:1ffd:8165::/48",
+    "2600:1ffd:8166::/48",
+    "2600:1ffd:816c::/48",
+    "2600:1ffd:8188::/48",
+    "2600:1ffd:818b::/48",
+    "2600:1ffd:818f::/48",
     "2600:1ffd:8190::/48",
-    "2600:1ffd:857f::/48",
+    "2600:1ffd:8195::/48",
+    "2600:1ffd:819f::/48",
+    "2600:1ffd:81a0::/48",
+    "2600:1ffd:81a3::/48",
+    "2600:1ffd:81a5::/48",
+    "2600:1ffd:81a7::/48",
+    "2600:1ffd:81ab::/48",
+    "2600:1ffd:81ad::/48",
+    "2600:1ffd:81bb::/48",
+    "2600:1ffd:81be::/48",
+    "2600:1ffd:84a0::/48",
+    "2600:1ffd:84a4::/48",
+    "2600:1ffd:84ab::/48",
+    "2600:1ffd:84ae::/48",
+    "2600:1ffd:84af::/48",
+    "2600:1ffd:84b4::/48",
+    "2600:1ffd:84bd::/48",
     "2600:1ffe:1000::/40",
     "2600:1ffe:2000::/40",
     "2600:1ffe:4000::/40",
@@ -1216,7 +1276,14 @@
     "3.216.144.0/23",
     "3.216.148.0/22",
     "3.217.228.0/22",
-    "3.218.180.0/22",
+    "3.218.180.0/25",
+    "3.218.180.128/25",
+    "3.218.181.0/25",
+    "3.218.181.128/25",
+    "3.218.182.0/25",
+    "3.218.182.128/25",
+    "3.218.183.0/25",
+    "3.218.183.128/25",
     "3.224.0.0/12",
     "3.227.250.128/25",
     "3.231.2.0/25",
@@ -2423,7 +2490,8 @@
     "99.83.64.0/21",
     "99.83.98.0/24",
     "99.84.0.0/16",
-    "99.86.0.0/16"
+    "99.86.0.0/16",
+    "99.87.0.0/22"
   ],
   "matching_attributes": [
     "ip-src",
@@ -2432,5 +2500,5 @@
   ],
   "name": "List of known Amazon AWS IP address ranges",
   "type": "cidr",
-  "version": 20200721
+  "version": 20200727
 }