[Launch test] [Maintenance] Update of the repository structure

.gitignore

@@ -103,3 +103,7 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+
+# Container
+whitelisting.py

.travis.yml

@@ -1,10 +1,13 @@
 env:
   global:
-    - secure: StNY8sT9988Ew92wW3QAUnQcSA7+SkkWdPcKDi8v3DtRI/adVC4oVS7DzFYxPN5tpQfZljJ23WN2oWg9GNcFfkZ9hQkceHLMnl7Y4NPbgp169AmGTNqOfbsprHRbU+ps5kI7kr+fRhZJJyq66ivdpiLwlO+T6+KxWg6uf+1RgdmcIfEd+clS3iW5wKC/1b+e784bxyx9rj+SlSvzTjxy2Y5HmTolBciv8foKzO6WnNLZy3pJWA70u22NucUZJJRPZ2+Xte9/H24ksNOP/IPKlvGrX3CTQBVWaC5tFmBVUuWOTfwBiXgmWVSD3eEMtBmohlXTUvMb3aw8mHf+OGk07ODguF2ZSMRr68a6FVd4iCkVQQ1vODHoNseCJrwhZD2C7qeeLRK1D7DE5oVPpsnl5VyfLgfisbJEmyI0u/wWdmSwvW8hIkrmmq/mHDc4eLRoAiALfz2lgzJdKJGf8C3L/x2QQMhgzAwJUAp5QPzua8DS2ml298gJGePT3gQ5KhjWgiGljY5Q6W3LfU56pic+VrPclWHn/uuZa+Z8MX4BjyNCTQ9Kb3pAueKJpTQKK83jYEmfC5yIcN3vHajjfxq8GI+oB32eyuDeHDZKjhb+YLIT5d2R4+BecOIPb6/+a8B9Sdj3NS/J4tgfwk2s7YzoRcExClg6Hgz+pfKNRiSanUQ=
+    - secure: Fxek3lFNPq4gXEl0jdah5e/Kbgkt+sivwSHS6/YyjGgxapBtAAOHI7SO69EWr1KZLgH8P/MyBOXrH+sZel6o8lA1OIFwt9QL8zY5EnMCV5aG4OBP+naTHiOBnE0AzRKsCmtds0CRIy1oQ00WecmrLK3ZfMFDM6nrxpjxO8nbpndMRVVD3Jcpw3Dzac99OIot9qPiEBUq8oczlBE3+jz867BWPxM0wRcVeq39mV/Ve3lgXiZwg/AmVFkA+dJxoKWDEQpcc7NnmdEnlYlr/M+ys1GMJsx9QDQegYocCTiVF8mamrnHWU4TqfHWI8ViABkbbDsKobK4fUDRURqNOvAFBAnK4ibwDwnn3b8sU6VmWZZHxgMpZslDD9v19rB2Se6tWqNc0uwq2Sg6x9i2qz7GQh6xO8+DDCwtBaoRyco5OcGIrQECtnma5ayjjD8PX6gxhaH3HQiayHFQzOwW2ZEnmvu7Gxb2nJg3VlOXchh5QDFpoTC7y9C2zeaSWZHGom9HbGZm9x5jJ/iTeZdHTIsMPy/ojs5NJuZNLUnL1E5dCAUzzNhGIcJWMa4hz5EvI+/qrqIfcVP/6SbKFm10hlajkxITJTL1vxHvOHhlXPlzhNsJd0fDiWUApSpq+uMbQAuM12wnWtg+Rn+HT3JmlIbCkvdX2wCWdmuLpyfwZwab6Rw=
     - GIT_NAME: Travis CI
     - GIT_EMAIL: [email protected]
     - TRAVIS_REPO_SLUG: Ultimate-Hosts-Blacklist/Phishing.Database
     - GIT_BRANCH: master
+    - WHITELISTING_LOCATION: https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/dev-center/whitelisting/whitelisting.py
+    - UPDATE_LOCATION: https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/repository-structure/master/update.py
+    - ADMINISTATION_LOCATION: https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/repository-structure/master/administration.py
 
 dist: xenial
 sudo: required
@@ -27,10 +30,13 @@ addons:
 install:
   - pip3 install --upgrade pip
   - pip3 install requests
+  - wget "${WHITELISTING_LOCATION}" -o /dev/null -O whitelisting.py && chmod +x whitelisting.py
+  - if [[ ! -f .admin ]]; then wget "${UPDATE_LOCATION}" -o /dev/null -O update.py && chmod +x update.py; else echo "No need to download/update update.py."; fi
+  - if [[ ! -f .admin ]]; then wget "${ADMINISTATION_LOCATION}" -o /dev/null -O administration.py && chmod +x administration.py; else echo "No need to download/update administration.py."; fi
 
 script:
   - python3 ./update.py
 
 notifications:
-  on_success:   change
-  on_failure:   always
+  on_success: change
+  on_failure: always

.trigger

@@ -1 +1 @@
-1543607196
+1547242105

administration.py

@@ -15,6 +15,7 @@
     @GitHubUsername, Name, Email (optional)
 """
 from update import Helpers, Settings, path, strftime
+from whitelisting import Whitelist
 
 INFO = {}
 
@@ -28,7 +29,7 @@ def get_administration_file():
         content = Helpers.File(Settings.repository_info).read()
         INFO.update(Helpers.Dict().from_json(content))
     else:
-        raise Exception("Unabel to find the administration file.")
+        raise Exception("Unable to find the administration file.")
 
 
 def update_adminisation_file():
@@ -47,16 +48,16 @@ def save_administration_file():
     Helpers.Dict(INFO).to_json(Settings.repository_info)
 
 
-def generate_clean_list():
+def generate_clean_and_whitelisted_list():
     """
-    Update `clean.list`.
+    Update/Create `clean.list` and `whitelisted.list`.
     """
 
     if bool(int(INFO["clean_original"])):
         clean_list = []
 
         list_special_content = Helpers.Regex(
-            Helpers.File(Settings.file_to_test + INFO["list_name"]).to_list(), r"ALL\s"
+            Helpers.File(Settings.file_to_test + Settings.list_name).to_list(), r"ALL\s"
         ).matching_list()
 
         active = Settings.current_directory + "output/domains/ACTIVE/list"
@@ -68,15 +69,20 @@ def generate_clean_list():
             )
 
         clean_list = Helpers.List(clean_list).format()
+        whitelisted = Whitelist(string="\n".join(clean_list)).get()
 
         Helpers.File(Settings.clean_list_file).write(
             "\n".join(clean_list), overwrite=True
         )
 
+        Helpers.File(Settings.whitelisted_list_file).write(whitelisted, overwrite=True)
+
+        Helpers.File("whitelisting.py").delete()
+
 
 if __name__ == "__main__":
     get_administration_file()
     update_adminisation_file()
     save_administration_file()
 
-    generate_clean_list()
+    generate_clean_and_whitelisted_list()

info.json

@@ -6,7 +6,7 @@
     "days_until_next_test": "2",
     "last_test": "1547235378",
     "list_name": "domains.list",
-    "name": "Domain Status Testing for Ultimate Hosts Blacklist",
+    "name": "Phishing.Database",
     "raw_link": "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-ACTIVE.txt",
     "stable": "1"
 }

update.py

@@ -61,10 +61,23 @@ class Settings:  # pylint: disable=too-few-public-methods
     #    - [email protected]
     #
     # Note: The variable name should not be changed.
-    # Note: This variable is auto updated by Initiate()
     #
     # Example: "[email protected]"
-    list_name = ""
+    list_name = "domains.list"
+
+    # This variable is used to set the location of the file for the list without
+    # dead/inactive domains.
+    #
+    # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS!
+    # Note: This variable is auto updated by Initiate()
+    clean_list_file = "clean.list"
+
+    # This variable is used to set the location of the file for the list without
+    # dead/inactive domains and whitelisted domains.
+    #
+    # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS!
+    # Note: This variable is auto updated by Initiate()
+    whitelisted_list_file = "whitelisted.list"
 
     # This variable will help us know where we are working into the filesystem.
     #
@@ -108,7 +121,7 @@ class Settings:  # pylint: disable=too-few-public-methods
     # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS!
     PyFunceble = {
         ".PyFunceble_production.yaml": "https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml",  # pylint: disable=line-too-long
-        ".PyFunceble_LICENSE": "https://raw.githubusercontent.com/funilrys/PyFunceble/dev/LICENSE",
+        ".PyFunceble_LICENSE": "https://raw.githubusercontent.com/funilrys/PyFunceble/master/LICENSE",  # pylint: disable=line-too-long
     }
 
     # This variable is used to match [ci skip] from the git log.
@@ -152,13 +165,6 @@ class Settings:  # pylint: disable=too-few-public-methods
     # Note: This variable is auto updated by Initiate()
     clean_original = False
 
-    # This variable is used to set the location of the file for the list without
-    # dead/inactive domains.
-    #
-    # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS!
-    # Note: This variable is auto updated by Initiate()
-    clean_list_file = "clean.list"
-
     # This variable is used to know if we need to convert the list into idna
     # before we write the file PyFunceble is going to test.
     #
@@ -279,7 +285,7 @@ def _fix_cross_repo_config(cls):
                 r"travis:.*": "travis: True",
                 r"travis_autosave_commit:.*": 'travis_autosave_commit: "[Autosave] Testing for Ultimate Hosts Blacklist"',  # pylint: disable=line-too-long
                 r"travis_autosave_final_commit:.*": 'travis_autosave_final_commit: "[Results] Testing for Ultimate Hosts Blacklist"',  # pylint: disable=line-too-long
-                r"travis_branch:.*": "travis_branch: master",
+                r"travis_branch:.*": "travis_branch: %s" % environ['GIT_BRANCH'],
                 r"travis_autosave_minutes:.*": "travis_autosave_minutes: %s"
                 % Settings.autosave_minutes,
             }
@@ -608,6 +614,11 @@ def PyFunceble(self):  # pylint: disable=invalid-name
             Settings.informations.update(
                 {"last_test": strftime("%s"), "currently_under_test": str(int(True))}
             )
+
+            for index in ['clean_list_file', 'list_name']:
+                if index in Settings.informations:
+                    del Settings.informations[index]
+
             Helpers.Dict(Settings.informations).to_json(Settings.repository_info)
 
             Helpers.Command(command_to_execute, True).execute()