diff --git a/.gitignore b/.gitignore index b989be6ca15..0b85796f600 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,7 @@ venv.bak/ # mypy .mypy_cache/ + + +# Container +whitelisting.py \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index fda138ef076..adcd7cde7b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,13 @@ env: global: - - secure: rJU4phUpA9W0sFujoJU7L1DkoqYPOgclzdNgdn5/aJrblDYL73kjAge7EqKiJStVNX5CfGX4/8u/j+7EyK6mMpp9HFkpyRKoZK4GEgRuJWGxFKlIVymXA7Qzl2P94Qv84e/uMF9NGrklFYAwdr6iqgvpmhSFmYMVqhsUNNAn6pSofAChivpMBYTXqMe363/Ws3kAxjqhqIan81onxguTyJS7aHzUXokni5+zae5KrwnwA3awKJmOyzakMjYY89b5qVrF0uCjpAChdQbRacRcCNBCqwZz7Odq0ODWmmqD6IjxzTDCnNrcbnsHYloQws2WhQtzX2BhQXBm9JeV3/gY3dM3jIoGQnau4HgqlG0fxy1RNqxd0WNVhldb0Vbqu7/Kog2tUCK3zFkzUIcBR/uRilYx/UqnqdK8OwNmgq8pTtla1gj+UDBt7HSJ0wo/0OxoLjAXe23qd0Qu2w7xNJ4++Ffkd1M3hXg/461sz33AozGPBNko4dzI7DEXmUTSbuB2ysNodCwtZMnEe5ARcmOx2fBWPkagzSFHm9ucsR/kq01mT75NdCJuhh7FMZjr0LLfmUKbFmpmy2FN3Umtn5A+8nF63lrPJGdD9ZRfzLVo7Ze5rAzrrdHDx4sSm1Ag4lXAVYHV0nxi0xfBWBvRoQY1tGXomD/p0cfO8KeylcxV2rs= + - secure: eCjUTD9HNCwPYJxb7nYAhZz87cpX5k11EssJ4+n0o3ws0odKWWndb7wzOV5U+MMnDleWi1P98PADcM9wQgdmBzydM9uXr48Zj0UYAYOpv1nkkwh1Aqx9gyC9M/x3PSX63xGDKz2yTmuzxpFQ0Av/Ei5e1byASKDo6S30CMvGJVHSHWKNN8ZrrHHRpTXiYDuEo+B1gE4f1z/TvGtqZIahiBTfmxVNsGUykLyGVFlLvefM23mR4KaJMrq5+/mHdXPB9lGPcviVeetgeeqr99/mJrJIxCuf3e35bLtJ/kevmseJCFhx/nJSEFQwZ2dl8VIUERdD0en/N+ZQYUp9IEyHAEHT0+WuF0JnmA+F2aA1BV9Qcw6HqI8/gG67U7qkElL/0vFN8khkV2/acg23lsZigY17JvpH3qzTZJFnQybaGZyk+IitC89+LE5Pld4YFzShRA34exq+VPY1yqS08x2EtQlEMpiPDK8USXSf0zELIctRIpbVEvCXMD8fsQjBPMGDArrls3u9V6RWzzyMdo71JwDeBUrwCDHjvRWZaxm6AZ8TKwt3g/yiQASEvDKmOS69x3uUQtc2sqWIz7YzcV+NSmhQcyOd/p63PHV7lsIw0Qp8iDXTCw3ZMJLBRxstrjawwXYHw/J7IOOnNCia4GS54NFI6FkdIY7Iychz6cXMMZc= - GIT_NAME: Travis CI - GIT_EMAIL: dead-hosts@funilrys.com - TRAVIS_REPO_SLUG: Ultimate-Hosts-Blacklist/justdomains_mirror1.malwaredomains.com - GIT_BRANCH: master + - WHITELISTING_LOCATION: https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/dev-center/whitelisting/whitelisting.py + - UPDATE_LOCATION: https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/repository-structure/master/update.py + - ADMINISTATION_LOCATION: https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/repository-structure/master/administration.py dist: xenial sudo: required @@ -27,10 +30,13 @@ addons: install: - pip3 install --upgrade pip - pip3 install requests + - wget "${WHITELISTING_LOCATION}" -o /dev/null -O whitelisting.py && chmod +x whitelisting.py + - if [[ ! -f .admin ]]; then wget "${UPDATE_LOCATION}" -o /dev/null -O update.py && chmod +x update.py; else echo "No need to download/update update.py."; fi + - if [[ ! -f .admin ]]; then wget "${ADMINISTATION_LOCATION}" -o /dev/null -O administration.py && chmod +x administration.py; else echo "No need to download/update administration.py."; fi script: - python3 ./update.py notifications: - on_success: change - on_failure: always + on_success: change + on_failure: always diff --git a/administration.py b/administration.py index 7433c79c61d..10bbb3c2b4d 100755 --- a/administration.py +++ b/administration.py @@ -15,6 +15,7 @@ @GitHubUsername, Name, Email (optional) """ from update import Helpers, Settings, path, strftime +from whitelisting import Whitelist INFO = {} @@ -28,7 +29,7 @@ def get_administration_file(): content = Helpers.File(Settings.repository_info).read() INFO.update(Helpers.Dict().from_json(content)) else: - raise Exception("Unabel to find the administration file.") + raise Exception("Unable to find the administration file.") def update_adminisation_file(): @@ -47,16 +48,16 @@ def save_administration_file(): Helpers.Dict(INFO).to_json(Settings.repository_info) -def generate_clean_list(): +def generate_clean_and_whitelisted_list(): """ - Update `clean.list`. + Update/Create `clean.list` and `whitelisted.list`. """ if bool(int(INFO["clean_original"])): clean_list = [] list_special_content = Helpers.Regex( - Helpers.File(Settings.file_to_test + INFO["list_name"]).to_list(), r"ALL\s" + Helpers.File(Settings.file_to_test + Settings.list_name).to_list(), r"ALL\s" ).matching_list() active = Settings.current_directory + "output/domains/ACTIVE/list" @@ -68,15 +69,20 @@ def generate_clean_list(): ) clean_list = Helpers.List(clean_list).format() + whitelisted = Whitelist(string="\n".join(clean_list)).get() Helpers.File(Settings.clean_list_file).write( "\n".join(clean_list), overwrite=True ) + Helpers.File(Settings.whitelisted_list_file).write(whitelisted, overwrite=True) + + Helpers.File("whitelisting.py").delete() + if __name__ == "__main__": get_administration_file() update_adminisation_file() save_administration_file() - generate_clean_list() + generate_clean_and_whitelisted_list() diff --git a/info.json b/info.json index e054712c366..fc6eaa398df 100644 --- a/info.json +++ b/info.json @@ -6,7 +6,7 @@ "days_until_next_test": "2", "last_test": "1547486789", "list_name": "domains.list", - "name": "Domain Status Testing for Ultimate Hosts Blacklist", + "name": "justdomains_mirror1.malwaredomains.com", "raw_link": "http://mirror1.malwaredomains.com/files/justdomains", "stable": "1" } \ No newline at end of file diff --git a/update.py b/update.py index be25a3c9ebc..961b535c609 100755 --- a/update.py +++ b/update.py @@ -61,10 +61,23 @@ class Settings: # pylint: disable=too-few-public-methods # - websiteDomainName.com@listName.list # # Note: The variable name should not be changed. - # Note: This variable is auto updated by Initiate() # # Example: "adaway.github.io@AdAway.list" - list_name = "" + list_name = "domains.list" + + # This variable is used to set the location of the file for the list without + # dead/inactive domains. + # + # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS! + # Note: This variable is auto updated by Initiate() + clean_list_file = "clean.list" + + # This variable is used to set the location of the file for the list without + # dead/inactive domains and whitelisted domains. + # + # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS! + # Note: This variable is auto updated by Initiate() + whitelisted_list_file = "whitelisted.list" # This variable will help us know where we are working into the filesystem. # @@ -108,7 +121,7 @@ class Settings: # pylint: disable=too-few-public-methods # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS! PyFunceble = { ".PyFunceble_production.yaml": "https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml", # pylint: disable=line-too-long - ".PyFunceble_LICENSE": "https://raw.githubusercontent.com/funilrys/PyFunceble/dev/LICENSE", + ".PyFunceble_LICENSE": "https://raw.githubusercontent.com/funilrys/PyFunceble/master/LICENSE", # pylint: disable=line-too-long } # This variable is used to match [ci skip] from the git log. @@ -152,13 +165,6 @@ class Settings: # pylint: disable=too-few-public-methods # Note: This variable is auto updated by Initiate() clean_original = False - # This variable is used to set the location of the file for the list without - # dead/inactive domains. - # - # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS! - # Note: This variable is auto updated by Initiate() - clean_list_file = "clean.list" - # This variable is used to know if we need to convert the list into idna # before we write the file PyFunceble is going to test. # @@ -279,7 +285,7 @@ def _fix_cross_repo_config(cls): r"travis:.*": "travis: True", r"travis_autosave_commit:.*": 'travis_autosave_commit: "[Autosave] Testing for Ultimate Hosts Blacklist"', # pylint: disable=line-too-long r"travis_autosave_final_commit:.*": 'travis_autosave_final_commit: "[Results] Testing for Ultimate Hosts Blacklist"', # pylint: disable=line-too-long - r"travis_branch:.*": "travis_branch: master", + r"travis_branch:.*": "travis_branch: %s" % environ['GIT_BRANCH'], r"travis_autosave_minutes:.*": "travis_autosave_minutes: %s" % Settings.autosave_minutes, } @@ -608,6 +614,11 @@ def PyFunceble(self): # pylint: disable=invalid-name Settings.informations.update( {"last_test": strftime("%s"), "currently_under_test": str(int(True))} ) + + for index in ['clean_list_file', 'list_name']: + if index in Settings.informations: + del Settings.informations[index] + Helpers.Dict(Settings.informations).to_json(Settings.repository_info) Helpers.Command(command_to_execute, True).execute()