From ed37f6ec395c16ed19ab2663d5181d42dc82d30e Mon Sep 17 00:00:00 2001 From: funilrys Date: Fri, 11 Jan 2019 22:28:28 +0100 Subject: [PATCH] [Launch test] [Maintenance] Update of the repository structure --- .gitignore | 4 ++++ .travis.yml | 12 +++++++++--- .trigger | 2 +- administration.py | 16 +++++++++++----- info.json | 2 +- update.py | 33 ++++++++++++++++++++++----------- 6 files changed, 48 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index b989be6ca15..0b85796f600 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,7 @@ venv.bak/ # mypy .mypy_cache/ + + +# Container +whitelisting.py \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 4dce4c3d184..d5b886bff63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,13 @@ env: global: - - secure: StNY8sT9988Ew92wW3QAUnQcSA7+SkkWdPcKDi8v3DtRI/adVC4oVS7DzFYxPN5tpQfZljJ23WN2oWg9GNcFfkZ9hQkceHLMnl7Y4NPbgp169AmGTNqOfbsprHRbU+ps5kI7kr+fRhZJJyq66ivdpiLwlO+T6+KxWg6uf+1RgdmcIfEd+clS3iW5wKC/1b+e784bxyx9rj+SlSvzTjxy2Y5HmTolBciv8foKzO6WnNLZy3pJWA70u22NucUZJJRPZ2+Xte9/H24ksNOP/IPKlvGrX3CTQBVWaC5tFmBVUuWOTfwBiXgmWVSD3eEMtBmohlXTUvMb3aw8mHf+OGk07ODguF2ZSMRr68a6FVd4iCkVQQ1vODHoNseCJrwhZD2C7qeeLRK1D7DE5oVPpsnl5VyfLgfisbJEmyI0u/wWdmSwvW8hIkrmmq/mHDc4eLRoAiALfz2lgzJdKJGf8C3L/x2QQMhgzAwJUAp5QPzua8DS2ml298gJGePT3gQ5KhjWgiGljY5Q6W3LfU56pic+VrPclWHn/uuZa+Z8MX4BjyNCTQ9Kb3pAueKJpTQKK83jYEmfC5yIcN3vHajjfxq8GI+oB32eyuDeHDZKjhb+YLIT5d2R4+BecOIPb6/+a8B9Sdj3NS/J4tgfwk2s7YzoRcExClg6Hgz+pfKNRiSanUQ= + - secure: Fxek3lFNPq4gXEl0jdah5e/Kbgkt+sivwSHS6/YyjGgxapBtAAOHI7SO69EWr1KZLgH8P/MyBOXrH+sZel6o8lA1OIFwt9QL8zY5EnMCV5aG4OBP+naTHiOBnE0AzRKsCmtds0CRIy1oQ00WecmrLK3ZfMFDM6nrxpjxO8nbpndMRVVD3Jcpw3Dzac99OIot9qPiEBUq8oczlBE3+jz867BWPxM0wRcVeq39mV/Ve3lgXiZwg/AmVFkA+dJxoKWDEQpcc7NnmdEnlYlr/M+ys1GMJsx9QDQegYocCTiVF8mamrnHWU4TqfHWI8ViABkbbDsKobK4fUDRURqNOvAFBAnK4ibwDwnn3b8sU6VmWZZHxgMpZslDD9v19rB2Se6tWqNc0uwq2Sg6x9i2qz7GQh6xO8+DDCwtBaoRyco5OcGIrQECtnma5ayjjD8PX6gxhaH3HQiayHFQzOwW2ZEnmvu7Gxb2nJg3VlOXchh5QDFpoTC7y9C2zeaSWZHGom9HbGZm9x5jJ/iTeZdHTIsMPy/ojs5NJuZNLUnL1E5dCAUzzNhGIcJWMa4hz5EvI+/qrqIfcVP/6SbKFm10hlajkxITJTL1vxHvOHhlXPlzhNsJd0fDiWUApSpq+uMbQAuM12wnWtg+Rn+HT3JmlIbCkvdX2wCWdmuLpyfwZwab6Rw= - GIT_NAME: Travis CI - GIT_EMAIL: dead-hosts@funilrys.com - TRAVIS_REPO_SLUG: Ultimate-Hosts-Blacklist/Phishing.Database - GIT_BRANCH: master + - WHITELISTING_LOCATION: https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/dev-center/whitelisting/whitelisting.py + - UPDATE_LOCATION: https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/repository-structure/master/update.py + - ADMINISTATION_LOCATION: https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/repository-structure/master/administration.py dist: xenial sudo: required @@ -27,10 +30,13 @@ addons: install: - pip3 install --upgrade pip - pip3 install requests + - wget "${WHITELISTING_LOCATION}" -o /dev/null -O whitelisting.py && chmod +x whitelisting.py + - if [[ ! -f .admin ]]; then wget "${UPDATE_LOCATION}" -o /dev/null -O update.py && chmod +x update.py; else echo "No need to download/update update.py."; fi + - if [[ ! -f .admin ]]; then wget "${ADMINISTATION_LOCATION}" -o /dev/null -O administration.py && chmod +x administration.py; else echo "No need to download/update administration.py."; fi script: - python3 ./update.py notifications: - on_success: change - on_failure: always + on_success: change + on_failure: always diff --git a/.trigger b/.trigger index ed7b1c11105..6544ed7cb4b 100644 --- a/.trigger +++ b/.trigger @@ -1 +1 @@ -1543607196 \ No newline at end of file +1547242105 \ No newline at end of file diff --git a/administration.py b/administration.py index 7433c79c61d..10bbb3c2b4d 100755 --- a/administration.py +++ b/administration.py @@ -15,6 +15,7 @@ @GitHubUsername, Name, Email (optional) """ from update import Helpers, Settings, path, strftime +from whitelisting import Whitelist INFO = {} @@ -28,7 +29,7 @@ def get_administration_file(): content = Helpers.File(Settings.repository_info).read() INFO.update(Helpers.Dict().from_json(content)) else: - raise Exception("Unabel to find the administration file.") + raise Exception("Unable to find the administration file.") def update_adminisation_file(): @@ -47,16 +48,16 @@ def save_administration_file(): Helpers.Dict(INFO).to_json(Settings.repository_info) -def generate_clean_list(): +def generate_clean_and_whitelisted_list(): """ - Update `clean.list`. + Update/Create `clean.list` and `whitelisted.list`. """ if bool(int(INFO["clean_original"])): clean_list = [] list_special_content = Helpers.Regex( - Helpers.File(Settings.file_to_test + INFO["list_name"]).to_list(), r"ALL\s" + Helpers.File(Settings.file_to_test + Settings.list_name).to_list(), r"ALL\s" ).matching_list() active = Settings.current_directory + "output/domains/ACTIVE/list" @@ -68,15 +69,20 @@ def generate_clean_list(): ) clean_list = Helpers.List(clean_list).format() + whitelisted = Whitelist(string="\n".join(clean_list)).get() Helpers.File(Settings.clean_list_file).write( "\n".join(clean_list), overwrite=True ) + Helpers.File(Settings.whitelisted_list_file).write(whitelisted, overwrite=True) + + Helpers.File("whitelisting.py").delete() + if __name__ == "__main__": get_administration_file() update_adminisation_file() save_administration_file() - generate_clean_list() + generate_clean_and_whitelisted_list() diff --git a/info.json b/info.json index 57af569058f..040271af253 100644 --- a/info.json +++ b/info.json @@ -6,7 +6,7 @@ "days_until_next_test": "2", "last_test": "1547235378", "list_name": "domains.list", - "name": "Domain Status Testing for Ultimate Hosts Blacklist", + "name": "Phishing.Database", "raw_link": "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-ACTIVE.txt", "stable": "1" } \ No newline at end of file diff --git a/update.py b/update.py index be25a3c9ebc..961b535c609 100755 --- a/update.py +++ b/update.py @@ -61,10 +61,23 @@ class Settings: # pylint: disable=too-few-public-methods # - websiteDomainName.com@listName.list # # Note: The variable name should not be changed. - # Note: This variable is auto updated by Initiate() # # Example: "adaway.github.io@AdAway.list" - list_name = "" + list_name = "domains.list" + + # This variable is used to set the location of the file for the list without + # dead/inactive domains. + # + # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS! + # Note: This variable is auto updated by Initiate() + clean_list_file = "clean.list" + + # This variable is used to set the location of the file for the list without + # dead/inactive domains and whitelisted domains. + # + # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS! + # Note: This variable is auto updated by Initiate() + whitelisted_list_file = "whitelisted.list" # This variable will help us know where we are working into the filesystem. # @@ -108,7 +121,7 @@ class Settings: # pylint: disable=too-few-public-methods # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS! PyFunceble = { ".PyFunceble_production.yaml": "https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml", # pylint: disable=line-too-long - ".PyFunceble_LICENSE": "https://raw.githubusercontent.com/funilrys/PyFunceble/dev/LICENSE", + ".PyFunceble_LICENSE": "https://raw.githubusercontent.com/funilrys/PyFunceble/master/LICENSE", # pylint: disable=line-too-long } # This variable is used to match [ci skip] from the git log. @@ -152,13 +165,6 @@ class Settings: # pylint: disable=too-few-public-methods # Note: This variable is auto updated by Initiate() clean_original = False - # This variable is used to set the location of the file for the list without - # dead/inactive domains. - # - # Note: DO NOT TOUCH UNLESS YOU KNOW WHAT IT MEANS! - # Note: This variable is auto updated by Initiate() - clean_list_file = "clean.list" - # This variable is used to know if we need to convert the list into idna # before we write the file PyFunceble is going to test. # @@ -279,7 +285,7 @@ def _fix_cross_repo_config(cls): r"travis:.*": "travis: True", r"travis_autosave_commit:.*": 'travis_autosave_commit: "[Autosave] Testing for Ultimate Hosts Blacklist"', # pylint: disable=line-too-long r"travis_autosave_final_commit:.*": 'travis_autosave_final_commit: "[Results] Testing for Ultimate Hosts Blacklist"', # pylint: disable=line-too-long - r"travis_branch:.*": "travis_branch: master", + r"travis_branch:.*": "travis_branch: %s" % environ['GIT_BRANCH'], r"travis_autosave_minutes:.*": "travis_autosave_minutes: %s" % Settings.autosave_minutes, } @@ -608,6 +614,11 @@ def PyFunceble(self): # pylint: disable=invalid-name Settings.informations.update( {"last_test": strftime("%s"), "currently_under_test": str(int(True))} ) + + for index in ['clean_list_file', 'list_name']: + if index in Settings.informations: + del Settings.informations[index] + Helpers.Dict(Settings.informations).to_json(Settings.repository_info) Helpers.Command(command_to_execute, True).execute()