Switch to pipenv and introduce a Dockerfile (#285)

.dockerignore

@@ -0,0 +1,15 @@
+**/*.pyc
+
+data/
+data_resized/
+inconsistencies.csv
+
+geckodriver.log
+**/.DS_Store
+.coverage
+.pytest_cache
+.idea
+
+.git
+.gitignore
+.travis.yml

.travis.yml

@@ -1,9 +1,14 @@
+dist: xenial
 language: python
 python:
-  - "3.6"
+  - "3.7"
+services:
+  - docker
+
 install:
-  - pip install -r requirements.txt
-  - pip install -r test-requirements.txt
+  - pip install pipenv
+  - pipenv install --dev --ignore-pipfile --deploy --system
+  - docker build . --tag test-autowebcompat
 script:
   - flake8 .
   - pytest ./tests/test_*.py --cov=./

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.7-buster
+
+RUN mkdir /autowebcompat
+WORKDIR /autowebcompat
+
+ADD pip.conf Pipfile Pipfile.lock ./
+ENV PYTHONUNBUFFERED=yes PIP_CONFIG_FILE=/autowebcompat/pip.conf
+RUN pip install pipenv \
+ && pipenv install --system --deploy --ignore-pipfile \
+ && pip uninstall --yes pipenv
+
+ADD . ./
+
+ENV PYTHONUNBUFFERED=yes
+ENV PYTHONPATH "${PYTHONPATH}:/autowebcompat"

Pipfile

@@ -0,0 +1,27 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+flake8 = "*"
+flake8-isort = "*"
+flake8-quotes = "*"
+pytest = "*"
+pytest-cov = "*"
+
+[packages]
+# ml, imgproc
+Keras = "*"
+Pillow = "*"
+opencv-python = "*"
+scikit-learn = "*"
+tensorflow = "*"
+
+# scraping
+h5py = "*"
+selenium = "*"
+lxml = "*"
+
+[requires]
+python_version = "3.7"

README.md

@@ -46,8 +46,7 @@ For the unsupervised training, we are using a related problem for which we alrea
 
 - Install [Git Large File Storage](https://git-lfs.github.com/), either manually or through a package like `git-lfs` if available on your system (in case of using [PackageCloud](https://github.com/git-lfs/git-lfs/blob/master/INSTALLING.md)).
 - Clone the repository with submodules: `git lfs clone --recurse-submodules REPO_URL`
-- Install the dependencies in requirements.txt: `pip install -r requirements.txt`.
-- Install the dependencies in test-requirements.txt: `pip install -r test-requirements.txt`.
+- Install all dependencies: `pip install pipenv && pipenv install --dev && pipenv shell`.
 
 ## Training the network
 - The **pretrain.py** or **train.py** script can be run to train the neural network, with the following options:

autowebcompat/network.py

@@ -227,8 +227,7 @@ def contrastive_loss(y_true, y_pred):
     http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
     """
     margin = 1
-    return K.mean(y_true * K.square(y_pred) +
-                  (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
+    return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
 
 
 def accuracy(y_true, y_pred):

autowebcompat/utils.py

@@ -241,7 +241,7 @@ def get_machine_info():
         parameter_value_map['GPU_{}_name'.format(i + 1)] = device.name
         parameter_value_map['GPU_{}_memory_limit'.format(i + 1)] = device.memory_limit
         parameter_value_map['GPU_{}_description'.format(i + 1)] = device.physical_device_desc
-    lscpu = subprocess.check_output('lscpu | grep \'^CPU(s):\|Core\|Thread\'', shell=True).strip().decode()
+    lscpu = subprocess.check_output("lscpu | grep '^CPU(s):\\|Core\\|Thread'", shell=True).strip().decode()
     lscpu = lscpu.split('\n')
     for row in lscpu:
         row = row.split(':')

collect.py

@@ -321,7 +321,7 @@ def run_test_both(bug, firefox_driver, chrome_driver):
 
     try:
         firefox_driver.get(bug['url'])
-    except TimeoutException as e:
+    except TimeoutException:
         # Ignore timeouts, as they are too frequent.
         traceback.print_exc()
         print('Continuing...')
@@ -331,7 +331,7 @@ def run_test_both(bug, firefox_driver, chrome_driver):
 
     try:
         chrome_driver.get(bug['url'])
-    except TimeoutException as e:
+    except TimeoutException:
         # Ignore timeouts, as they are too frequent.
         traceback.print_exc()
         print('Continuing...')

pip.conf

@@ -0,0 +1,3 @@
+[global]
+no-cache-dir = True
+disable-pip-version-check = True