Side effect

src/pynorare/api.py

@@ -148,11 +148,7 @@ def download_file(self, url, target=None, overwrite=False):
         if not target:
             target = urllib.parse.urlparse(url).path.split('/')[-1]
         if (not self.raw_dir.joinpath(target).exists()) or overwrite:
-            try:
-                urllib.request.urlretrieve(url, str(self.raw_dir / target))
-            except urllib.error.HTTPError:  # pragma: no cover
-                # Try with requests:
-                download_file(url, self.raw_dir / target)
+            download_file(url, self.raw_dir / target)
             self.log.info('Downloaded {0} successfully.'.format(url))
         return self.raw_dir / target
 

src/pynorare/files.py

@@ -39,7 +39,8 @@ def get_excel(path, sheet_index, dicts=False):
 
 
 def download_file(url, path):  # pragma: no cover
-    with requests.get(url, stream=True) as r:
+    headers = {'User-Agent': 'norare/1.1.0'}
+    with requests.get(url, headers=headers, stream=True) as r:
         r.raise_for_status()
         with path.open('wb') as f:
             for chunk in r.iter_content(chunk_size=8192):

tests/test_api.py

@@ -16,8 +16,7 @@ def test_NoRaRe(api):
 
 def test_Dataset_download_zip(api, mocker):
     mocker.patch(
-        'pynorare.api.urllib.request',
-        mocker.Mock(urlretrieve=lambda u, t: 1))
+        'pynorare.api.download_file', lambda _url, path: path)
     ds = api.datasets['ds2']
     ds.download_zip('x', 'f.zip', 'norare.xlsx')
     assert len(ds.get_excel('norare.xlsx')) == 2

tests/test_cli.py

@@ -1,3 +1,4 @@
+import types
 import pathlib
 
 import pytest
@@ -27,18 +28,22 @@ def test_stats(_main, capsys):
     assert out.strip().startswith('No.')
 
 
+def make_pretend_data(_url, path):
+    data = (
+        'gloss,float,int,POS\n'
+        'the gloss,1.2,3,noun\n'
+        'other gloss,1.2,3')
+    path.write_text(data, encoding='utf-8')
+    return path
+
+
 def test_workflow(_main, mocker):
-    mocker.patch(
-        'pynorare.api.urllib.request',
-        mocker.Mock(urlretrieve=lambda u, f: pathlib.Path(f).write_text(
-            'gloss,float,int,POS\nthe gloss,1.2,3,noun\nother gloss,1.2,3', encoding='utf8')))
+    mocker.patch('pynorare.api.download_file', make_pretend_data)
     _main('download', 'dsid')
     _main('map', 'dsid')
     _main('validate', 'dsid')
 
-    mocker.patch(
-        'pynorare.api.urllib.request',
-        mocker.Mock(urlretrieve=lambda u, t: 1))
+    mocker.patch('pynorare.api.download_file', lambda _url, path: path)
     _main('download', 'ds2')
     _main('map', 'ds2')