diff --git a/src/pynorare/api.py b/src/pynorare/api.py index d6b10cd..b5f271d 100644 --- a/src/pynorare/api.py +++ b/src/pynorare/api.py @@ -148,11 +148,7 @@ def download_file(self, url, target=None, overwrite=False): if not target: target = urllib.parse.urlparse(url).path.split('/')[-1] if (not self.raw_dir.joinpath(target).exists()) or overwrite: - try: - urllib.request.urlretrieve(url, str(self.raw_dir / target)) - except urllib.error.HTTPError: # pragma: no cover - # Try with requests: - download_file(url, self.raw_dir / target) + download_file(url, self.raw_dir / target) self.log.info('Downloaded {0} successfully.'.format(url)) return self.raw_dir / target diff --git a/src/pynorare/files.py b/src/pynorare/files.py index e450d89..806929e 100644 --- a/src/pynorare/files.py +++ b/src/pynorare/files.py @@ -39,7 +39,8 @@ def get_excel(path, sheet_index, dicts=False): def download_file(url, path): # pragma: no cover - with requests.get(url, stream=True) as r: + headers = {'User-Agent': 'norare/1.1.0'} + with requests.get(url, headers=headers, stream=True) as r: r.raise_for_status() with path.open('wb') as f: for chunk in r.iter_content(chunk_size=8192): diff --git a/tests/test_api.py b/tests/test_api.py index cc25f31..25e2e14 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -16,8 +16,7 @@ def test_NoRaRe(api): def test_Dataset_download_zip(api, mocker): mocker.patch( - 'pynorare.api.urllib.request', - mocker.Mock(urlretrieve=lambda u, t: 1)) + 'pynorare.api.download_file', lambda _url, path: path) ds = api.datasets['ds2'] ds.download_zip('x', 'f.zip', 'norare.xlsx') assert len(ds.get_excel('norare.xlsx')) == 2 diff --git a/tests/test_cli.py b/tests/test_cli.py index 23e5108..eb202ee 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,4 @@ +import types import pathlib import pytest @@ -27,18 +28,22 @@ def test_stats(_main, capsys): assert out.strip().startswith('No.') +def make_pretend_data(_url, path): + data = ( + 'gloss,float,int,POS\n' + 'the gloss,1.2,3,noun\n' + 'other gloss,1.2,3') + path.write_text(data, encoding='utf-8') + return path + + def test_workflow(_main, mocker): - mocker.patch( - 'pynorare.api.urllib.request', - mocker.Mock(urlretrieve=lambda u, f: pathlib.Path(f).write_text( - 'gloss,float,int,POS\nthe gloss,1.2,3,noun\nother gloss,1.2,3', encoding='utf8'))) + mocker.patch('pynorare.api.download_file', make_pretend_data) _main('download', 'dsid') _main('map', 'dsid') _main('validate', 'dsid') - mocker.patch( - 'pynorare.api.urllib.request', - mocker.Mock(urlretrieve=lambda u, t: 1)) + mocker.patch('pynorare.api.download_file', lambda _url, path: path) _main('download', 'ds2') _main('map', 'ds2')