diff --git a/api/odl.py b/api/odl.py index 67c5e014b7..7ab1c2ee5c 100644 --- a/api/odl.py +++ b/api/odl.py @@ -1,7 +1,9 @@ import datetime import json +import logging import uuid from io import StringIO +from typing import Callable, Optional import dateutil import feedparser @@ -14,12 +16,7 @@ from core import util from core.analytics import Analytics -from core.metadata_layer import ( - CirculationData, - FormatData, - LicenseData, - TimestampData, -) +from core.metadata_layer import CirculationData, FormatData, LicenseData, TimestampData from core.model import ( Collection, ConfigurationSetting, @@ -32,38 +29,20 @@ LicensePool, Loan, MediaTypes, + Representation, RightsStatus, Session, get_one, get_one_or_create, - Representation) -from core.monitor import ( - CollectionMonitor, - IdentifierSweepMonitor) -from core.opds_import import ( - OPDSXMLParser, - OPDSImporter, - OPDSImportMonitor, -) -from core.testing import ( - DatabaseTest, - MockRequestsResponse, -) -from core.util.datetime_helpers import ( - utc_now, -) -from core.util.http import ( - HTTP, - BadResponseException, - RemoteIntegrationException, ) +from core.monitor import CollectionMonitor, IdentifierSweepMonitor +from core.opds_import import OPDSImporter, OPDSImportMonitor, OPDSXMLParser +from core.testing import DatabaseTest, MockRequestsResponse +from core.util.datetime_helpers import utc_now +from core.util.http import HTTP, BadResponseException, RemoteIntegrationException from core.util.string_helpers import base64 -from .circulation import ( - BaseCirculationAPI, - LoanInfo, - FulfillmentInfo, - HoldInfo, -) + +from .circulation import BaseCirculationAPI, FulfillmentInfo, HoldInfo, LoanInfo from .circulation_exceptions import * from .shared_collection import BaseSharedCollectionAPI @@ -786,6 +765,105 @@ class ODLImporter(OPDSImporter): # about the license. LICENSE_INFO_DOCUMENT_MEDIA_TYPE = 'application/vnd.odl.info+json' + @classmethod + def parse_license( + cls, + identifier: str, + total_checkouts: Optional[int], + concurrent_checkouts: Optional[int], + expires: Optional[datetime.datetime], + checkout_link: Optional[str], + odl_status_link: Optional[str], + do_get: Callable + ) -> Optional[LicenseData]: + """Check the license's attributes passed as parameters: + - if they're correct, turn them into a LicenseData object + - otherwise, return a None + + :param identifier: License's identifier + :param total_checkouts: Total number of checkouts before the license expires + :param concurrent_checkouts: Number of concurrent checkouts allowed for this license + :param expires: Date & time until the license is valid + :param checkout_link: License's checkout link + :param odl_status_link: License Info Document's link + :param do_get: Callback performing HTTP GET method + + :return: LicenseData if all the license's attributes are correct, None, otherwise + """ + remaining_checkouts = None + available_concurrent_checkouts = None + + # This cycle ends in two different cases: + # - when at least one of the parameters is invalid; in this case, the method returns None. + # - when all the parameters are valid; in this case, the method returns a LicenseData object. + while True: + if total_checkouts is not None: + total_checkouts = int(total_checkouts) + + if total_checkouts <= 0: + logging.info( + f"License # {identifier} expired since " + f"the total number of checkouts is {total_checkouts}" + ) + break + + if expires: + if not isinstance(expires, datetime.datetime): + expires = dateutil.parser.parse(expires) + + expires = util.datetime_helpers.to_utc(expires) + now = util.datetime_helpers.utc_now() + + if expires <= now: + logging.info( + f"License # {identifier} expired at {expires} (now is {now})" + ) + break + + if odl_status_link: + status_code, _, response = do_get( + odl_status_link, headers={} + ) + + if status_code in (200, 201): + status = json.loads(response) + checkouts = status.get("checkouts", {}) + remaining_checkouts = checkouts.get("left") + available_concurrent_checkouts = checkouts.get("available") + else: + logging.warning( + f"License # {identifier}'s Info Document is not available. " + f"Status link failed with {status_code} code" + ) + break + + if remaining_checkouts is None: + remaining_checkouts = total_checkouts + + if remaining_checkouts is not None: + remaining_checkouts = int(remaining_checkouts) + + if remaining_checkouts <= 0: + logging.info( + f"License # {identifier} expired since " + f"the remaining number of checkouts is {remaining_checkouts}" + ) + break + + if available_concurrent_checkouts is None: + available_concurrent_checkouts = concurrent_checkouts + + return LicenseData( + identifier=identifier, + checkout_url=checkout_link, + status_url=odl_status_link, + expires=expires, + remaining_checkouts=remaining_checkouts, + concurrent_checkouts=available_concurrent_checkouts, + ) + + return None + @classmethod def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get=None): do_get = do_get or Representation.cautious_http_get @@ -843,11 +921,6 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get= data['medium'] = medium - expires = None - remaining_checkouts = None - available_checkouts = None - concurrent_checkouts = None - checkout_link = None for link_tag in parser._xpath(odl_license_tag, 'odl:tlink') or []: rel = link_tag.attrib.get("rel") @@ -866,37 +939,33 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get= odl_status_link = attrib.get("href") break - # If we found one, retrieve it and get licensing information about this book. - if odl_status_link: - ignore, ignore, response = do_get(odl_status_link, headers={}) - status = json.loads(response) - checkouts = status.get("checkouts", {}) - remaining_checkouts = checkouts.get("left") - available_checkouts = checkouts.get("available") + expires = None + total_checkouts = None + concurrent_checkouts = None terms = parser._xpath(odl_license_tag, "odl:terms") if terms: + total_checkouts = subtag(terms[0], "odl:total_checkouts") concurrent_checkouts = subtag(terms[0], "odl:concurrent_checkouts") expires = subtag(terms[0], "odl:expires") - if expires: - expires = util.datetime_helpers.to_utc(dateutil.parser.parse(expires)) - now = util.datetime_helpers.utc_now() + license = cls.parse_license( + identifier, + total_checkouts, + concurrent_checkouts, + expires, + checkout_link, + odl_status_link, + do_get + ) - if expires <= now: - continue + if not license: + continue - licenses_owned += int(concurrent_checkouts or 0) - licenses_available += int(available_checkouts or 0) + licenses_owned += int(license.remaining_checkouts or 0) + licenses_available += int(license.concurrent_checkouts or 0) - licenses.append(LicenseData( - identifier=identifier, - checkout_url=checkout_link, - status_url=odl_status_link, - expires=expires, - remaining_checkouts=remaining_checkouts, - concurrent_checkouts=concurrent_checkouts, - )) + licenses.append(license) if not data.get('circulation'): data['circulation'] = dict() @@ -1552,17 +1621,20 @@ def __init__(self, _db, collection): def process_item(self, identifier): for licensepool in identifier.licensed_through: - licenses_owned = licensepool.licenses_owned - licenses_available = licensepool.licenses_available - - for license in licensepool.licenses: - if license.is_expired: - licenses_owned -= 1 - licenses_available -= 1 - - if licenses_owned != licensepool.licenses_owned or licenses_available != licensepool.licenses_available: - licenses_owned = max(licenses_owned, 0) - licenses_available = max(licenses_available, 0) + remaining_checkouts = 0 # total number of checkouts across all the licenses in the pool + concurrent_checkouts = 0 # number of concurrent checkouts allowed across all the licenses in the pool + + # 0 is a starting point, + # we're going through all the valid licenses in the pool and count up available checkouts. + for license_pool_license in licensepool.licenses: + if not license_pool_license.is_expired: + remaining_checkouts += license_pool_license.remaining_checkouts + concurrent_checkouts += license_pool_license.concurrent_checkouts + + if remaining_checkouts != licensepool.licenses_owned or \ + concurrent_checkouts != licensepool.licenses_available: + licenses_owned = max(remaining_checkouts, 0) + licenses_available = max(concurrent_checkouts, 0) circulation_data = CirculationData( data_source=licensepool.data_source, diff --git a/api/odl2.py b/api/odl2.py index f22e5e3abf..dfbb279c47 100644 --- a/api/odl2.py +++ b/api/odl2.py @@ -1,14 +1,13 @@ import json import logging +from api.odl import ODLAPI, ODLExpiredItemsReaper, ODLImporter from contextlib2 import contextmanager from flask_babel import lazy_gettext as _ from webpub_manifest_parser.odl import ODLFeedParserFactory from webpub_manifest_parser.opds2.registry import OPDS2LinkRelationsRegistry -from api.odl import ODLAPI, ODLExpiredItemsReaper -from core import util -from core.metadata_layer import FormatData, LicenseData +from core.metadata_layer import FormatData from core.model import DeliveryMechanism, Edition, MediaTypes, RightsStatus from core.model.configuration import ( ConfigurationAttributeType, @@ -209,11 +208,6 @@ def _extract_publication_metadata(self, feed, publication, data_source_name): ) ) - expires = None - remaining_checkouts = None - available_checkouts = None - concurrent_checkouts = None - checkout_link = first_or_default( license.links.get_by_rel(OPDS2LinkRelationsRegistry.BORROW.key) ) @@ -226,41 +220,32 @@ def _extract_publication_metadata(self, feed, publication, data_source_name): if odl_status_link: odl_status_link = odl_status_link.href - if odl_status_link: - status_code, _, response = self.http_get( - odl_status_link, headers={} - ) - - if status_code < 400: - status = json.loads(response) - checkouts = status.get("checkouts", {}) - remaining_checkouts = checkouts.get("left") - available_checkouts = checkouts.get("available") + expires = None + total_checkouts = None + concurrent_checkouts = None if license.metadata.terms: - expires = license.metadata.terms.expires + total_checkouts = license.metadata.terms.checkouts concurrent_checkouts = license.metadata.terms.concurrency + expires = license.metadata.terms.expires - if expires: - expires = util.datetime_helpers.to_utc(expires) - now = util.datetime_helpers.utc_now() + license = ODLImporter.parse_license( + identifier, + total_checkouts, + concurrent_checkouts, + expires, + checkout_link, + odl_status_link, + self.http_get + ) - if expires <= now: - continue + if not license: + continue - licenses_owned += int(concurrent_checkouts or 0) - licenses_available += int(available_checkouts or 0) + licenses_owned += int(license.remaining_checkouts or 0) + licenses_available += int(license.concurrent_checkouts or 0) - licenses.append( - LicenseData( - identifier=identifier, - checkout_url=checkout_link, - status_url=odl_status_link, - expires=expires, - remaining_checkouts=remaining_checkouts, - concurrent_checkouts=concurrent_checkouts, - ) - ) + licenses.append(license) metadata.circulation.licenses_owned = licenses_owned metadata.circulation.licenses_available = licenses_available diff --git a/tests/files/odl/single_license.opds b/tests/files/odl/feed_template.xml.jinja similarity index 69% rename from tests/files/odl/single_license.opds rename to tests/files/odl/feed_template.xml.jinja index e89a17d610..0c1580d7cc 100644 --- a/tests/files/odl/single_license.opds +++ b/tests/files/odl/feed_template.xml.jinja @@ -45,35 +45,48 @@ - - urn:uuid:c981d61e-26f4-4070-aaa8-83df952cf61b - application/epub+zip - text/html - http://www.cantook.net/ - 40.00 - cant-2461538-24501117858552614-libraries - 2020-03-02T20:20:17+01:00 - - {{expires}} - 1 - 5097600 - - - application/vnd.adobe.adept+xml - 6 - true - false - false - - - application/vnd.readium.lcp.license.v1.0+json - 6 - true - false - false - - - - + + {% for license in licenses %} + + urn:uuid:{{ license.identifier }} + application/epub+zip + text/html + http://www.cantook.net/ + 40.00 + cant-2461538-24501117858552614-libraries + 2020-03-02T20:20:17+01:00 + + {% if license.total_checkouts is not none %} + {{ license.total_checkouts }} + {% endif %} + + {% if license.concurrent_checkouts is not none %} + {{ license.concurrent_checkouts }} + {% endif %} + + {% if license.expires is not none %} + {{ license.expires.isoformat() }} + {% endif %} + + 5097600 + + + application/vnd.adobe.adept+xml + 6 + true + false + false + + + application/vnd.readium.lcp.license.v1.0+json + 6 + true + false + false + + + + + {% endfor %} - \ No newline at end of file + diff --git a/tests/files/odl2/feed_template.json.jinja b/tests/files/odl2/feed_template.json.jinja new file mode 100644 index 0000000000..0c44af253e --- /dev/null +++ b/tests/files/odl2/feed_template.json.jinja @@ -0,0 +1,118 @@ +{ + "metadata": { + "title": "Test", + "itemsPerPage": 10, + "currentPage": 1, + "numberOfItems": 100 + }, + "links": [ + { + "type": "application/opds+json", + "rel": "self", + "href": "https://market.feedbooks.com/api/libraries/harvest.json" + } + ], + "publications": [ + { + "metadata": { + "@type": "http://schema.org/Book", + "title": "Moby-Dick", + "author": "Herman Melville", + "identifier": "urn:isbn:978-3-16-148410-0", + "language": "en", + "publisher": { + "name": "Test Publisher" + }, + "published": "2015-09-29T00:00:00Z", + "modified": "2015-09-29T17:00:00Z", + "subject": [ + { + "scheme": "http://schema.org/audience", + "code": "juvenile-fiction", + "name": "Juvenile Fiction", + "links": [] + } + ] + }, + "links": [ + { + "rel": "self", + "href": "http://example.org/publication.json", + "type": "application/opds-publication+json" + } + ], + "images": [ + { + "href": "http://example.org/cover.jpg", + "type": "image/jpeg", + "height": 1400, + "width": 800 + }, + { + "href": "http://example.org/cover-small.jpg", + "type": "image/jpeg", + "height": 700, + "width": 400 + }, + { + "href": "http://example.org/cover.svg", + "type": "image/svg+xml" + } + ], + "licenses": [ + {% for license in licenses %} + { + "metadata": { + "identifier": "urn:uuid:{{ license.identifier }}", + "format": [ + "application/epub+zip", + "text/html", + "application/audiobook+json; protection=http://www.feedbooks.com/audiobooks/access-restriction" + ], + "price": { + "currency": "USD", + "value": 7.99 + }, + "created": "2014-04-25T12:25:21+02:00", + "terms": { + {% if license.total_checkouts is not none %} + "checkouts": {{ license.total_checkouts }}, + {% endif %} + {% if license.concurrent_checkouts is not none %} + "concurrency": {{ license.concurrent_checkouts }}, + {% endif %} + {% if license.expires is not none %} + "expires": "{{ license.expires.isoformat() }}", + {% endif %} + "length": 5097600 + }, + "protection": { + "format": [ + "application/vnd.adobe.adept+xml", + "application/vnd.readium.lcp.license.v1.0+json" + ], + "devices": 6, + "copy": false, + "print": false, + "tts": false + } + }, + "links": [ + { + "rel": "http://opds-spec.org/acquisition/borrow", + "href": "http://www.example.com/get{?id,checkout_id,expires,patron_id,passphrase,hint,hint_url,notification_url}", + "type": "application/vnd.readium.license.status.v1.0+json", + "templated": true + }, + { + "rel": "self", + "href": "http://www.example.com/status/294024", + "type": "application/vnd.odl.info+json" + } + ] + }{{ ", " if not loop.last else "" }} + {% endfor %} + ] + } + ] +} diff --git a/tests/files/odl2/single_license.json b/tests/files/odl2/single_license.json deleted file mode 100644 index 695f12f51e..0000000000 --- a/tests/files/odl2/single_license.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "metadata": { - "title": "Test", - "itemsPerPage": 10, - "currentPage": 1, - "numberOfItems": 100 - }, - "links": [ - { - "type": "application/opds+json", - "rel": "self", - "href": "https://market.feedbooks.com/api/libraries/harvest.json" - } - ], - "publications": [ - { - "metadata": { - "@type": "http://schema.org/Book", - "title": "Moby-Dick", - "author": "Herman Melville", - "identifier": "urn:isbn:978-3-16-148410-0", - "language": "en", - "publisher": { - "name": "Test Publisher" - }, - "published": "2015-09-29T00:00:00Z", - "modified": "2015-09-29T17:00:00Z", - "subject": [ - { - "scheme": "http://schema.org/audience", - "code": "juvenile-fiction", - "name": "Juvenile Fiction", - "links": [] - } - ] - }, - "links": [ - { - "rel": "self", - "href": "http://example.org/publication.json", - "type": "application/opds-publication+json" - } - ], - "images": [ - { - "href": "http://example.org/cover.jpg", - "type": "image/jpeg", - "height": 1400, - "width": 800 - }, - { - "href": "http://example.org/cover-small.jpg", - "type": "image/jpeg", - "height": 700, - "width": 400 - }, - { - "href": "http://example.org/cover.svg", - "type": "image/svg+xml" - } - ], - "licenses": [ - { - "metadata": { - "identifier": "urn:uuid:f7847120-fc6f-11e3-8158-56847afe9799", - "format": [ - "application/epub+zip", - "text/html", - "application/audiobook+json; protection=http://www.feedbooks.com/audiobooks/access-restriction" - ], - "price": { - "currency": "USD", - "value": 7.99 - }, - "created": "2014-04-25T12:25:21+02:00", - "terms": { - "checkouts": 1, - "expires": "{{expires}}", - "concurrency": 1, - "length": 5097600 - }, - "protection": { - "format": [ - "application/vnd.adobe.adept+xml", - "application/vnd.readium.lcp.license.v1.0+json" - ], - "devices": 6, - "copy": false, - "print": false, - "tts": false - } - }, - "links": [ - { - "rel": "http://opds-spec.org/acquisition/borrow", - "href": "http://www.example.com/get{?id,checkout_id,expires,patron_id,passphrase,hint,hint_url,notification_url}", - "type": "application/vnd.readium.license.status.v1.0+json", - "templated": true - }, - { - "rel": "self", - "href": "http://www.example.com/status/294024", - "type": "application/vnd.odl.info+json" - } - ] - } - ] - } - ] -} diff --git a/tests/test_odl.py b/tests/test_odl.py index a7028f90d4..3bc8352731 100644 --- a/tests/test_odl.py +++ b/tests/test_odl.py @@ -2,13 +2,11 @@ import json import os import urllib.parse +import uuid +from typing import List, Optional, Tuple import dateutil import pytest -from dateutil.tz import tzoffset -from freezegun import freeze_time -from mock import MagicMock - from api.circulation_exceptions import * from api.odl import ( ODLAPI, @@ -20,6 +18,11 @@ SharedODLAPI, SharedODLImporter, ) +from freezegun import freeze_time +from jinja2 import Environment, FileSystemLoader, select_autoescape +from mock import MagicMock, PropertyMock, patch +from parameterized import parameterized + from core.model import ( Collection, ConfigurationSetting, @@ -29,10 +32,12 @@ ExternalIntegration, Hold, Hyperlink, + LicensePool, Loan, MediaTypes, Representation, RightsStatus, + Work, ) from core.scripts import RunCollectionMonitorScript from core.testing import DatabaseTest @@ -1307,7 +1312,7 @@ def canonicalize_author_name(self, identifier, working_display_name): return working_display_name metadata_client = MockMetadataClient() - warrior_time_limited = dict(checkouts=dict(available=1)) + warrior_time_limited = dict(checkouts=dict(left=52, available=1)) canadianity_loan_limited = dict(checkouts=dict(left=40, available=10)) canadianity_perpetual = dict(checkouts=dict(available=1)) midnight_loan_limited_1 = dict(checkouts=dict(left=20, available=1)) @@ -1367,7 +1372,7 @@ def do_get(url, headers): assert Representation.EPUB_MEDIA_TYPE == lpdm.delivery_mechanism.content_type assert DeliveryMechanism.ADOBE_DRM == lpdm.delivery_mechanism.drm_scheme assert RightsStatus.IN_COPYRIGHT == lpdm.rights_status.uri - assert 1 == warrior_pool.licenses_owned + assert 52 == warrior_pool.licenses_owned # 52 remaining checkouts in the License Info Document assert 1 == warrior_pool.licenses_available [license] = warrior_pool.licenses assert "1" == license.identifier @@ -1384,7 +1389,7 @@ def do_get(url, headers): assert datetime.datetime( 2019, 3, 31, 3, 13, 35, tzinfo=dateutil.tz.tzoffset("", 3600*2) ) == license.expires - assert None == license.remaining_checkouts + assert 52 == license.remaining_checkouts # 52 remaining checkouts in the License Info Document assert 1 == license.concurrent_checkouts # This item is an open access audiobook. @@ -1418,7 +1423,7 @@ def do_get(url, headers): assert Representation.EPUB_MEDIA_TYPE == lpdm.delivery_mechanism.content_type assert DeliveryMechanism.ADOBE_DRM == lpdm.delivery_mechanism.drm_scheme assert RightsStatus.IN_COPYRIGHT == lpdm.rights_status.uri - assert 11 == canadianity_pool.licenses_owned + assert 40 == canadianity_pool.licenses_owned # 40 remaining checkouts in the License Info Document assert 11 == canadianity_pool.licenses_available [license1, license2] = sorted(canadianity_pool.licenses, key=lambda x: x.identifier) assert "2" == license1.identifier @@ -1450,7 +1455,7 @@ def do_get(url, headers): [lpdm.delivery_mechanism.drm_scheme for lpdm in lpdms]) assert ([RightsStatus.IN_COPYRIGHT, RightsStatus.IN_COPYRIGHT] == [lpdm.rights_status.uri for lpdm in lpdms]) - assert 2 == midnight_pool.licenses_owned + assert 72 == midnight_pool.licenses_owned # 20 + 52 remaining checkouts in corresponding License Info Documents assert 2 == midnight_pool.licenses_available [license1, license2] = sorted(midnight_pool.licenses, key=lambda x: x.identifier) assert "4" == license1.identifier @@ -1927,29 +1932,128 @@ def canonicalize_author_name(self, identifier, working_display_name): assert Representation.EPUB_MEDIA_TYPE == lpdm.delivery_mechanism.content_type assert DeliveryMechanism.ADOBE_DRM == lpdm.delivery_mechanism.drm_scheme assert RightsStatus.IN_COPYRIGHT == lpdm.rights_status.uri - [borrow_link] = [l for l in essex_pool.identifier.links if l.rel == Hyperlink.BORROW] - assert 'http://localhost:6500/AL/works/URI/http://www.feedbooks.com/item/1946289/borrow' == borrow_link.resource.url + [borrow_link] = [ + l for l in essex_pool.identifier.links if l.rel == Hyperlink.BORROW + ] + assert ( + "http://localhost:6500/AL/works/URI/http://www.feedbooks.com/item/1946289/borrow" + == borrow_link.resource.url + ) + + +class TestLicense: + """Represents an ODL license.""" + + def __init__( + self, + identifier: Optional[str] = None, + total_checkouts: Optional[int] = None, + concurrent_checkouts: Optional[int] = None, + expires: Optional[datetime.datetime] = None, + ) -> None: + """Initialize a new instance of TestLicense class. + + :param identifier: License's identifier + :param total_checkouts: Total number of checkouts before a license expires + :param concurrent_checkouts: Number of concurrent checkouts allowed + :param expires: Date & time when a license expires + """ + self._identifier: str = identifier if identifier else str(uuid.uuid1()) + self._total_checkouts: Optional[int] = total_checkouts + self._concurrent_checkouts: Optional[int] = concurrent_checkouts + self._expires: Optional[datetime.datetime] = expires + + @property + def identifier(self) -> str: + """Return the license's identifier. + + :return: License's identifier + """ + return self._identifier + + @property + def total_checkouts(self) -> Optional[int]: + """Return the total number of checkouts before a license expires. + + :return: Total number of checkouts before a license expires + """ + return self._total_checkouts + + @property + def concurrent_checkouts(self) -> Optional[int]: + """Return the number of concurrent checkouts allowed. + + :return: Number of concurrent checkouts allowed + """ + return self._concurrent_checkouts + + @property + def expires(self) -> Optional[datetime.datetime]: + """Return the date & time when a license expires. + + :return: Date & time when a license expires + """ + return self._expires + + +class TestLicenseInfo: + """Represents information about the current state of a license stored in the License Info Document.""" + + def __init__( + self, remaining_checkouts: int, available_concurrent_checkouts: int + ) -> None: + """Initialize a new instance of TestLicenseInfo class. + + :param remaining_checkouts: Total number of checkouts left for a License + :param available_concurrent_checkouts: Number of concurrent checkouts currently available + """ + self._remaining_checkouts: int = remaining_checkouts + self._available_concurrent_checkouts: int = available_concurrent_checkouts + + @property + def remaining_checkouts(self) -> int: + """Return the total number of checkouts left for a License. + + :return: Total number of checkouts left for a License + """ + return self._remaining_checkouts + + @property + def available_concurrent_checkouts(self) -> int: + """Return the number of concurrent checkouts currently available. + + :return: Number of concurrent checkouts currently available + """ + return self._available_concurrent_checkouts + + def __str__(self) -> str: + """Return a JSON representation of a part of the License Info Document.""" + return json.dumps( + { + "checkouts": { + "left": self.remaining_checkouts, + "available": self.available_concurrent_checkouts, + } + } + ) class TestODLExpiredItemsReaper(DatabaseTest, BaseODLTest): + """Base class for all ODL reaper tests.""" + ODL_PROTOCOL = ODLAPI.NAME - ODL_FEED_FILENAME_WITH_SINGLE_ODL_LICENSE = "single_license.opds" - ODL_LICENSE_EXPIRATION_TIME_PLACEHOLDER = "{{expires}}" + ODL_TEMPLATE_DIR = os.path.join(BaseODLTest.base_path, "files", "odl") + ODL_TEMPLATE_FILENAME = "feed_template.xml.jinja" ODL_REAPER_CLASS = ODLExpiredItemsReaper - SECONDS_PER_HOUR = 3600 def _create_importer(self, collection, http_get): """Create a new ODL importer with the specified parameters. :param collection: Collection object - :type collection: core.model.collection.Collection - :param http_get: Use this method to make an HTTP GET request. This can be replaced with a stub method for testing purposes. - :type http_get: Callable :return: ODLImporter object - :rtype: ODLImporter """ importer = ODLImporter( self._db, @@ -1959,60 +2063,96 @@ def _create_importer(self, collection, http_get): return importer - def _get_test_feed_with_single_odl_license(self, expires): - """Get the feed with a single ODL license with the specific expiration date. + def _get_test_feed(self, licenses: List[TestLicense]) -> str: + """Get the test ODL feed with specific licensing information. - :param expires: Expiration date of the ODL license - :type expires: datetime.datetime + :param licenses: List of ODL licenses - :return: Test ODL feed with a single ODL license with the specific expiration date - :rtype: str + :return: Test ODL feed """ - feed = self.get_data(self.ODL_FEED_FILENAME_WITH_SINGLE_ODL_LICENSE) - feed = feed.replace(self.ODL_LICENSE_EXPIRATION_TIME_PLACEHOLDER, expires.isoformat()) + env = Environment( + loader=FileSystemLoader(self.ODL_TEMPLATE_DIR), autoescape=select_autoescape() + ) + template = env.get_template(self.ODL_TEMPLATE_FILENAME) + feed = template.render(licenses=licenses) return feed - def _import_test_feed_with_single_odl_license(self, expires): - """Import the test ODL feed with a single ODL license with the specific expiration date. + def _import_test_feed( + self, + licenses: List[TestLicense], + license_infos: Optional[List[Optional[TestLicenseInfo]]] = None, + ) -> Tuple[List[Edition], List[LicensePool], List[Work]]: + """Import the test ODL feed with specific licensing information. - :param expires: Expiration date of the ODL license - :type expires: datetime.datetime + :param licenses: List of ODL licenses + :param license_infos: List of License Info Documents :return: 3-tuple containing imported editions, license pools and works - :rtype: Tuple[List[Edition], List[LicensePool], List[Work]] """ - feed = self._get_test_feed_with_single_odl_license(expires) + feed = self._get_test_feed(licenses) data_source = DataSource.lookup(self._db, "Feedbooks", autocreate=True) collection = MockODLAPI.mock_collection(self._db, protocol=self.ODL_PROTOCOL) collection.external_integration.set_setting( - Collection.DATA_SOURCE_NAME_SETTING, - data_source.name + Collection.DATA_SOURCE_NAME_SETTING, data_source.name + ) + license_status_response = MagicMock( + side_effect=[ + (200, {}, str(license_status) if license_status else "{}") for license_status in license_infos + ] + if license_infos + else [(200, {}, {})] ) - license_status = { - "checkouts": { - "available": 1 - } - } - license_status_response = MagicMock(return_value=(200, {}, json.dumps(license_status))) importer = self._create_importer(collection, license_status_response) - imported_editions, imported_pools, imported_works, _ = ( - importer.import_from_feed(feed) - ) + ( + imported_editions, + imported_pools, + imported_works, + _, + ) = importer.import_from_feed(feed) return imported_editions, imported_pools, imported_works + +class TestODLExpiredItemsReaperSingleLicense(TestODLExpiredItemsReaper): + """Class testing that the ODL 1.x reaper correctly processes publications with a single license.""" + + @parameterized.expand([ + ( + "expiration_date_in_the_past", + # The license expires 2021-01-01T00:01:00+01:00 that equals to 2010-01-01T00:00:00+00:00, the current time. + # It means the license had already expired at the time of the import. + TestLicense(expires=dateutil.parser.isoparse("2021-01-01T00:01:00+01:00")) + ), + ( + "total_checkouts_is_zero", + TestLicense(total_checkouts=0) + ), + ( + "remaining_checkouts_is_zero", + TestLicense(total_checkouts=10, concurrent_checkouts=5), + TestLicenseInfo(remaining_checkouts=0, available_concurrent_checkouts=0) + ) + ]) @freeze_time("2021-01-01T00:00:00+00:00") - def test_odl_importer_skips_expired_licenses(self): + def test_odl_importer_skips_expired_licenses( + self, + _, + test_license: TestLicense, + test_license_info: Optional[TestLicenseInfo] = None + ) -> None: """Ensure ODLImporter skips expired licenses - and does not count them in the total number of available licenses.""" + and does not count them in the total number of available licenses. + + :param test_license: An example of an expired ODL license + :param test_license_info: An example of an ODL License Info Document belonging to an expired ODL license + (if required) + """ # 1.1. Import the test feed with an expired ODL license. - # The license expires 2021-01-01T00:01:00+01:00 that equals to 2010-01-01T00:00:00+00:00, the current time. - # It means the license had already expired at the time of the import. - license_expiration_date = datetime.datetime(2021, 1, 1, 1, 0, 0, tzinfo=tzoffset(None, self.SECONDS_PER_HOUR)) - imported_editions, imported_pools, imported_works = self._import_test_feed_with_single_odl_license( - license_expiration_date + imported_editions, imported_pools, imported_works = self._import_test_feed( + [test_license], + [test_license_info] ) # Commit to expire the SQLAlchemy cache. @@ -2033,9 +2173,28 @@ def test_odl_reaper_removes_expired_licenses(self): # 1.1. Import the test feed with an ODL license that is still valid. # The license will be valid for one more day since this very moment. - license_expiration_date = datetime_helpers.utc_now() + datetime.timedelta(days=1) - imported_editions, imported_pools, imported_works = self._import_test_feed_with_single_odl_license( - license_expiration_date + # The feed declares that there 10 checkouts available in total + # but the License Info Document shows that there are only 9 available at the moment of import. + total_checkouts = 10 + available_concurrent_checkouts = 5 + remaining_checkouts = 9 + license_expiration_date = datetime_helpers.utc_now() + datetime.timedelta( + days=1 + ) + imported_editions, imported_pools, imported_works = self._import_test_feed( + [ + TestLicense( + expires=license_expiration_date, + total_checkouts=total_checkouts, + concurrent_checkouts=available_concurrent_checkouts, + ) + ], + [ + TestLicenseInfo( + remaining_checkouts=remaining_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts, + ) + ], ) # Commit to expire the SQLAlchemy cache. @@ -2045,8 +2204,8 @@ def test_odl_reaper_removes_expired_licenses(self): assert len(imported_pools) == 1 [imported_pool] = imported_pools - assert imported_pool.licenses_owned == 1 - assert imported_pool.licenses_available == 1 + assert imported_pool.licenses_owned == remaining_checkouts + assert imported_pool.licenses_available == available_concurrent_checkouts assert len(imported_pool.licenses) == 1 [license] = imported_pool.licenses @@ -2056,37 +2215,210 @@ def test_odl_reaper_removes_expired_licenses(self): loan, _ = license.loan_to(patron) # 3.1. Run ODLExpiredItemsReaper. This time nothing should happen since the license is still valid. - script = RunCollectionMonitorScript(self.ODL_REAPER_CLASS, _db=self._db, cmd_args=["Test ODL Collection"]) + script = RunCollectionMonitorScript( + self.ODL_REAPER_CLASS, _db=self._db, cmd_args=["Test ODL Collection"] + ) script.run() # Commit to expire the SQLAlchemy cache. self._db.commit() # 3.2. Ensure that availability of the license pool didn't change. - assert imported_pool.licenses_owned == 1 - assert imported_pool.licenses_available == 1 + assert imported_pool.licenses_owned == remaining_checkouts + assert imported_pool.licenses_available == available_concurrent_checkouts # 4. Expire the license. - # Set the expiration date to 2021-01-01T00:01:00+01:00 - # that equals to 2010-01-01T00:00:00+00:00, the current time. - license.expires = datetime.datetime(2021, 1, 1, 1, 0, 0, tzinfo=tzoffset(None, self.SECONDS_PER_HOUR)) + with patch("core.model.License.is_expired", new_callable=PropertyMock) as is_expired: + is_expired.return_value = True + + # 5.1. Run ODLExpiredItemsReaper again. This time it should remove the expired license. + script.run() + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 5.2. Ensure that availability of the license pool was updated + # and now it doesn't have any available licenses. + assert imported_pool.licenses_owned == 0 + assert imported_pool.licenses_available == 0 + + # 6.1. Run ODLExpiredItemsReaper again to ensure that number of licenses won't become negative. + script.run() + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 6.2. Ensure that number of licenses is still 0. + assert imported_pool.licenses_owned == 0 + assert imported_pool.licenses_available == 0 + + +class TestODLExpiredItemsReaperMultipleLicense(TestODLExpiredItemsReaper): + """Class testing that the ODL 1.x reaper correctly processes publications with multiple licenses.""" + + @freeze_time("2021-01-01T00:00:00+00:00") + def test_odl_importer_skips_expired_licenses(self): + """Ensure ODLImporter skips expired licenses + and does not count them in the total number of available licenses.""" + # 1.1. Import the test feed with three expired ODL licenses and two valid licenses. + remaining_checkouts = 9 + available_concurrent_checkouts = 5 + imported_editions, imported_pools, imported_works = self._import_test_feed( + [ + TestLicense( # Expired + total_checkouts=10, # (expiry date in the past) + concurrent_checkouts=5, + expires=datetime_helpers.utc_now() - datetime.timedelta(days=1), + ), + TestLicense( # Expired + total_checkouts=0, # (total_checkouts is 0) + concurrent_checkouts=0, + expires=datetime_helpers.utc_now() + datetime.timedelta(days=1), + ), + TestLicense( # Expired + total_checkouts=10, # (remaining_checkout is 0) + concurrent_checkouts=5, + expires=datetime_helpers.utc_now() + datetime.timedelta(days=1), + ), + TestLicense( # Valid + total_checkouts=10, + concurrent_checkouts=5, + expires=datetime_helpers.utc_now() + datetime.timedelta(days=2), + ), + TestLicense( # Valid + total_checkouts=10, + concurrent_checkouts=5, + expires=datetime_helpers.utc_now() + datetime.timedelta(weeks=12), + ), + ], + [ + TestLicenseInfo( + remaining_checkouts=0, + available_concurrent_checkouts=0 + ), + TestLicenseInfo( + remaining_checkouts=remaining_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts + ), + TestLicenseInfo( + remaining_checkouts=remaining_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts + ) + ], + ) + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 1.2. Ensure that the license pool was successfully created + assert len(imported_pools) == 1 + [imported_pool] = imported_pools - # 5.1. Run ODLExpiredItemsReaper again. This time it should remove the expired license. + # 1.3. Ensure that the two valid licenses were imported. + assert len(imported_pool.licenses) == 2 + + # 1.4 Make sure that the license statistics is correct and include only checkouts owned by two valid licenses. + assert imported_pool.licenses_owned == remaining_checkouts * 2 + assert imported_pool.licenses_available == available_concurrent_checkouts * 2 + + @freeze_time("2021-01-01T00:00:00+00:00") + def test_odl_reaper_removes_expired_licenses(self): + """Ensure ODLExpiredItemsReaper removes expired licenses.""" + # 1.1. Import the test feed with ODL licenses that are not expired. + total_checkouts = 10 + remaining_checkouts = 9 + available_concurrent_checkouts = 5 + imported_editions, imported_pools, imported_works = self._import_test_feed( + [ + TestLicense( + total_checkouts=total_checkouts, + concurrent_checkouts=available_concurrent_checkouts, + expires=datetime_helpers.utc_now() + datetime.timedelta(days=1), + ), + TestLicense( + total_checkouts=total_checkouts, + concurrent_checkouts=available_concurrent_checkouts, + expires=datetime_helpers.utc_now() + datetime.timedelta(days=2), + ), + TestLicense( + total_checkouts=total_checkouts, + concurrent_checkouts=available_concurrent_checkouts, + expires=datetime_helpers.utc_now() + datetime.timedelta(weeks=12), + ), + ], + [ + TestLicenseInfo( + remaining_checkouts=total_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts + ), + TestLicenseInfo( + remaining_checkouts=remaining_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts + ), + TestLicenseInfo( + remaining_checkouts=remaining_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts + ) + ], + ) + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 1.2. Ensure that there is a license pool with available license. + assert len(imported_pools) == 1 + + [imported_pool] = imported_pools + assert len(imported_pool.licenses) == 3 + + [license1, license2, license3] = imported_pool.licenses + assert license1.remaining_checkouts == total_checkouts + assert license1.concurrent_checkouts == available_concurrent_checkouts + + assert license2.remaining_checkouts == remaining_checkouts + assert license2.concurrent_checkouts == available_concurrent_checkouts + + assert license3.remaining_checkouts == remaining_checkouts + assert license3.concurrent_checkouts == available_concurrent_checkouts + + assert imported_pool.licenses_owned == total_checkouts + 2 * remaining_checkouts + assert imported_pool.licenses_available == 3 * available_concurrent_checkouts + + # 2.1. Run ODLExpiredItemsReaper. This time nothing should happen since the license is still valid. + script = RunCollectionMonitorScript( + self.ODL_REAPER_CLASS, _db=self._db, cmd_args=["Test ODL Collection"] + ) script.run() # Commit to expire the SQLAlchemy cache. self._db.commit() - # 5.2. Ensure that availability of the license pool was updated and now it doesn't have any available licenses. - assert imported_pool.licenses_owned == 0 - assert imported_pool.licenses_available == 0 + # 2.2. Ensure that availability of the license pool didn't change. + assert len(imported_pool.licenses) == 3 + assert imported_pool.licenses_owned == total_checkouts + 2 * remaining_checkouts + assert imported_pool.licenses_available == 3 * available_concurrent_checkouts - # 6.1. Run ODLExpiredItemsReaper again to ensure that number of licenses won't become negative. + # 3. Expire the license. + license1.expires = datetime_helpers.utc_now() - datetime.timedelta(days=1) + + # 3.1. Run ODLExpiredItemsReaper again. This time it should remove the expired license. script.run() # Commit to expire the SQLAlchemy cache. self._db.commit() - # 6.2. Ensure that number of licenses is still 0. - assert imported_pool.licenses_owned == 0 - assert imported_pool.licenses_available == 0 + # 3.2. Ensure that availability of the license pool was updated. + assert len(imported_pool.licenses) == 3 + assert imported_pool.licenses_owned == 2 * remaining_checkouts + assert imported_pool.licenses_available == 2 * available_concurrent_checkouts + + # 4.1. Run ODLExpiredItemsReaper again to make sure that licenses are not expired twice. + script.run() + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 4.2. Ensure that number of licenses remains the same as in step 3.2. + assert len(imported_pool.licenses) == 3 + assert imported_pool.licenses_owned == 2 * remaining_checkouts + assert imported_pool.licenses_available == 2 * available_concurrent_checkouts diff --git a/tests/test_odl2.py b/tests/test_odl2.py index 85d4370096..cadf2c0d88 100644 --- a/tests/test_odl2.py +++ b/tests/test_odl2.py @@ -3,7 +3,10 @@ import os import requests_mock +from api.odl import ODLImporter +from api.odl2 import ODL2API, ODL2APIConfiguration, ODL2ExpiredItemsReaper, ODL2Importer from freezegun import freeze_time +from mock import MagicMock from webpub_manifest_parser.core.ast import PresentationMetadata from webpub_manifest_parser.odl import ODLFeedParserFactory from webpub_manifest_parser.odl.ast import ODLPublication @@ -11,8 +14,6 @@ ODL_PUBLICATION_MUST_CONTAIN_EITHER_LICENSES_OR_OA_ACQUISITION_LINK_ERROR, ) -from api.odl import ODLImporter -from api.odl2 import ODL2API, ODL2APIConfiguration, ODL2ExpiredItemsReaper, ODL2Importer from core.coverage import CoverageFailure from core.model import ( Contribution, @@ -28,7 +29,11 @@ from core.model.configuration import ConfigurationFactory, ConfigurationStorage from core.opds2_import import RWPMManifestParser from core.tests.test_opds2_import import OPDS2Test -from tests.test_odl import TestODLExpiredItemsReaper +from tests.test_odl import ( + TestODLExpiredItemsReaper, + TestODLExpiredItemsReaperMultipleLicense, + TestODLExpiredItemsReaperSingleLicense, +) class TestODL2Importer(OPDS2Test): @@ -254,11 +259,11 @@ def test(self): class TestODL2ExpiredItemsReaper(TestODLExpiredItemsReaper): - __base_path = os.path.split(__file__)[0] - resource_path = os.path.join(__base_path, "files", "odl2") + """Base class for all ODL 2.x reaper tests.""" ODL_PROTOCOL = ODL2API.NAME - ODL_FEED_FILENAME_WITH_SINGLE_ODL_LICENSE = "single_license.json" + ODL_TEMPLATE_DIR = os.path.join(TestODLExpiredItemsReaper.base_path, "files", "odl2") + ODL_TEMPLATE_FILENAME = "feed_template.json.jinja" ODL_REAPER_CLASS = ODL2ExpiredItemsReaper def _create_importer(self, collection, http_get): @@ -280,5 +285,14 @@ def _create_importer(self, collection, http_get): parser=RWPMManifestParser(ODLFeedParserFactory()), http_get=http_get, ) + importer._is_identifier_allowed = MagicMock(return_value=True) return importer + + +class TestODL2ExpiredItemsReaperSingleLicense(TestODL2ExpiredItemsReaper, TestODLExpiredItemsReaperSingleLicense): + """Class testing that the ODL 2.x reaper correctly processes publications with a single license.""" + + +class TestODL2ExpiredItemsReaperMultipleLicense(TestODL2ExpiredItemsReaper, TestODLExpiredItemsReaperMultipleLicense): + """Class testing that the ODL 2.x reaper correctly processes publications with multiple licenses."""