diff --git a/api/odl.py b/api/odl.py index dae53c0ae..7ab1c2ee5 100644 --- a/api/odl.py +++ b/api/odl.py @@ -1,83 +1,52 @@ import datetime -import dateutil import json +import logging import uuid -from flask_babel import lazy_gettext as _ -import urllib.parse -from collections import defaultdict -import flask -from flask import Response -import feedparser -from lxml import etree -from .problem_details import NO_LICENSES from io import StringIO -import re -from uritemplate import URITemplate +from typing import Callable, Optional +import dateutil +import feedparser +import flask +from flask import url_for +from flask_babel import lazy_gettext as _ +from lxml import etree from sqlalchemy.sql.expression import or_ +from uritemplate import URITemplate -from core.opds_import import ( - OPDSXMLParser, - OPDSImporter, - OPDSImportMonitor, -) -from core.monitor import ( - CollectionMonitor, - TimelineMonitor, -) +from core import util +from core.analytics import Analytics +from core.metadata_layer import CirculationData, FormatData, LicenseData, TimestampData from core.model import ( Collection, ConfigurationSetting, - Credential, DataSource, DeliveryMechanism, Edition, ExternalIntegration, Hold, Hyperlink, - Identifier, - IntegrationClient, LicensePool, Loan, MediaTypes, + Representation, RightsStatus, Session, - create, get_one, get_one_or_create, ) -from core.metadata_layer import ( - CirculationData, - FormatData, - IdentifierData, - LicenseData, - TimestampData, -) -from .circulation import ( - BaseCirculationAPI, - LoanInfo, - FulfillmentInfo, - HoldInfo, -) -from core.analytics import Analytics -from core.util.datetime_helpers import ( - utc_now, - strptime_utc, -) -from core.util.http import ( - HTTP, - BadResponseException, - RemoteIntegrationException, -) +from core.monitor import CollectionMonitor, IdentifierSweepMonitor +from core.opds_import import OPDSImporter, OPDSImportMonitor, OPDSXMLParser +from core.testing import DatabaseTest, MockRequestsResponse +from core.util.datetime_helpers import utc_now +from core.util.http import HTTP, BadResponseException, RemoteIntegrationException from core.util.string_helpers import base64 -from flask import url_for -from core.testing import ( - DatabaseTest, - MockRequestsResponse, -) + +from .circulation import BaseCirculationAPI, FulfillmentInfo, HoldInfo, LoanInfo from .circulation_exceptions import * from .shared_collection import BaseSharedCollectionAPI + class ODLAPI(BaseCirculationAPI, BaseSharedCollectionAPI): """ODL (Open Distribution to Libraries) is a specification that allows libraries to manage their own loans and holds. It offers a deeper level @@ -596,7 +565,7 @@ def update_hold_queue(self, licensepool): Loan.end>utc_now() ) ).count() - remaining_licenses = licensepool.licenses_owned - loans_count + remaining_licenses = max(licensepool.licenses_owned - loans_count, 0) holds = _db.query(Hold).filter( Hold.license_pool_id==licensepool.id @@ -782,6 +751,7 @@ class ODLXMLParser(OPDSXMLParser): NAMESPACES = dict(OPDSXMLParser.NAMESPACES, odl="http://opds-spec.org/odl") + class ODLImporter(OPDSImporter): """Import information and formats from an ODL feed. @@ -795,6 +765,105 @@ class ODLImporter(OPDSImporter): # about the license. LICENSE_INFO_DOCUMENT_MEDIA_TYPE = 'application/vnd.odl.info+json' + @classmethod + def parse_license( + cls, + identifier: str, + total_checkouts: Optional[int], + concurrent_checkouts: Optional[int], + expires: Optional[datetime.datetime], + checkout_link: Optional[str], + odl_status_link: Optional[str], + do_get: Callable + ) -> Optional[LicenseData]: + """Check the license's attributes passed as parameters: + - if they're correct, turn them into a LicenseData object + - otherwise, return a None + + :param identifier: License's identifier + :param total_checkouts: Total number of checkouts before the license expires + :param concurrent_checkouts: Number of concurrent checkouts allowed for this license + :param expires: Date & time until the license is valid + :param checkout_link: License's checkout link + :param odl_status_link: License Info Document's link + :param do_get: Callback performing HTTP GET method + + :return: LicenseData if all the license's attributes are correct, None, otherwise + """ + remaining_checkouts = None + available_concurrent_checkouts = None + + # This cycle ends in two different cases: + # - when at least one of the parameters is invalid; in this case, the method returns None. + # - when all the parameters are valid; in this case, the method returns a LicenseData object. + while True: + if total_checkouts is not None: + total_checkouts = int(total_checkouts) + + if total_checkouts <= 0: + logging.info( + f"License # {identifier} expired since " + f"the total number of checkouts is {total_checkouts}" + ) + break + + if expires: + if not isinstance(expires, datetime.datetime): + expires = dateutil.parser.parse(expires) + + expires = util.datetime_helpers.to_utc(expires) + now = util.datetime_helpers.utc_now() + + if expires <= now: + logging.info( + f"License # {identifier} expired at {expires} (now is {now})" + ) + break + + if odl_status_link: + status_code, _, response = do_get( + odl_status_link, headers={} + ) + + if status_code in (200, 201): + status = json.loads(response) + checkouts = status.get("checkouts", {}) + remaining_checkouts = checkouts.get("left") + available_concurrent_checkouts = checkouts.get("available") + else: + logging.warning( + f"License # {identifier}'s Info Document is not available. " + f"Status link failed with {status_code} code" + ) + break + + if remaining_checkouts is None: + remaining_checkouts = total_checkouts + + if remaining_checkouts is not None: + remaining_checkouts = int(remaining_checkouts) + + if remaining_checkouts <= 0: + logging.info( + f"License # {identifier} expired since " + f"the remaining number of checkouts is {remaining_checkouts}" + ) + break + + if available_concurrent_checkouts is None: + available_concurrent_checkouts = concurrent_checkouts + + return LicenseData( + identifier=identifier, + checkout_url=checkout_link, + status_url=odl_status_link, + expires=expires, + remaining_checkouts=remaining_checkouts, + concurrent_checkouts=available_concurrent_checkouts, + ) + + return None + @classmethod def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get=None): do_get = do_get or Representation.cautious_http_get @@ -852,11 +921,6 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get= data['medium'] = medium - expires = None - remaining_checkouts = None - available_checkouts = None - concurrent_checkouts = None - checkout_link = None for link_tag in parser._xpath(odl_license_tag, 'odl:tlink') or []: rel = link_tag.attrib.get("rel") @@ -875,32 +939,33 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get= odl_status_link = attrib.get("href") break - # If we found one, retrieve it and get licensing information about this book. - if odl_status_link: - ignore, ignore, response = do_get(odl_status_link, headers={}) - status = json.loads(response) - checkouts = status.get("checkouts", {}) - remaining_checkouts = checkouts.get("left") - available_checkouts = checkouts.get("available") + expires = None + total_checkouts = None + concurrent_checkouts = None terms = parser._xpath(odl_license_tag, "odl:terms") if terms: + total_checkouts = subtag(terms[0], "odl:total_checkouts") concurrent_checkouts = subtag(terms[0], "odl:concurrent_checkouts") expires = subtag(terms[0], "odl:expires") - if expires: - expires = dateutil.parser.parse(expires) - licenses_owned += int(concurrent_checkouts or 0) - licenses_available += int(available_checkouts or 0) + license = cls.parse_license( + identifier, + total_checkouts, + concurrent_checkouts, + expires, + checkout_link, + odl_status_link, + do_get + ) - licenses.append(LicenseData( - identifier=identifier, - checkout_url=checkout_link, - status_url=odl_status_link, - expires=expires, - remaining_checkouts=remaining_checkouts, - concurrent_checkouts=concurrent_checkouts, - )) + if not license: + continue + + licenses_owned += int(license.remaining_checkouts or 0) + licenses_available += int(license.concurrent_checkouts or 0) + + licenses.append(license) if not data.get('circulation'): data['circulation'] = dict() @@ -914,6 +979,7 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get= data['circulation']['licenses_available'] = licenses_available return data + class ODLImportMonitor(OPDSImportMonitor): """Import information from an ODL feed.""" PROTOCOL = ODLImporter.NAME @@ -959,11 +1025,12 @@ def run_once(self, progress): progress = TimestampData(achievements=message) return progress + class MockODLAPI(ODLAPI): """Mock API for tests that overrides _get and _url_for and tracks requests.""" @classmethod - def mock_collection(self, _db): + def mock_collection(cls, _db, protocol=ODLAPI.NAME): """Create a mock ODL collection to use in tests.""" library = DatabaseTest.make_default_library(_db) collection, ignore = get_one_or_create( @@ -973,7 +1040,7 @@ def mock_collection(self, _db): ) ) integration = collection.create_external_integration( - protocol=ODLAPI.NAME + protocol=protocol ) integration.username = 'a' integration.password = 'b' @@ -1040,6 +1107,25 @@ def __init__(self, _db, collection): self.base_url = collection.external_account_id + @staticmethod + def _parse_feed_from_response(response): + """Parse ODL (Atom) feed from the HTTP response. + + :param response: HTTP response + :type response: requests.Response + + :return: Parsed ODL (Atom) feed + :rtype: dict + """ + response_content = response.content + + if not isinstance(response_content, (str, bytes)): + raise ValueError("Response content must be a string or byte-encoded value") + + feed = feedparser.parse(response_content) + + return feed + def internal_format(self, delivery_mechanism): """Each consolidated copy is only available in one format, so we don't need a mapping to internal formats. @@ -1091,7 +1177,8 @@ def checkout(self, patron, pin, licensepool, internal_format): hold_info_response = self._get(hold.external_identifier) except RemoteIntegrationException as e: raise CannotLoan() - feed = feedparser.parse(str(hold_info_response.content)) + + feed = self._parse_feed_from_response(hold_info_response) entries = feed.get("entries") if len(entries) < 1: raise CannotLoan() @@ -1117,7 +1204,8 @@ def checkout(self, patron, pin, licensepool, internal_format): elif response.status_code == 404: if hasattr(response, 'json') and response.json().get('type', '') == NO_LICENSES.uri: raise NoLicenses() - feed = feedparser.parse(str(response.content)) + + feed = self._parse_feed_from_response(response) entries = feed.get("entries") if len(entries) < 1: raise CannotLoan() @@ -1181,7 +1269,8 @@ def checkin(self, patron, pin, licensepool): raise CannotReturn() if response.status_code == 404: raise NotCheckedOut() - feed = feedparser.parse(str(response.content)) + + feed = self._parse_feed_from_response(response) entries = feed.get("entries") if len(entries) < 1: raise CannotReturn() @@ -1286,7 +1375,8 @@ def release_hold(self, patron, pin, licensepool): raise CannotReleaseHold() if response.status_code == 404: raise NotOnHold() - feed = feedparser.parse(str(response.content)) + + feed = self._parse_feed_from_response(response) entries = feed.get("entries") if len(entries) < 1: raise CannotReleaseHold() @@ -1325,7 +1415,7 @@ def patron_activity(self, patron, pin): if response.status_code == 404: # 404 is returned when the loan has been deleted. Leave this loan out of the result. continue - feed = feedparser.parse(str(response.content)) + feed = self._parse_feed_from_response(response) entries = feed.get("entries") if len(entries) < 1: raise CirculationException() @@ -1354,7 +1444,7 @@ def patron_activity(self, patron, pin): if response.status_code == 404: # 404 is returned when the hold has been deleted. Leave this hold out of the result. continue - feed = feedparser.parse(str(response.content)) + feed = self._parse_feed_from_response(response) entries = feed.get("entries") if len(entries) < 1: raise CirculationException() @@ -1518,3 +1608,41 @@ def _get(self, url, patron=None, headers=None, allowed_response_codes=None): self.request_args.append((patron, headers, allowed_response_codes)) response = self.responses.pop() return HTTP._process_response(url, response, allowed_response_codes=allowed_response_codes) + + +class ODLExpiredItemsReaper(IdentifierSweepMonitor): + """Responsible for removing expired ODL licenses.""" + + SERVICE_NAME = "ODL Expired Items Reaper" + PROTOCOL = ODLAPI.NAME + + def __init__(self, _db, collection): + super(ODLExpiredItemsReaper, self).__init__(_db, collection) + + def process_item(self, identifier): + for licensepool in identifier.licensed_through: + remaining_checkouts = 0 # total number of checkouts across all the licenses in the pool + concurrent_checkouts = 0 # number of concurrent checkouts allowed across all the licenses in the pool + + # 0 is a starting point, + # we're going through all the valid licenses in the pool and count up available checkouts. + for license_pool_license in licensepool.licenses: + if not license_pool_license.is_expired: + remaining_checkouts += license_pool_license.remaining_checkouts + concurrent_checkouts += license_pool_license.concurrent_checkouts + + if remaining_checkouts != licensepool.licenses_owned or \ + concurrent_checkouts != licensepool.licenses_available: + licenses_owned = max(remaining_checkouts, 0) + licenses_available = max(concurrent_checkouts, 0) + + circulation_data = CirculationData( + data_source=licensepool.data_source, + primary_identifier=identifier, + licenses_owned=licenses_owned, + licenses_available=licenses_available, + licenses_reserved=licensepool.licenses_reserved, + patrons_in_hold_queue=licensepool.patrons_in_hold_queue, + ) + + circulation_data.apply(self._db, self.collection) diff --git a/bin/odl_reaper b/bin/odl_reaper new file mode 100755 index 000000000..66dd03f7d --- /dev/null +++ b/bin/odl_reaper @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +"""Remove all expired licenses from ODL 1.x collections.""" +import os +import sys +bin_dir = os.path.split(__file__)[0] +package_dir = os.path.join(bin_dir, "..") +sys.path.append(os.path.abspath(package_dir)) +from core.scripts import RunCollectionMonitorScript +from api.odl import ODLExpiredItemsReaper +RunCollectionMonitorScript(ODLExpiredItemsReaper).run() diff --git a/docker/services/simplified_crontab b/docker/services/simplified_crontab index fba32d6ab..c0d914dfd 100644 --- a/docker/services/simplified_crontab +++ b/docker/services/simplified_crontab @@ -113,6 +113,7 @@ HOME=/var/www/circulation # 0 6 * * * root core/bin/run odl_import_monitor >> /var/log/cron.log 2>&1 0 */8 * * * root core/bin/run odl_hold_reaper >> /var/log/cron.log 2>&1 +0 */14 * * * root core/bin/run odl_reaper >> /var/log/cron.log 2>&1 5 */6 * * * root core/bin/run shared_odl_import_monitor >> /var/log/cron.log 2>&1 # Odilo diff --git a/tests/files/odl/feed_template.xml.jinja b/tests/files/odl/feed_template.xml.jinja new file mode 100644 index 000000000..0c1580d7c --- /dev/null +++ b/tests/files/odl/feed_template.xml.jinja @@ -0,0 +1,92 @@ + + + https://market.feedbooks.com/api/libraries/harvest.atom + Feedbooks + 2021-08-16T09:07:14Z + /favicon.ico + + Feedbooks + https://market.feedbooks.com + support@feedbooks.zendesk.com + + + 481 + 100 + + + The Golden State + https://www.feedbooks.com/item/2895246 + urn:ISBN:9780374718060 + urn:ISBN:9780374164836 + + Lydia Kiesling + https://market.feedbooks.com/store/browse/recent.atom?author_id=954566&lang=en + + 2018-08-12T00:16:43Z + 2020-05-21T11:13:26Z + en + Mcd + 2018-09-03 + NATIONAL BOOK FOUNDATION 5 UNDER 35 PICK. LONGLISTED FOR THE CENTER FOR FICTION'S FIRST NOVEL PRIZE. Named one of the Best Books of 2018 by NPR, Bookforum and Bustle. One of Entertainment Weekly's 10 Best Debut Novels of 2018. An Amazon Best Book of the Month and named a fall read by Buzzfeed, Nylon, Entertainment Weekly, Elle, Vanity Fair, Vulture, Refinery29 and Mind Body GreenA gorgeous, raw debut novel about a young woman braving the ups and downs of motherhood in a fractured AmericaIn Lydia Kiesling's razor-sharp debut novel, The Golden State, we accompany Daphne, a young mother on the edge of a breakdown, as she flees her sensible but strained life in San Francisco for the high desert of Altavista with her toddler, Honey. Bucking under the weight of being a single parent--her Turkish husband is unable to return to the United States because of a "processing error"--Daphne takes refuge in a mobile home left to her by her grandparents in hopes that the quiet will bring clarity. But clarity proves elusive. Over the next ten days Daphne is anxious, she behaves a little erratically, she drinks too much. She wanders the town looking for anyone and anything to punctuate the long hours alone with the baby. Among others, she meets Cindy, a neighbor who is active in a secessionist movement, and befriends the elderly Alice, who has traveled to Altavista as she approaches the end of her life. When her relationships with these women culminate in a dangerous standoff, Daphne must reconcile her inner narrative with the reality of a deeply divided world. Keenly observed, bristling with humor, and set against the beauty of a little-known part of California, The Golden State is about class and cultural breakdowns, and desperate attempts to bridge old and new worlds. But more than anything, it is about motherhood: its voracious worry, frequent tedium, and enthralling, wondrous love. + 4 MB + + + + + + + + + 40.00 + + + + + + + + + {% for license in licenses %} + + urn:uuid:{{ license.identifier }} + application/epub+zip + text/html + http://www.cantook.net/ + 40.00 + cant-2461538-24501117858552614-libraries + 2020-03-02T20:20:17+01:00 + + {% if license.total_checkouts is not none %} + {{ license.total_checkouts }} + {% endif %} + + {% if license.concurrent_checkouts is not none %} + {{ license.concurrent_checkouts }} + {% endif %} + + {% if license.expires is not none %} + {{ license.expires.isoformat() }} + {% endif %} + + 5097600 + + + application/vnd.adobe.adept+xml + 6 + true + false + false + + + application/vnd.readium.lcp.license.v1.0+json + 6 + true + false + false + + + + + {% endfor %} + + diff --git a/tests/test_odl.py b/tests/test_odl.py index a3b19deaf..3bc835273 100644 --- a/tests/test_odl.py +++ b/tests/test_odl.py @@ -1,50 +1,52 @@ -import pytest -import os -import json import datetime -import dateutil -import re +import json +import os import urllib.parse -from pdb import set_trace -from core.testing import DatabaseTest -from core.metadata_layer import TimestampData +import uuid +from typing import List, Optional, Tuple + +import dateutil +import pytest +from api.circulation_exceptions import * +from api.odl import ( + ODLAPI, + MockODLAPI, + MockSharedODLAPI, + ODLExpiredItemsReaper, + ODLHoldReaper, + ODLImporter, + SharedODLAPI, + SharedODLImporter, +) +from freezegun import freeze_time +from jinja2 import Environment, FileSystemLoader, select_autoescape +from mock import MagicMock, PropertyMock, patch +from parameterized import parameterized + from core.model import ( Collection, ConfigurationSetting, - Credential, DataSource, DeliveryMechanism, Edition, ExternalIntegration, Hold, Hyperlink, - Identifier, + LicensePool, Loan, MediaTypes, Representation, RightsStatus, - get_one, -) -from api.odl import ( - ODLImporter, - ODLHoldReaper, - MockODLAPI, - SharedODLAPI, - MockSharedODLAPI, - SharedODLImporter, -) -from api.circulation_exceptions import * -from core.util.datetime_helpers import ( - datetime_utc, - strptime_utc, - utc_now, -) -from core.util.http import ( - BadResponseException, - RemoteIntegrationException, + Work, ) +from core.scripts import RunCollectionMonitorScript +from core.testing import DatabaseTest +from core.util import datetime_helpers +from core.util.datetime_helpers import datetime_utc, utc_now +from core.util.http import BadResponseException, RemoteIntegrationException from core.util.string_helpers import base64 + class BaseODLTest(object): base_path = os.path.split(__file__)[0] resource_path = os.path.join(base_path, "files", "odl") @@ -52,7 +54,8 @@ class BaseODLTest(object): @classmethod def get_data(cls, filename): path = os.path.join(cls.resource_path, filename) - return open(path, "rb").read() + return open(path, "r").read() + class TestODLAPI(DatabaseTest, BaseODLTest): @@ -1290,8 +1293,12 @@ def test_release_hold_from_external_library(self): class TestODLImporter(DatabaseTest, BaseODLTest): - + @freeze_time("2019-01-01T00:00:00+00:00") def test_import(self): + """Ensure that ODLImporter correctly processes and imports the ODL feed encoded using OPDS 1.x. + + NOTE: `freeze_time` decorator is required to treat the licenses in the ODL feed as non-expired. + """ feed = self.get_data("feedbooks_bibliographic.atom") data_source = DataSource.lookup(self._db, "Feedbooks", autocreate=True) collection = MockODLAPI.mock_collection(self._db) @@ -1305,7 +1312,7 @@ def canonicalize_author_name(self, identifier, working_display_name): return working_display_name metadata_client = MockMetadataClient() - warrior_time_limited = dict(checkouts=dict(available=1)) + warrior_time_limited = dict(checkouts=dict(left=52, available=1)) canadianity_loan_limited = dict(checkouts=dict(left=40, available=10)) canadianity_perpetual = dict(checkouts=dict(available=1)) midnight_loan_limited_1 = dict(checkouts=dict(left=20, available=1)) @@ -1365,7 +1372,7 @@ def do_get(url, headers): assert Representation.EPUB_MEDIA_TYPE == lpdm.delivery_mechanism.content_type assert DeliveryMechanism.ADOBE_DRM == lpdm.delivery_mechanism.drm_scheme assert RightsStatus.IN_COPYRIGHT == lpdm.rights_status.uri - assert 1 == warrior_pool.licenses_owned + assert 52 == warrior_pool.licenses_owned # 52 remaining checkouts in the License Info Document assert 1 == warrior_pool.licenses_available [license] = warrior_pool.licenses assert "1" == license.identifier @@ -1382,7 +1389,7 @@ def do_get(url, headers): assert datetime.datetime( 2019, 3, 31, 3, 13, 35, tzinfo=dateutil.tz.tzoffset("", 3600*2) ) == license.expires - assert None == license.remaining_checkouts + assert 52 == license.remaining_checkouts # 52 remaining checkouts in the License Info Document assert 1 == license.concurrent_checkouts # This item is an open access audiobook. @@ -1416,7 +1423,7 @@ def do_get(url, headers): assert Representation.EPUB_MEDIA_TYPE == lpdm.delivery_mechanism.content_type assert DeliveryMechanism.ADOBE_DRM == lpdm.delivery_mechanism.drm_scheme assert RightsStatus.IN_COPYRIGHT == lpdm.rights_status.uri - assert 11 == canadianity_pool.licenses_owned + assert 40 == canadianity_pool.licenses_owned # 40 remaining checkouts in the License Info Document assert 11 == canadianity_pool.licenses_available [license1, license2] = sorted(canadianity_pool.licenses, key=lambda x: x.identifier) assert "2" == license1.identifier @@ -1448,7 +1455,7 @@ def do_get(url, headers): [lpdm.delivery_mechanism.drm_scheme for lpdm in lpdms]) assert ([RightsStatus.IN_COPYRIGHT, RightsStatus.IN_COPYRIGHT] == [lpdm.rights_status.uri for lpdm in lpdms]) - assert 2 == midnight_pool.licenses_owned + assert 72 == midnight_pool.licenses_owned # 20 + 52 remaining checkouts in corresponding License Info Documents assert 2 == midnight_pool.licenses_available [license1, license2] = sorted(midnight_pool.licenses, key=lambda x: x.identifier) assert "4" == license1.identifier @@ -1517,7 +1524,6 @@ def test_run_once(self): assert None == progress.finish - class TestSharedODLAPI(DatabaseTest, BaseODLTest): def setup_method(self): @@ -1847,6 +1853,7 @@ def test_patron_activity_remote_integration_exception(self): pytest.raises(RemoteIntegrationException, self.api.patron_activity, self.patron, "pin") assert [hold.external_identifier] == self.api.requests[1:] + class TestSharedODLImporter(DatabaseTest, BaseODLTest): def test_get_fulfill_url(self): @@ -1925,7 +1932,493 @@ def canonicalize_author_name(self, identifier, working_display_name): assert Representation.EPUB_MEDIA_TYPE == lpdm.delivery_mechanism.content_type assert DeliveryMechanism.ADOBE_DRM == lpdm.delivery_mechanism.drm_scheme assert RightsStatus.IN_COPYRIGHT == lpdm.rights_status.uri - [borrow_link] = [l for l in essex_pool.identifier.links if l.rel == Hyperlink.BORROW] - assert 'http://localhost:6500/AL/works/URI/http://www.feedbooks.com/item/1946289/borrow' == borrow_link.resource.url + [borrow_link] = [ + l for l in essex_pool.identifier.links if l.rel == Hyperlink.BORROW + ] + assert ( + "http://localhost:6500/AL/works/URI/http://www.feedbooks.com/item/1946289/borrow" + == borrow_link.resource.url + ) + + +class TestLicense: + """Represents an ODL license.""" + + def __init__( + self, + identifier: Optional[str] = None, + total_checkouts: Optional[int] = None, + concurrent_checkouts: Optional[int] = None, + expires: Optional[datetime.datetime] = None, + ) -> None: + """Initialize a new instance of TestLicense class. + + :param identifier: License's identifier + :param total_checkouts: Total number of checkouts before a license expires + :param concurrent_checkouts: Number of concurrent checkouts allowed + :param expires: Date & time when a license expires + """ + self._identifier: str = identifier if identifier else str(uuid.uuid1()) + self._total_checkouts: Optional[int] = total_checkouts + self._concurrent_checkouts: Optional[int] = concurrent_checkouts + self._expires: Optional[datetime.datetime] = expires + + @property + def identifier(self) -> str: + """Return the license's identifier. + + :return: License's identifier + """ + return self._identifier + + @property + def total_checkouts(self) -> Optional[int]: + """Return the total number of checkouts before a license expires. + + :return: Total number of checkouts before a license expires + """ + return self._total_checkouts + + @property + def concurrent_checkouts(self) -> Optional[int]: + """Return the number of concurrent checkouts allowed. + + :return: Number of concurrent checkouts allowed + """ + return self._concurrent_checkouts + + @property + def expires(self) -> Optional[datetime.datetime]: + """Return the date & time when a license expires. + + :return: Date & time when a license expires + """ + return self._expires + + +class TestLicenseInfo: + """Represents information about the current state of a license stored in the License Info Document.""" + + def __init__( + self, remaining_checkouts: int, available_concurrent_checkouts: int + ) -> None: + """Initialize a new instance of TestLicenseInfo class. + + :param remaining_checkouts: Total number of checkouts left for a License + :param available_concurrent_checkouts: Number of concurrent checkouts currently available + """ + self._remaining_checkouts: int = remaining_checkouts + self._available_concurrent_checkouts: int = available_concurrent_checkouts + + @property + def remaining_checkouts(self) -> int: + """Return the total number of checkouts left for a License. + + :return: Total number of checkouts left for a License + """ + return self._remaining_checkouts + + @property + def available_concurrent_checkouts(self) -> int: + """Return the number of concurrent checkouts currently available. + + :return: Number of concurrent checkouts currently available + """ + return self._available_concurrent_checkouts + + def __str__(self) -> str: + """Return a JSON representation of a part of the License Info Document.""" + return json.dumps( + { + "checkouts": { + "left": self.remaining_checkouts, + "available": self.available_concurrent_checkouts, + } + } + ) + + +class TestODLExpiredItemsReaper(DatabaseTest, BaseODLTest): + """Base class for all ODL reaper tests.""" + + ODL_PROTOCOL = ODLAPI.NAME + ODL_TEMPLATE_DIR = os.path.join(BaseODLTest.base_path, "files", "odl") + ODL_TEMPLATE_FILENAME = "feed_template.xml.jinja" + ODL_REAPER_CLASS = ODLExpiredItemsReaper + + def _create_importer(self, collection, http_get): + """Create a new ODL importer with the specified parameters. + + :param collection: Collection object + :param http_get: Use this method to make an HTTP GET request. + This can be replaced with a stub method for testing purposes. + + :return: ODLImporter object + """ + importer = ODLImporter( + self._db, + collection=collection, + http_get=http_get, + ) + + return importer + + def _get_test_feed(self, licenses: List[TestLicense]) -> str: + """Get the test ODL feed with specific licensing information. + + :param licenses: List of ODL licenses + :return: Test ODL feed + """ + env = Environment( + loader=FileSystemLoader(self.ODL_TEMPLATE_DIR), autoescape=select_autoescape() + ) + template = env.get_template(self.ODL_TEMPLATE_FILENAME) + feed = template.render(licenses=licenses) + + return feed + + def _import_test_feed( + self, + licenses: List[TestLicense], + license_infos: Optional[List[Optional[TestLicenseInfo]]] = None, + ) -> Tuple[List[Edition], List[LicensePool], List[Work]]: + """Import the test ODL feed with specific licensing information. + + :param licenses: List of ODL licenses + :param license_infos: List of License Info Documents + + :return: 3-tuple containing imported editions, license pools and works + """ + feed = self._get_test_feed(licenses) + data_source = DataSource.lookup(self._db, "Feedbooks", autocreate=True) + collection = MockODLAPI.mock_collection(self._db, protocol=self.ODL_PROTOCOL) + collection.external_integration.set_setting( + Collection.DATA_SOURCE_NAME_SETTING, data_source.name + ) + license_status_response = MagicMock( + side_effect=[ + (200, {}, str(license_status) if license_status else "{}") for license_status in license_infos + ] + if license_infos + else [(200, {}, {})] + ) + importer = self._create_importer(collection, license_status_response) + + ( + imported_editions, + imported_pools, + imported_works, + _, + ) = importer.import_from_feed(feed) + + return imported_editions, imported_pools, imported_works + + +class TestODLExpiredItemsReaperSingleLicense(TestODLExpiredItemsReaper): + """Class testing that the ODL 1.x reaper correctly processes publications with a single license.""" + + @parameterized.expand([ + ( + "expiration_date_in_the_past", + # The license expires 2021-01-01T00:01:00+01:00 that equals to 2010-01-01T00:00:00+00:00, the current time. + # It means the license had already expired at the time of the import. + TestLicense(expires=dateutil.parser.isoparse("2021-01-01T00:01:00+01:00")) + ), + ( + "total_checkouts_is_zero", + TestLicense(total_checkouts=0) + ), + ( + "remaining_checkouts_is_zero", + TestLicense(total_checkouts=10, concurrent_checkouts=5), + TestLicenseInfo(remaining_checkouts=0, available_concurrent_checkouts=0) + ) + ]) + @freeze_time("2021-01-01T00:00:00+00:00") + def test_odl_importer_skips_expired_licenses( + self, + _, + test_license: TestLicense, + test_license_info: Optional[TestLicenseInfo] = None + ) -> None: + """Ensure ODLImporter skips expired licenses + and does not count them in the total number of available licenses. + + :param test_license: An example of an expired ODL license + :param test_license_info: An example of an ODL License Info Document belonging to an expired ODL license + (if required) + """ + # 1.1. Import the test feed with an expired ODL license. + imported_editions, imported_pools, imported_works = self._import_test_feed( + [test_license], + [test_license_info] + ) + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 1.2. Ensure that the license pool was successfully created but it does not have any available licenses. + assert len(imported_pools) == 1 + + [imported_pool] = imported_pools + assert imported_pool.licenses_owned == 0 + assert imported_pool.licenses_available == 0 + assert len(imported_pool.licenses) == 0 + + @freeze_time("2021-01-01T00:00:00+00:00") + def test_odl_reaper_removes_expired_licenses(self): + """Ensure ODLExpiredItemsReaper removes expired licenses.""" + patron = self._patron() + + # 1.1. Import the test feed with an ODL license that is still valid. + # The license will be valid for one more day since this very moment. + # The feed declares that there 10 checkouts available in total + # but the License Info Document shows that there are only 9 available at the moment of import. + total_checkouts = 10 + available_concurrent_checkouts = 5 + remaining_checkouts = 9 + license_expiration_date = datetime_helpers.utc_now() + datetime.timedelta( + days=1 + ) + imported_editions, imported_pools, imported_works = self._import_test_feed( + [ + TestLicense( + expires=license_expiration_date, + total_checkouts=total_checkouts, + concurrent_checkouts=available_concurrent_checkouts, + ) + ], + [ + TestLicenseInfo( + remaining_checkouts=remaining_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts, + ) + ], + ) + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 1.2. Ensure that there is a license pool with available license. + assert len(imported_pools) == 1 + + [imported_pool] = imported_pools + assert imported_pool.licenses_owned == remaining_checkouts + assert imported_pool.licenses_available == available_concurrent_checkouts + + assert len(imported_pool.licenses) == 1 + [license] = imported_pool.licenses + assert license.expires == license_expiration_date + + # 2. Create a loan to ensure that the licence with active loan can also be removed (hidden). + loan, _ = license.loan_to(patron) + + # 3.1. Run ODLExpiredItemsReaper. This time nothing should happen since the license is still valid. + script = RunCollectionMonitorScript( + self.ODL_REAPER_CLASS, _db=self._db, cmd_args=["Test ODL Collection"] + ) + script.run() + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 3.2. Ensure that availability of the license pool didn't change. + assert imported_pool.licenses_owned == remaining_checkouts + assert imported_pool.licenses_available == available_concurrent_checkouts + + # 4. Expire the license. + with patch("core.model.License.is_expired", new_callable=PropertyMock) as is_expired: + is_expired.return_value = True + + # 5.1. Run ODLExpiredItemsReaper again. This time it should remove the expired license. + script.run() + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 5.2. Ensure that availability of the license pool was updated + # and now it doesn't have any available licenses. + assert imported_pool.licenses_owned == 0 + assert imported_pool.licenses_available == 0 + + # 6.1. Run ODLExpiredItemsReaper again to ensure that number of licenses won't become negative. + script.run() + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 6.2. Ensure that number of licenses is still 0. + assert imported_pool.licenses_owned == 0 + assert imported_pool.licenses_available == 0 + + +class TestODLExpiredItemsReaperMultipleLicense(TestODLExpiredItemsReaper): + """Class testing that the ODL 1.x reaper correctly processes publications with multiple licenses.""" + + @freeze_time("2021-01-01T00:00:00+00:00") + def test_odl_importer_skips_expired_licenses(self): + """Ensure ODLImporter skips expired licenses + and does not count them in the total number of available licenses.""" + # 1.1. Import the test feed with three expired ODL licenses and two valid licenses. + remaining_checkouts = 9 + available_concurrent_checkouts = 5 + imported_editions, imported_pools, imported_works = self._import_test_feed( + [ + TestLicense( # Expired + total_checkouts=10, # (expiry date in the past) + concurrent_checkouts=5, + expires=datetime_helpers.utc_now() - datetime.timedelta(days=1), + ), + TestLicense( # Expired + total_checkouts=0, # (total_checkouts is 0) + concurrent_checkouts=0, + expires=datetime_helpers.utc_now() + datetime.timedelta(days=1), + ), + TestLicense( # Expired + total_checkouts=10, # (remaining_checkout is 0) + concurrent_checkouts=5, + expires=datetime_helpers.utc_now() + datetime.timedelta(days=1), + ), + TestLicense( # Valid + total_checkouts=10, + concurrent_checkouts=5, + expires=datetime_helpers.utc_now() + datetime.timedelta(days=2), + ), + TestLicense( # Valid + total_checkouts=10, + concurrent_checkouts=5, + expires=datetime_helpers.utc_now() + datetime.timedelta(weeks=12), + ), + ], + [ + TestLicenseInfo( + remaining_checkouts=0, + available_concurrent_checkouts=0 + ), + TestLicenseInfo( + remaining_checkouts=remaining_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts + ), + TestLicenseInfo( + remaining_checkouts=remaining_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts + ) + ], + ) + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 1.2. Ensure that the license pool was successfully created + assert len(imported_pools) == 1 + [imported_pool] = imported_pools + + # 1.3. Ensure that the two valid licenses were imported. + assert len(imported_pool.licenses) == 2 + + # 1.4 Make sure that the license statistics is correct and include only checkouts owned by two valid licenses. + assert imported_pool.licenses_owned == remaining_checkouts * 2 + assert imported_pool.licenses_available == available_concurrent_checkouts * 2 + + @freeze_time("2021-01-01T00:00:00+00:00") + def test_odl_reaper_removes_expired_licenses(self): + """Ensure ODLExpiredItemsReaper removes expired licenses.""" + # 1.1. Import the test feed with ODL licenses that are not expired. + total_checkouts = 10 + remaining_checkouts = 9 + available_concurrent_checkouts = 5 + imported_editions, imported_pools, imported_works = self._import_test_feed( + [ + TestLicense( + total_checkouts=total_checkouts, + concurrent_checkouts=available_concurrent_checkouts, + expires=datetime_helpers.utc_now() + datetime.timedelta(days=1), + ), + TestLicense( + total_checkouts=total_checkouts, + concurrent_checkouts=available_concurrent_checkouts, + expires=datetime_helpers.utc_now() + datetime.timedelta(days=2), + ), + TestLicense( + total_checkouts=total_checkouts, + concurrent_checkouts=available_concurrent_checkouts, + expires=datetime_helpers.utc_now() + datetime.timedelta(weeks=12), + ), + ], + [ + TestLicenseInfo( + remaining_checkouts=total_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts + ), + TestLicenseInfo( + remaining_checkouts=remaining_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts + ), + TestLicenseInfo( + remaining_checkouts=remaining_checkouts, + available_concurrent_checkouts=available_concurrent_checkouts + ) + ], + ) + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 1.2. Ensure that there is a license pool with available license. + assert len(imported_pools) == 1 + + [imported_pool] = imported_pools + assert len(imported_pool.licenses) == 3 + + [license1, license2, license3] = imported_pool.licenses + assert license1.remaining_checkouts == total_checkouts + assert license1.concurrent_checkouts == available_concurrent_checkouts + + assert license2.remaining_checkouts == remaining_checkouts + assert license2.concurrent_checkouts == available_concurrent_checkouts + + assert license3.remaining_checkouts == remaining_checkouts + assert license3.concurrent_checkouts == available_concurrent_checkouts + + assert imported_pool.licenses_owned == total_checkouts + 2 * remaining_checkouts + assert imported_pool.licenses_available == 3 * available_concurrent_checkouts + + # 2.1. Run ODLExpiredItemsReaper. This time nothing should happen since the license is still valid. + script = RunCollectionMonitorScript( + self.ODL_REAPER_CLASS, _db=self._db, cmd_args=["Test ODL Collection"] + ) + script.run() + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 2.2. Ensure that availability of the license pool didn't change. + assert len(imported_pool.licenses) == 3 + assert imported_pool.licenses_owned == total_checkouts + 2 * remaining_checkouts + assert imported_pool.licenses_available == 3 * available_concurrent_checkouts + + # 3. Expire the license. + license1.expires = datetime_helpers.utc_now() - datetime.timedelta(days=1) + + # 3.1. Run ODLExpiredItemsReaper again. This time it should remove the expired license. + script.run() + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + + # 3.2. Ensure that availability of the license pool was updated. + assert len(imported_pool.licenses) == 3 + assert imported_pool.licenses_owned == 2 * remaining_checkouts + assert imported_pool.licenses_available == 2 * available_concurrent_checkouts + + # 4.1. Run ODLExpiredItemsReaper again to make sure that licenses are not expired twice. + script.run() + + # Commit to expire the SQLAlchemy cache. + self._db.commit() + # 4.2. Ensure that number of licenses remains the same as in step 3.2. + assert len(imported_pool.licenses) == 3 + assert imported_pool.licenses_owned == 2 * remaining_checkouts + assert imported_pool.licenses_available == 2 * available_concurrent_checkouts