ODL with multiple license blocks #39
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,9 @@ | ||
| import datetime | ||
| import json | ||
| import logging | ||
| import uuid | ||
| from io import StringIO | ||
| from typing import Callable, Optional | ||
|
|
||
| import dateutil | ||
| import feedparser | ||
|
|
@@ -14,12 +16,7 @@ | |
|
|
||
| from core import util | ||
| from core.analytics import Analytics | ||
| from core.metadata_layer import CirculationData, FormatData, LicenseData, TimestampData | ||
| from core.model import ( | ||
| Collection, | ||
| ConfigurationSetting, | ||
|
|
@@ -32,38 +29,20 @@ | |
| LicensePool, | ||
| Loan, | ||
| MediaTypes, | ||
| Representation, | ||
| RightsStatus, | ||
| Session, | ||
| get_one, | ||
| get_one_or_create, | ||
| ) | ||
| from core.monitor import CollectionMonitor, IdentifierSweepMonitor | ||
| from core.opds_import import OPDSImporter, OPDSImportMonitor, OPDSXMLParser | ||
| from core.testing import DatabaseTest, MockRequestsResponse | ||
| from core.util.datetime_helpers import utc_now | ||
| from core.util.http import HTTP, BadResponseException, RemoteIntegrationException | ||
| from core.util.string_helpers import base64 | ||
|
|
||
| from .circulation import BaseCirculationAPI, FulfillmentInfo, HoldInfo, LoanInfo | ||
| from .circulation_exceptions import * | ||
| from .shared_collection import BaseSharedCollectionAPI | ||
|
|
||
|
|
@@ -786,6 +765,105 @@ class ODLImporter(OPDSImporter): | |
| # about the license. | ||
| LICENSE_INFO_DOCUMENT_MEDIA_TYPE = 'application/vnd.odl.info+json' | ||
|
|
||
| @classmethod | ||
| def parse_license( | ||
| cls, | ||
| identifier: str, | ||
| total_checkouts: Optional[int], | ||
| concurrent_checkouts: Optional[int], | ||
| expires: Optional[datetime.datetime], | ||
| checkout_link: Optional[str], | ||
| odl_status_link: Optional[str], | ||
| do_get: Callable | ||
| ) -> Optional[LicenseData]: | ||
| """Check the license's attributes passed as parameters: | ||
| - if they're correct, turn them into a LicenseData object | ||
| - otherwise, return a None | ||
|
|
||
| :param identifier: License's identifier | ||
| :param total_checkouts: Total number of checkouts before the license expires | ||
| :param concurrent_checkouts: Number of concurrent checkouts allowed for this license | ||
| :param expires: Date & time until the license is valid | ||
| :param checkout_link: License's checkout link | ||
| :param odl_status_link: License Info Document's link | ||
| :param do_get: Callback performing HTTP GET method | ||
|
|
||
| :return: LicenseData if all the license's attributes are correct, None, otherwise | ||
| """ | ||
| remaining_checkouts = None | ||
| available_concurrent_checkouts = None | ||
|
|
||
| # This cycle ends in two different cases: | ||
| # - when at least one of the parameters is invalid; in this case, the method returns None. | ||
| # - when all the parameters are valid; in this case, the method returns a LicenseData object. | ||
| while True: | ||
| if total_checkouts is not None: | ||
| total_checkouts = int(total_checkouts) | ||
|
|
||
| if total_checkouts <= 0: | ||
| logging.info( | ||
| f"License # {identifier} expired since " | ||
| f"the total number of checkouts is {total_checkouts}" | ||
| ) | ||
| break | ||
|
|
||
| if expires: | ||
| if not isinstance(expires, datetime.datetime): | ||
| expires = dateutil.parser.parse(expires) | ||
|
|
||
| expires = util.datetime_helpers.to_utc(expires) | ||
| now = util.datetime_helpers.utc_now() | ||
|
|
||
| if expires <= now: | ||
| logging.info( | ||
| f"License # {identifier} expired at {expires} (now is {now})" | ||
| ) | ||
| break | ||
|
|
||
| if odl_status_link: | ||
| status_code, _, response = do_get( | ||
| odl_status_link, headers={} | ||
| ) | ||
|
|
||
| if status_code in (200, 201): | ||
| status = json.loads(response) | ||
| checkouts = status.get("checkouts", {}) | ||
| remaining_checkouts = checkouts.get("left") | ||
| available_concurrent_checkouts = checkouts.get("available") | ||
| else: | ||
| logging.warning( | ||
| f"License # {identifier}'s Info Document is not available. " | ||
| f"Status link failed with {status_code} code" | ||
| ) | ||
| break | ||
|
|
||
| if remaining_checkouts is None: | ||
|
Contributor
There was a problem hiding this comment. I think that if
Contributor
There was a problem hiding this comment. If
Contributor
Contributor
There was a problem hiding this comment. I put this question to De Marque. Hopefully will have a response soon.
Contributor
There was a problem hiding this comment. De Marque indicated that you should not expect to see Have you seen any cases where
Contributor
There was a problem hiding this comment. @tdilauro, yes, there are cases like that in the DPLA Exchange feed. I added a new cell to the Jupyter notebook where all the licenses like that are shown. |
||
| remaining_checkouts = total_checkouts | ||
|
|
||
| if remaining_checkouts is not None: | ||
| remaining_checkouts = int(remaining_checkouts) | ||
|
|
||
| if remaining_checkouts <= 0: | ||
| logging.info( | ||
| f"License # {identifier} expired since " | ||
| f"the remaining number of checkouts is {remaining_checkouts}" | ||
| ) | ||
| break | ||
|
|
||
| if available_concurrent_checkouts is None: | ||
| available_concurrent_checkouts = concurrent_checkouts | ||
|
|
||
| return LicenseData( | ||
| identifier=identifier, | ||
| checkout_url=checkout_link, | ||
| status_url=odl_status_link, | ||
| expires=expires, | ||
| remaining_checkouts=remaining_checkouts, | ||
| concurrent_checkouts=available_concurrent_checkouts, | ||
| ) | ||
|
|
||
| return None | ||
|
|
||
| @classmethod | ||
| def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get=None): | ||
| do_get = do_get or Representation.cautious_http_get | ||
|
|
@@ -843,11 +921,6 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get= | |
|
|
||
| data['medium'] = medium | ||
|
|
||
| checkout_link = None | ||
| for link_tag in parser._xpath(odl_license_tag, 'odl:tlink') or []: | ||
| rel = link_tag.attrib.get("rel") | ||
|
|
@@ -866,37 +939,33 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get= | |
| odl_status_link = attrib.get("href") | ||
| break | ||
|
|
||
| expires = None | ||
| total_checkouts = None | ||
| concurrent_checkouts = None | ||
|
|
||
| terms = parser._xpath(odl_license_tag, "odl:terms") | ||
| if terms: | ||
| total_checkouts = subtag(terms[0], "odl:total_checkouts") | ||
| concurrent_checkouts = subtag(terms[0], "odl:concurrent_checkouts") | ||
| expires = subtag(terms[0], "odl:expires") | ||
|
|
||
| license = cls.parse_license( | ||
| identifier, | ||
| total_checkouts, | ||
| concurrent_checkouts, | ||
| expires, | ||
| checkout_link, | ||
| odl_status_link, | ||
| do_get | ||
| ) | ||
|
|
||
| if not license: | ||
| continue | ||
|
|
||
| licenses_owned += int(license.remaining_checkouts or 0) | ||
| licenses_available += int(license.concurrent_checkouts or 0) | ||
|
|
||
| licenses.append(license) | ||
|
|
||
| if not data.get('circulation'): | ||
| data['circulation'] = dict() | ||
|
|
@@ -1552,17 +1621,20 @@ def __init__(self, _db, collection): | |
|
|
||
| def process_item(self, identifier): | ||
| for licensepool in identifier.licensed_through: | ||
| remaining_checkouts = 0 # total number of checkouts across all the licenses in the pool | ||
| concurrent_checkouts = 0 # number of concurrent checkouts allowed across all the licenses in the pool | ||
|
|
||
| # 0 is a starting point, | ||
| # we're going through all the valid licenses in the pool and count up available checkouts. | ||
| for license_pool_license in licensepool.licenses: | ||
| if not license_pool_license.is_expired: | ||
| remaining_checkouts += license_pool_license.remaining_checkouts | ||
| concurrent_checkouts += license_pool_license.concurrent_checkouts | ||
|
|
||
| if remaining_checkouts != licensepool.licenses_owned or \ | ||
| concurrent_checkouts != licensepool.licenses_available: | ||
| licenses_owned = max(remaining_checkouts, 0) | ||
| licenses_available = max(concurrent_checkouts, 0) | ||
|
|
||
| circulation_data = CirculationData( | ||
| data_source=licensepool.data_source, | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.