Merge main into production Nov24

.github/pull_request_template.md

@@ -1,35 +1,29 @@
 <!--
-Thanks for contributing!
-
-Please ensure the name of your PR is written in imperative present tense. For example:
-
-- "fix color contrast on submit buttons"
-- "add 'favourite food' value to Author model"
+Thanks for contributing! This template has some checkboxes that help keep track of what changes go into a release.
 
 To check (tick) a list item, replace the space between square brackets with an x, like this:
 
 - [x] I have checked the box
 
 You can find more information and tips for BookWyrm contributors at https://docs.joinbookwyrm.com/contributing.html
 -->
-
-## Are you finished?
-
-### Linters
+## Description
 <!--
-Please run linters on your code before submitting your PR.
-If you miss this step it is likely that the GitHub task runners will fail.
+Describe what your pull request does here
 -->
 
-- [ ] I have checked my code with `black`, `pylint`, and `mypy`, or `./bw-dev formatters`
 
-### Tests
-<!-- Check one -->
+<!--
+For pull requests that relate or close an issue, please include them
+below.  We like to follow [Github's guidance on linking issues to pull requests](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue).
 
-- [ ] My changes do not need new tests
-- [ ] All tests I have added are passing
-- [ ] I have written tests but need help to make them pass
-- [ ] I have not written tests and need help to write them
+For example having the text: "closes #1234" would connect the current pull
+request to issue 1234.  And when we merge the pull request, Github will
+automatically close the issue.
+-->
+
+- Related Issue #
+- Closes #
 
 ## What type of Pull Request is this?
 <!-- Check all that apply -->
@@ -48,21 +42,6 @@ If you miss this step it is likely that the GitHub task runners will fail.
 
 ### Details of breaking or configuration changes (if any of above checked)
 
-## Description
-
-<!--
-Describe what your pull request does here.
-
-For pull requests that relate or close an issue, please include them
-below.  We like to follow [Github's guidance on linking issues to pull requests](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue).
-
-For example having the text: "closes #1234" would connect the current pull
-request to issue 1234.  And when we merge the pull request, Github will
-automatically close the issue.
--->
-
-- Related Issue #
-- Closes #
 
 ## Documentation
 <!--
@@ -76,3 +55,14 @@ Our documentation is maintained in a separate repository at https://github.com/b
 - [ ] I have created a matching pull request in the Documentation repository
 - [ ] I intend to create a matching pull request in the Documentation repository after this PR is merged
 
+<!-- Amazing! Thanks for filling that out. Your PR will need to have passing tests and happy linters before we can merge
+You will need to check your code with `black`, `pylint`, and `mypy`, or `./bw-dev formatters`
+-->
+
+### Tests
+<!-- Check one -->
+
+- [ ] My changes do not need new tests
+- [ ] All tests I have added are passing
+- [ ] I have written tests but need help to make them pass
+- [ ] I have not written tests and need help to write them

.github/workflows/python.yml

@@ -48,7 +48,7 @@ jobs:
     - name: Set up .env
       run: cp .env.example .env
     - name: Check migrations up-to-date
-      run: python ./manage.py makemigrations --check
+      run: python ./manage.py makemigrations --check -v 3
     - name: Run Tests
       run: pytest -n 3
 

.pylintrc

@@ -3,7 +3,19 @@ ignore=migrations
 load-plugins=pylint.extensions.no_self_use
 
 [MESSAGES CONTROL]
-disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801,C3001,import-error
+disable =
+    cyclic-import,
+    duplicate-code,
+    fixme,
+    no-member,
+    raise-missing-from,
+    too-few-public-methods,
+    too-many-ancestors,
+    too-many-instance-attributes,
+    unnecessary-lambda-assignment,
+    unsubscriptable-object,
+enable =
+    useless-suppression
 
 [FORMAT]
 max-line-length=88

bookwyrm/activitypub/base_activity.py

@@ -369,17 +369,13 @@ def resolve_remote_id(
 
     # load the data and create the object
     try:
-        data = get_data(remote_id)
+        data = get_activitypub_data(remote_id)
     except ConnectionError:
         logger.info("Could not connect to host for remote_id: %s", remote_id)
         return None
     except requests.HTTPError as e:
-        if (e.response is not None) and e.response.status_code == 401:
-            # This most likely means it's a mastodon with secure fetch enabled.
-            data = get_activitypub_data(remote_id)
-        else:
-            logger.info("Could not connect to host for remote_id: %s", remote_id)
-            return None
+        logger.exception("HTTP error - remote_id: %s - error: %s", remote_id, e)
+        return None
     # determine the model implicitly, if not provided
     # or if it's a model with subclasses like Status, check again
     if not model or hasattr(model.objects, "select_subclasses"):

bookwyrm/activitypub/book.py

@@ -67,7 +67,6 @@ class Edition(Book):
     type: str = "Edition"
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Work(Book):
     """work instance of a book object"""

bookwyrm/activitypub/ordered_collection.py

@@ -18,7 +18,6 @@ class OrderedCollection(ActivityObject):
     type: str = "OrderedCollection"
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class OrderedCollectionPrivate(OrderedCollection):
     """an ordered collection with privacy settings"""

bookwyrm/activitypub/verbs.py

@@ -22,7 +22,6 @@ def action(self, allow_external_connections=True):
             self.object.to_model(allow_external_connections=allow_external_connections)
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Create(Verb):
     """Create activity"""
@@ -33,7 +32,6 @@ class Create(Verb):
     type: str = "Create"
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Delete(Verb):
     """Create activity"""
@@ -63,7 +61,6 @@ def action(self, allow_external_connections=True):
         # if we can't find it, we don't need to delete it because we don't have it
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Update(Verb):
     """Update activity"""
@@ -227,7 +224,6 @@ def action(self, allow_external_connections=True):
         self.to_model(allow_external_connections=allow_external_connections)
 
 
-# pylint: disable=invalid-name
 @dataclass(init=False)
 class Announce(Verb):
     """boosting a status"""

bookwyrm/activitystreams.py

@@ -32,7 +32,7 @@ def unread_by_status_type_id(self, user_id):
         stream_id = self.stream_id(user_id)
         return f"{stream_id}-unread-by-type"
 
-    def get_rank(self, obj):  # pylint: disable=no-self-use
+    def get_rank(self, obj):
         """statuses are sorted by date published"""
         return obj.published_date.timestamp()
 

bookwyrm/apps.py

@@ -33,7 +33,6 @@ class BookwyrmConfig(AppConfig):
     name = "bookwyrm"
     verbose_name = "BookWyrm"
 
-    # pylint: disable=no-self-use
     def ready(self):
         """set up OTLP and preview image files, if desired"""
         if settings.OTEL_EXPORTER_OTLP_ENDPOINT or settings.OTEL_EXPORTER_CONSOLE:

bookwyrm/book_search.py

@@ -36,7 +36,6 @@ def search(
     ...
 
 
-# pylint: disable=arguments-differ
 def search(
     query: str,
     *,

bookwyrm/connectors/abstract_connector.py

@@ -86,7 +86,7 @@ async def get_results(
             ),
             "User-Agent": USER_AGENT,
         }
-        params = {"min_confidence": min_confidence}
+        params = {"min_confidence": str(min_confidence)}
         try:
             async with session.get(url, headers=headers, params=params) as response:
                 if not response.ok:

bookwyrm/context_processors.py

@@ -2,7 +2,7 @@
 from bookwyrm import models, settings
 
 
-def site_settings(request):  # pylint: disable=unused-argument
+def site_settings(request):
     """include the custom info about the site"""
     request_protocol = "https://"
     if not request.is_secure():

bookwyrm/forms/custom_form.py

@@ -15,9 +15,9 @@ def __init__(self, *args, **kwargs):
         css_classes["number"] = "input"
         css_classes["checkbox"] = "checkbox"
         css_classes["textarea"] = "textarea"
-        # pylint: disable=super-with-arguments
         super().__init__(*args, **kwargs)
         for visible in self.visible_fields():
+            input_type = ""
             if hasattr(visible.field.widget, "input_type"):
                 input_type = visible.field.widget.input_type
             if isinstance(visible.field.widget, Textarea):

bookwyrm/forms/edit_user.py

@@ -18,6 +18,7 @@ class Meta:
             "email",
             "summary",
             "show_goal",
+            "show_ratings",
             "show_suggested_users",
             "manually_approves_followers",
             "default_post_privacy",

bookwyrm/forms/landing.py

@@ -34,7 +34,6 @@ def infer_username(self):
 
     def add_invalid_password_error(self):
         """We don't want to be too specific about this"""
-        # pylint: disable=attribute-defined-outside-init
         self.non_field_errors = _("Username or password are incorrect")
 
 

bookwyrm/forms/widgets.py

@@ -5,8 +5,6 @@
 class ArrayWidget(forms.widgets.TextInput):
     """Inputs for postgres array fields"""
 
-    # pylint: disable=unused-argument
-    # pylint: disable=no-self-use
     def value_from_datadict(self, data, files, name):
         """get all values for this name"""
         return [i for i in data.getlist(name) if i]

bookwyrm/importers/__init__.py

@@ -1,7 +1,7 @@
 """ import classes """
 
 from .importer import Importer
-from .bookwyrm_import import BookwyrmImporter
+from .bookwyrm_import import BookwyrmImporter, BookwyrmBooksImporter
 from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter

bookwyrm/importers/bookwyrm_import.py

@@ -3,6 +3,7 @@
 
 from bookwyrm.models import User
 from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
+from . import Importer
 
 
 class BookwyrmImporter:
@@ -22,3 +23,17 @@ def process_import(
             user=user, archive_file=archive_file, required=required
         )
         return job
+
+
+class BookwyrmBooksImporter(Importer):
+    """
+    Handle reading a csv from BookWyrm.
+    Goodreads is the default importer, we basically just use the same structure
+    But BookWyrm has additional attributes in the csv
+    """
+
+    service = "BookWyrm"
+    row_mappings_guesses = Importer.row_mappings_guesses + [
+        ("shelf_name", ["shelf_name"]),
+        ("review_published", ["review_published"]),
+    ]

bookwyrm/importers/importer.py

@@ -18,27 +18,41 @@ class Importer:
     row_mappings_guesses = [
         ("id", ["id", "book id"]),
         ("title", ["title"]),
-        ("authors", ["author", "authors", "primary author"]),
-        ("isbn_10", ["isbn10", "isbn", "isbn/uid"]),
-        ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]),
+        ("authors", ["author_text", "author", "authors", "primary author"]),
+        ("isbn_10", ["isbn_10", "isbn10", "isbn", "isbn/uid"]),
+        ("isbn_13", ["isbn_13", "isbn13", "isbn", "isbns", "isbn/uid"]),
         ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]),
-        ("review_name", ["review name"]),
-        ("review_body", ["my review", "review"]),
+        ("review_name", ["review_name", "review name"]),
+        ("review_body", ["review_content", "my review", "review"]),
         ("rating", ["my rating", "rating", "star rating"]),
-        ("date_added", ["date added", "entry date", "added"]),
-        ("date_started", ["date started", "started"]),
-        ("date_finished", ["date finished", "last date read", "date read", "finished"]),
+        (
+            "date_added",
+            ["shelf_date", "date_added", "date added", "entry date", "added"],
+        ),
+        ("date_started", ["start_date", "date started", "started"]),
+        (
+            "date_finished",
+            ["finish_date", "date finished", "last date read", "date read", "finished"],
+        ),
     ]
+
+    # TODO: stopped
+
     date_fields = ["date_added", "date_started", "date_finished"]
     shelf_mapping_guesses = {
         "to-read": ["to-read", "want to read"],
         "read": ["read", "already read"],
         "reading": ["currently-reading", "reading", "currently reading"],
     }
 
-    # pylint: disable=too-many-locals
+    # pylint: disable=too-many-arguments
     def create_job(
-        self, user: User, csv_file: Iterable[str], include_reviews: bool, privacy: str
+        self,
+        user: User,
+        csv_file: Iterable[str],
+        include_reviews: bool,
+        privacy: str,
+        create_shelves: bool = True,
     ) -> ImportJob:
         """check over a csv and creates a database entry for the job"""
         csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
@@ -55,6 +69,7 @@ def create_job(
         job = ImportJob.objects.create(
             user=user,
             include_reviews=include_reviews,
+            create_shelves=create_shelves,
             privacy=privacy,
             mappings=mappings,
             source=self.service,
@@ -114,7 +129,7 @@ def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         shelf = [
             s for (s, gs) in self.shelf_mapping_guesses.items() if shelf_name in gs
         ]
-        return shelf[0] if shelf else None
+        return shelf[0] if shelf else normalized_row.get("shelf") or None
 
     # pylint: disable=no-self-use
     def normalize_row(
@@ -149,6 +164,7 @@ def create_retry_job(
         job = ImportJob.objects.create(
             user=user,
             include_reviews=original_job.include_reviews,
+            create_shelves=original_job.create_shelves,
             privacy=original_job.privacy,
             source=original_job.source,
             # TODO: allow users to adjust mappings

bookwyrm/importers/librarything_import.py

@@ -20,7 +20,7 @@ class LibrarythingImporter(Importer):
 
     def normalize_row(
         self, entry: dict[str, str], mappings: dict[str, Optional[str]]
-    ) -> dict[str, Optional[str]]:  # pylint: disable=no-self-use
+    ) -> dict[str, Optional[str]]:
         """use the dataclass to create the formatted row of data"""
         normalized = {
             k: _remove_brackets(entry.get(v) if v else None)

bookwyrm/lists_stream.py

@@ -18,7 +18,7 @@ def stream_id(self, user):  # pylint: disable=no-self-use
             return f"{user}-lists"
         return f"{user.id}-lists"
 
-    def get_rank(self, obj):  # pylint: disable=no-self-use
+    def get_rank(self, obj):
         """lists are sorted by updated date"""
         return obj.updated_date.timestamp()