diff --git a/.env.example b/.env.example index c61ceba1e8..d2ae90f474 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,24 @@ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr" +# SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr" # SECURITY WARNING: don't run with debug turned on in production! DEBUG=false -USE_HTTPS=true - +# Your domain ("example.com"). If running development on localhost, set this to localhost DOMAIN=your.domain.here +# This is the email address registered by certbot when you set up https in production EMAIL=your@email.here +# If you use letsencrypt and get https directly to nginx, use https mode, + +# if you have something in front of nginx, like traefik/ngrok/apache that handles certificates and +# you want to pass just http to nginx, use reverse_proxy mode. +# For local development, use 'reverse_proxy' config to allow testing/development without an ssl certificate. +# +# If NGINX_SETUP is not defined, we default to https mode +# +# docker compose NGINX setup: https, reverse_proxy +NGINX_SETUP=https + # Instance default language (see options at bookwyrm/settings.py "LANGUAGES" LANGUAGE_CODE="en-us" # Used for deciding which editions to prefer @@ -16,16 +27,17 @@ DEFAULT_LANGUAGE="English" ## Leave unset to allow all hosts # ALLOWED_HOSTS="localhost,127.0.0.1,[::1]" -# Specify when the site is served from a port that is not the default -# for the protocol (80 for HTTP or 443 for HTTPS). -# Probably only necessary in development. +# Specifying PORT is only necessary in reverse_proxy mode. +# By default PORT is 443 unless you are using localhost for development work. +# If developing with localhost you do not need to set a port and it defaults to 80. # PORT=1333 +STATIC_ROOT=static/ MEDIA_ROOT=images/ # Database configuration PGPORT=5432 -POSTGRES_PASSWORD=securedbypassword123 +# POSTGRES_PASSWORD=securedbypassword123 POSTGRES_USER=bookwyrm POSTGRES_DB=bookwyrm POSTGRES_HOST=db @@ -34,7 +46,7 @@ POSTGRES_HOST=db MAX_STREAM_LENGTH=200 REDIS_ACTIVITY_HOST=redis_activity REDIS_ACTIVITY_PORT=6379 -REDIS_ACTIVITY_PASSWORD=redispassword345 +# REDIS_ACTIVITY_PASSWORD=redispassword345 # Optional, use a different redis database (defaults to 0) # REDIS_ACTIVITY_DB_INDEX=0 # Alternatively specify the full redis url, i.e. if you need to use a unix:// socket @@ -43,7 +55,7 @@ REDIS_ACTIVITY_PASSWORD=redispassword345 # Redis as celery broker REDIS_BROKER_HOST=redis_broker REDIS_BROKER_PORT=6379 -REDIS_BROKER_PASSWORD=redispassword123 +# REDIS_BROKER_PASSWORD=redispassword123 # Optional, use a different redis database (defaults to 0) # REDIS_BROKER_DB_INDEX=0 # Alternatively specify the full redis url, i.e. if you need to use a unix:// socket @@ -52,7 +64,7 @@ REDIS_BROKER_PASSWORD=redispassword123 # Monitoring for celery FLOWER_PORT=8888 FLOWER_USER=admin -FLOWER_PASSWORD=changeme +# FLOWER_PASSWORD=changeme # Email config EMAIL_HOST=smtp.mailgun.org @@ -92,6 +104,9 @@ S3_SIGNED_URL_EXPIRY=900 # AWS_S3_URL_PROTOCOL=None # "http:" # AWS_S3_REGION_NAME=None # "fr-par" # AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud" +# AWS_DEFAULT_ACL="" # AWS_DEFAULT_ACL defaults to public-read, however, not all providers, e.g. BackBlaze, support it. + # Uncomment and set this value to an empty string (for Back Blaze) or whatever is required by your provider to override it. + # Commented are example values if you use Azure Blob Storage # USE_AZURE=true @@ -131,13 +146,6 @@ OTEL_EXPORTER_OTLP_HEADERS= # Service name to identify your app OTEL_SERVICE_NAME= -# Set HTTP_X_FORWARDED_PROTO ONLY to true if you know what you are doing. -# Only use it if your proxy is "swallowing" if the original request was made -# via https. Please refer to the Django-Documentation and assess the risks -# for your instance: -# https://docs.djangoproject.com/en/3.2/ref/settings/#secure-proxy-ssl-header -HTTP_X_FORWARDED_PROTO=false - # TOTP settings # TWO_FACTOR_LOGIN_VALIDITY_WINDOW sets the number of codes either side # which will be accepted. @@ -150,7 +158,7 @@ TWO_FACTOR_LOGIN_MAX_SECONDS=60 CSP_ADDITIONAL_HOSTS= # Time before being logged out (in seconds) -# SESSION_COOKIE_AGE=2592000 # current default: 30 days +# SESSION_COOKIE_AGE=31536000 # current default: 365 days # Maximum allowed memory for file uploads (increase if users are having trouble # uploading BookWyrm export files). diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 99c92478db..570174248a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,10 +1,5 @@ - -## Are you finished? - -### Linters +## Description -- [ ] I have checked my code with `black`, `pylint`, and `mypy`, or `./bw-dev formatters` -### Tests - + + +- Related Issue # +- Closes # ## What type of Pull Request is this? @@ -48,21 +42,6 @@ If you miss this step it is likely that the GitHub task runners will fail. ### Details of breaking or configuration changes (if any of above checked) -## Description - - - -- Related Issue # -- Closes # ## Documentation + +### Tests + + +- [ ] My changes do not need new tests +- [ ] All tests I have added are passing +- [ ] I have written tests but need help to make them pass +- [ ] I have not written tests and need help to write them diff --git a/.github/workflows/lint-frontend.yaml b/.github/workflows/lint-frontend.yaml index b0322f371e..68142b9467 100644 --- a/.github/workflows/lint-frontend.yaml +++ b/.github/workflows/lint-frontend.yaml @@ -15,7 +15,7 @@ on: jobs: lint: name: Lint with stylelint and ESLint. - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. diff --git a/.github/workflows/prettier.yaml b/.github/workflows/prettier.yaml index 9c05c7476b..df56cafb00 100644 --- a/.github/workflows/prettier.yaml +++ b/.github/workflows/prettier.yaml @@ -10,7 +10,7 @@ on: jobs: lint: name: Lint with Prettier - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 01241b467d..baa4f3a22b 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: postgres:13 + image: postgres:17 env: # does not inherit from jobs.build.env POSTGRES_USER: postgres POSTGRES_PASSWORD: hunter2 @@ -48,7 +48,7 @@ jobs: - name: Set up .env run: cp .env.example .env - name: Check migrations up-to-date - run: python ./manage.py makemigrations --check + run: python ./manage.py makemigrations --check -v 3 - name: Run Tests run: pytest -n 3 diff --git a/.gitignore b/.gitignore index fd6cc7547c..e6ba03f7cb 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,8 @@ nginx/default.conf #macOS **/.DS_Store +#QTS (system in QNAP NAS) +**/.@__thumb/ + # Docker docker-compose.override.yml diff --git a/.pylintrc b/.pylintrc index 464638853e..e89f7d5363 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,7 +3,19 @@ ignore=migrations load-plugins=pylint.extensions.no_self_use [MESSAGES CONTROL] -disable=E1101,E1135,E1136,R0903,R0901,R0902,W0707,W0511,W0406,R0401,R0801,C3001,import-error +disable = + cyclic-import, + duplicate-code, + fixme, + no-member, + raise-missing-from, + too-few-public-methods, + too-many-ancestors, + too-many-instance-attributes, + unnecessary-lambda-assignment, + unsubscriptable-object, +enable = + useless-suppression [FORMAT] max-line-length=88 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..21164970fb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,53 @@ +# Contributing to BookWyrm + +Our goal is to make BookWyrm a kind and welcoming place where everyone can contribute to the success of the project. Here are some ways you can join the project: + +## Report things that are confusing + +We want BookWyrm to be a fun experience that is intuitive to understand. If you're confused by something, it's probably because it is confusing! We are always keen to improve our [documentation](https://docs.joinbookwyrm.com) and Guided Tour as well as the platform itself. + +You can [create an issue to improve our documentation](https://github.com/bookwyrm-social/documentation/issues) or if you prefer, [ask for help in our Matrix chat room](https://app.element.io/#/room/#bookwyrm:matrix.org). + +## Report bugs + +Sometimes things don't work the way we intended. We would love to have fewer bugs, but we can only fix them if we know about them. + +You can [report bugs](https://github.com/bookwyrm-social/bookwyrm/issues) by clicking "New Issue". The more information you can provide, the easier it will be to understand the problem and squash that bug! + +It's a good idea to search the Issues for key words associated with your bug first because someone else may have already reported it. + +## Request and discuss new features + +Got a great idea for an improvement to BookWyrm? You can [request new features](https://github.com/bookwyrm-social/bookwyrm/issues) by clicking "New Issue". + +It's a good idea to search the Issues for key words associated with your feature suggestion first because someone else may have already requested it. + +## Translate BookWyrm into international languages + +Books are written in many languages, and BookWyrm should be too. If you know more than one language, you might be able to help us to [translate BookWyrm](https://translate.joinbookwyrm.com/). You can find out more about translation [in the documentation](https://docs.joinbookwyrm.com/translation.html). + +## Keep the documentation up to date + +Good documentation is crucial so that people know how to use, contribute to, and administer BookWyrm. No matter how you are involved with BookWyrm, your perspective is valuable and you can contribute to our documentation. + +We managed documentation in [a separate GitHub repository](https://github.com/bookwyrm-social/documentation) where you can [log a documentation issue](https://github.com/bookwyrm-social/documentation/issues) or contribute to the documentation yourself. + +## Test draft versions + +Are you a BookWyrm instance administrator? You can help to test new features when we release them in a draft version of BookWyrm, and report back on your experiences. This is crucial to helping us to release stable versions with fewer bugs. + +## Contribute code + +If you're able to write code, you can contribute that way! Check out the [Guide to the developer environment](https://docs.joinbookwyrm.com/install-dev.html) and our code [style guide](https://docs.joinbookwyrm.com/style_guide.html). + +## Provide expert advice + +Bibliographic metadata wizard? Celery nerd? ActivityPub expert? SQL query obsessive? We need all kinds of expertise! You can contribute to discussions in [the Issues](https://github.com/bookwyrm-social/bookwyrm/issues) or reach out to make suggestions [in our Matrix chat room](https://app.element.io/#/room/#bookwyrm:matrix.org) or via an Issue of your own. + +## More information + +You can find out more about BookWyrm and contributing at [JoinBookWyrm.com](https://joinbookwyrm.com/get-involved/). + +Ensure you are aware of and agree to our [Code of Conduct](https://github.com/bookwyrm-social/bookwyrm/blob/main/CODE_OF_CONDUCT.md). + +Please note that the BookWyrm project is licensed under the [Anti-capitalist Software License](https://github.com/bookwyrm-social/bookwyrm/blob/main/LICENSE.md). \ No newline at end of file diff --git a/FEDERATION.md b/FEDERATION.md index d80e98bd3c..5b82b958d2 100644 --- a/FEDERATION.md +++ b/FEDERATION.md @@ -321,6 +321,8 @@ Bookwyrm uses the [Webfinger](https://datatracker.ietf.org/doc/html/rfc7033) sta Bookwyrm uses and requires HTTP signatures for all `POST` requests. `GET` requests are not signed by default, but if Bookwyrm receives a `403` response to a `GET` it will re-send the request, signed by the default server user. This usually will have a user id of `https://example.net/user/bookwyrm.instance.actor` +As of the first version to be released in 2025, all `GET` requests will be signed by the instance user instead of re-sending requests that are rejected. + #### publicKey id In older versions of Bookwyrm the `publicKey.id` was incorrectly listed in request headers as `https://example.net/user/username#main-key`. As of v0.6.3 the id is now listed correctly, as `https://example.net/user/username/#main-key`. In most ActivityPub implementations this will make no difference as the URL will usually resolve to the same place. diff --git a/README.md b/README.md index 7e27d44e66..0322308821 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ BookWyrm is built on [ActivityPub](http://activitypub.rocks/). With ActivityPub, Federation makes it possible to have small, self-determining communities, in contrast to the monolithic service you find on GoodReads or Twitter. An instance can be focused on a particular interest, be just for a group of friends, or anything else that brings people together. Each community can choose which other instances they want to federate with, and moderate and run their community autonomously. Check out https://runyourown.social/ to get a sense of the philosophy and logistics behind small, high-trust social networks. +Developers of other ActivityPub software can find out more about BookWyrm's implementation at [`FEDERATION.md`](https://github.com/bookwyrm-social/bookwyrm/blob/main/FEDERATION.md). + ## Features ### Post about books @@ -61,3 +63,7 @@ Deployment ## Set up BookWyrm The [documentation website](https://docs.joinbookwyrm.com/) has instruction on how to set up BookWyrm in a [developer environment](https://docs.joinbookwyrm.com/install-dev.html) or [production](https://docs.joinbookwyrm.com/install-prod.html). + +## Contributing + +There are many ways you can contribute to the success and health of the BookWyrm project! You do not have to know how to write code and we are always keen to see more people get involved. Find out how you can join the project at [CONTRIBUTING.md](https://github.com/bookwyrm-social/bookwyrm/blob/main/CONTRIBUTING.md) \ No newline at end of file diff --git a/VERSION b/VERSION index 0a1ffad4b4..8adc70fdd9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.4 +0.8.0 \ No newline at end of file diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index dc4b8f6ae1..2a52d99f03 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -120,6 +120,7 @@ def to_model( save: bool = True, overwrite: bool = True, allow_external_connections: bool = True, + trigger=None, ) -> Optional[TBookWyrmModel]: """convert from an activity to a model instance. Args: model: the django model that this object is being converted to @@ -133,6 +134,9 @@ def to_model( only update blank fields if false allow_external_connections: look up missing data if true, throw an exception if false and an external connection is needed + trigger: the object that originally triggered this + self.to_model. e.g. if this is a Work being dereferenced from + an incoming Edition """ model = model or get_model_from_type(self.type) @@ -223,6 +227,8 @@ def to_model( related_field_name = model_field.field.name for item in values: + if trigger and item == trigger.remote_id: + continue set_related_field.delay( related_model.__name__, instance.__class__.__name__, @@ -369,17 +375,24 @@ def resolve_remote_id( # load the data and create the object try: - data = get_data(remote_id) + data = get_activitypub_data(remote_id) except ConnectionError: logger.info("Could not connect to host for remote_id: %s", remote_id) return None except requests.HTTPError as e: - if (e.response is not None) and e.response.status_code == 401: - # This most likely means it's a mastodon with secure fetch enabled. - data = get_activitypub_data(remote_id) + if ( + hasattr(e, "response") + and hasattr(e.response, "status_code") + and e.response.status_code == 410 + ): + # only log a warning for "gone" since there is not much we can do + logger.warning( + "request for object dropped because it is gone (410) - remote_id: %s", + remote_id, + ) else: - logger.info("Could not connect to host for remote_id: %s", remote_id) - return None + logger.exception("HTTP error - remote_id: %s - error: %s", remote_id, e) + return None # determine the model implicitly, if not provided # or if it's a model with subclasses like Status, check again if not model or hasattr(model.objects, "select_subclasses"): diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py index a53222053c..33c327d0fe 100644 --- a/bookwyrm/activitypub/book.py +++ b/bookwyrm/activitypub/book.py @@ -13,6 +13,7 @@ class BookData(ActivityObject): openlibraryKey: Optional[str] = None inventaireId: Optional[str] = None + finnaKey: Optional[str] = None librarythingKey: Optional[str] = None goodreadsKey: Optional[str] = None bnfId: Optional[str] = None @@ -67,7 +68,6 @@ class Edition(Book): type: str = "Edition" -# pylint: disable=invalid-name @dataclass(init=False) class Work(Book): """work instance of a book object""" diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py index 6a081058cb..376560f6e8 100644 --- a/bookwyrm/activitypub/note.py +++ b/bookwyrm/activitypub/note.py @@ -50,6 +50,7 @@ def to_model( save=True, overwrite=True, allow_external_connections=True, + trigger=None, ): instance = super().to_model( model, instance, allow_create, save, overwrite, allow_external_connections diff --git a/bookwyrm/activitypub/ordered_collection.py b/bookwyrm/activitypub/ordered_collection.py index 32e37c9966..250490041d 100644 --- a/bookwyrm/activitypub/ordered_collection.py +++ b/bookwyrm/activitypub/ordered_collection.py @@ -18,7 +18,6 @@ class OrderedCollection(ActivityObject): type: str = "OrderedCollection" -# pylint: disable=invalid-name @dataclass(init=False) class OrderedCollectionPrivate(OrderedCollection): """an ordered collection with privacy settings""" diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py index a365f4cc07..549f14c9c0 100644 --- a/bookwyrm/activitypub/verbs.py +++ b/bookwyrm/activitypub/verbs.py @@ -22,7 +22,6 @@ def action(self, allow_external_connections=True): self.object.to_model(allow_external_connections=allow_external_connections) -# pylint: disable=invalid-name @dataclass(init=False) class Create(Verb): """Create activity""" @@ -33,7 +32,6 @@ class Create(Verb): type: str = "Create" -# pylint: disable=invalid-name @dataclass(init=False) class Delete(Verb): """Create activity""" @@ -63,7 +61,6 @@ def action(self, allow_external_connections=True): # if we can't find it, we don't need to delete it because we don't have it -# pylint: disable=invalid-name @dataclass(init=False) class Update(Verb): """Update activity""" @@ -227,7 +224,6 @@ def action(self, allow_external_connections=True): self.to_model(allow_external_connections=allow_external_connections) -# pylint: disable=invalid-name @dataclass(init=False) class Announce(Verb): """boosting a status""" diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 0009ac7a36..08fb757d5a 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -32,7 +32,7 @@ def unread_by_status_type_id(self, user_id): stream_id = self.stream_id(user_id) return f"{stream_id}-unread-by-type" - def get_rank(self, obj): # pylint: disable=no-self-use + def get_rank(self, obj): """statuses are sorted by date published""" return obj.published_date.timestamp() diff --git a/bookwyrm/apps.py b/bookwyrm/apps.py index 41b1a17a2e..d5384bb7b5 100644 --- a/bookwyrm/apps.py +++ b/bookwyrm/apps.py @@ -33,7 +33,6 @@ class BookwyrmConfig(AppConfig): name = "bookwyrm" verbose_name = "BookWyrm" - # pylint: disable=no-self-use def ready(self): """set up OTLP and preview image files, if desired""" if settings.OTEL_EXPORTER_OTLP_ENDPOINT or settings.OTEL_EXPORTER_CONSOLE: diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index cf48f4832e..106c68a83a 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -36,7 +36,6 @@ def search( ... -# pylint: disable=arguments-differ def search( query: str, *, @@ -143,7 +142,7 @@ def search_title_author( query = SearchQuery(query, config="simple") | SearchQuery(query, config="english") results = ( books.filter(*filters, search_vector=query) - .annotate(rank=SearchRank(F("search_vector"), query)) + .annotate(rank=SearchRank(F("search_vector"), query, normalization=32)) .filter(rank__gt=min_confidence) .order_by("-rank") ) diff --git a/bookwyrm/connectors/__init__.py b/bookwyrm/connectors/__init__.py index 3a4f5f3e03..ada2a78261 100644 --- a/bookwyrm/connectors/__init__.py +++ b/bookwyrm/connectors/__init__.py @@ -3,4 +3,4 @@ from .abstract_connector import ConnectorException from .abstract_connector import get_data, get_image, maybe_isbn -from .connector_manager import search, first_search_result +from .connector_manager import search, first_search_result, create_finna_connector diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index aa8edbeae9..178601f2ff 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -4,11 +4,10 @@ from typing import Optional, TypedDict, Any, Callable, Union, Iterator from urllib.parse import quote_plus -# pylint: disable-next=deprecated-module -import imghdr # Deprecated in 3.11 for removal in 3.13; no good alternative yet import logging import re import asyncio +from PIL import Image, UnidentifiedImageError import requests from requests.exceptions import RequestException import aiohttp @@ -86,7 +85,7 @@ async def get_results( ), "User-Agent": USER_AGENT, } - params = {"min_confidence": min_confidence} + params = {"min_confidence": str(min_confidence)} try: async with session.get(url, headers=headers, params=params) as response: if not response.ok: @@ -370,13 +369,14 @@ def get_image( return None, None image_content = ContentFile(resp.content) - extension = imghdr.what(None, image_content.read()) - if not extension: + try: + with Image.open(image_content) as im: + extension = str(im.format).lower() + return image_content, extension + except UnidentifiedImageError: logger.info("File requested was not an image: %s", url) return None, None - return image_content, extension - class Mapping: """associate a local database field with a field in an external dataset""" diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index ad68af1dc8..9965586da2 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -206,3 +206,26 @@ def raise_not_valid_url(url: str) -> None: if models.FederatedServer.is_blocked(url): raise ConnectorException(f"Attempting to load data from blocked url: {url}") + + +def create_finna_connector() -> None: + """create a Finna connector""" + + models.Connector.objects.create( + identifier="api.finna.fi", + name="Finna API", + connector_file="finna", + base_url="https://www.finna.fi", + books_url="https://api.finna.fi/api/v1/record" "?id=", + covers_url="https://api.finna.fi", + search_url="https://api.finna.fi/api/v1/search?limit=20" + "&filter[]=format%3a%220%2fBook%2f%22" + "&field[]=title&field[]=recordPage&field[]=authors" + "&field[]=year&field[]=id&field[]=formats&field[]=images" + "&lookfor=", + isbn_search_url="https://api.finna.fi/api/v1/search?limit=1" + "&filter[]=format%3a%220%2fBook%2f%22" + "&field[]=title&field[]=recordPage&field[]=authors&field[]=year" + "&field[]=id&field[]=formats&field[]=images" + "&lookfor=isbn:", + ) diff --git a/bookwyrm/connectors/finna.py b/bookwyrm/connectors/finna.py new file mode 100644 index 0000000000..59eedfbdcd --- /dev/null +++ b/bookwyrm/connectors/finna.py @@ -0,0 +1,398 @@ +"""finna data connector""" + +import re +from typing import Iterator + +from bookwyrm import models +from bookwyrm.book_search import SearchResult +from bookwyrm.models.book import FormatChoices +from .abstract_connector import AbstractConnector, Mapping, JsonDict +from .abstract_connector import get_data +from .connector_manager import ConnectorException, create_edition_task +from .openlibrary_languages import languages + + +class Connector(AbstractConnector): + """instantiate a connector for finna""" + + generated_remote_link_field = "id" + + def __init__(self, identifier: str): + super().__init__(identifier) + + get_first = lambda x, *args: x[0] if x else None + format_remote_id = lambda x: f"{self.books_url}{x}" + format_cover_url = lambda x: f"{self.covers_url}{x[0]}" if x else None + self.book_mappings = [ + Mapping("id", remote_field="id", formatter=format_remote_id), + Mapping("finnaKey", remote_field="id"), + Mapping("title", remote_field="shortTitle"), + Mapping("title", remote_field="title"), + Mapping("subtitle", remote_field="subTitle"), + Mapping("isbn10", remote_field="cleanIsbn"), + Mapping("languages", remote_field="languages", formatter=resolve_languages), + Mapping("authors", remote_field="authors", formatter=parse_authors), + Mapping("subjects", formatter=join_subject_list), + Mapping("publishedDate", remote_field="year"), + Mapping("cover", remote_field="images", formatter=format_cover_url), + Mapping("description", remote_field="summary", formatter=get_first), + Mapping("series", remote_field="series", formatter=parse_series_name), + Mapping( + "seriesNumber", + remote_field="series", + formatter=parse_series_number, + ), + Mapping("publishers", remote_field="publishers"), + Mapping( + "physicalFormat", + remote_field="formats", + formatter=describe_physical_format, + ), + Mapping( + "physicalFormatDetail", + remote_field="physicalDescriptions", + formatter=get_first, + ), + Mapping( + "pages", + remote_field="physicalDescriptions", + formatter=guess_page_numbers, + ), + ] + + self.author_mappings = [ + Mapping("id", remote_field="authors", formatter=self.get_remote_author_id), + Mapping("name", remote_field="authors", formatter=get_first_author), + ] + + def get_book_data(self, remote_id: str) -> JsonDict: + request_parameters = { + "field[]": [ + "authors", + "cleanIsbn", + "formats", + "id", + "images", + "isbns", + "languages", + "physicalDescriptions", + "publishers", + "recordPage", + "series", + "shortTitle", + "subjects", + "subTitle", + "summary", + "title", + "year", + ] + } + data = get_data( + url=remote_id, params=request_parameters # type:ignore[arg-type] + ) + extracted = data.get("records", []) + try: + data = extracted[0] + except (KeyError, IndexError): + raise ConnectorException("Invalid book data") + return data + + def get_remote_author_id(self, data: JsonDict) -> str | None: + """return search url for author info, as we don't + have way to retrieve author-id with the query""" + author = get_first_author(data) + if author: + return f"{self.search_url}{author}&type=Author" + return None + + def get_remote_id(self, data: JsonDict) -> str: + """return record-id page as book-id""" + return f"{self.books_url}{data.get('id')}" + + def parse_search_data( + self, data: JsonDict, min_confidence: float + ) -> Iterator[SearchResult]: + for idx, search_result in enumerate(data.get("records", [])): + authors = search_result.get("authors") + author = None + if authors: + author_list = parse_authors(authors) + if author_list: + author = "; ".join(author_list) + + confidence = 1 / (idx + 1) + if confidence < min_confidence: + break + + # Create some extra info on edition if it is audio-book or e-book + edition_info_title = describe_physical_format(search_result.get("formats")) + edition_info = "" + if edition_info_title and edition_info_title != "Hardcover": + for book_format, info_title in FormatChoices: + if book_format == edition_info_title: + edition_info = f" {info_title}" + break + + search_result = SearchResult( + title=f"{search_result.get('title')}{edition_info}", + key=f"{self.books_url}{search_result.get('id')}", + author=author, + cover=f"{self.covers_url}{search_result.get('images')[0]}" + if search_result.get("images") + else None, + year=search_result.get("year"), + view_link=f"{self.base_url}{search_result.get('recordPage')}", + confidence=confidence, + connector=self, + ) + yield search_result + + def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]: + """got some data""" + for idx, search_result in enumerate(data.get("records", [])): + authors = search_result.get("authors") + author = None + if authors: + author_list = parse_authors(authors) + if author_list: + author = "; ".join(author_list) + + confidence = 1 / (idx + 1) + yield SearchResult( + title=search_result.get("title"), + key=f"{self.books_url}{search_result.get('id')}", + author=author, + cover=f"{self.covers_url}{search_result.get('images')[0]}" + if search_result.get("images") + else None, + year=search_result.get("year"), + view_link=f"{self.base_url}{search_result.get('recordPage')}", + confidence=confidence, + connector=self, + ) + + def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]: + authors = data.get("authors") + if authors: + for author in parse_authors(authors): + model = self.get_or_create_author( + f"{self.search_url}{author}&type=Author" + ) + if model: + yield model + + def expand_book_data(self, book: models.Book) -> None: + work = book + # go from the edition to the work, if necessary + if isinstance(book, models.Edition): + work = book.parent_work + + try: + edition_options = retrieve_versions(work.finna_key) + except ConnectorException: + return + + for edition in edition_options: + remote_id = self.get_remote_id(edition) + if remote_id: + create_edition_task.delay(self.connector.id, work.id, edition) + + def get_remote_id_from_model(self, obj: models.BookDataModel) -> str: + """use get_remote_id to figure out the link from a model obj""" + return f"{self.books_url}{obj.finna_key}" + + def is_work_data(self, data: JsonDict) -> bool: + """ + https://api.finna.fi/v1/search?id=anders.1946700&search=versions&view=&lng=fi&field[]=formats&field[]=series&field[]=title&field[]=authors&field[]=summary&field[]=cleanIsbn&field[]=id + + No real ordering what is work and what is edition, so pick first version as work + """ + edition_list = retrieve_versions(data.get("id")) + if edition_list: + return data.get("id") == edition_list[0].get("id") + return True + + def get_edition_from_work_data(self, data: JsonDict) -> JsonDict: + """No real distinctions what is work/edition, + so check all versions and pick preferred edition""" + edition_list = retrieve_versions(data.get("id")) + if not edition_list: + raise ConnectorException("No editions found for work") + edition = pick_preferred_edition(edition_list) + if not edition: + raise ConnectorException("No editions found for work") + return edition + + def get_work_from_edition_data(self, data: JsonDict) -> JsonDict: + return retrieve_versions(data.get("id"))[0] + + +def guess_page_numbers(data: JsonDict) -> str | None: + """Try to retrieve page count of edition""" + for row in data: + # Try to match page count text in style of '134 pages' or '134 sivua' + page_search = re.search(r"(\d+) (sivua|s\.|sidor|pages)", row) + page_count = page_search.group(1) if page_search else None + if page_count: + return page_count + # If we didn't match, try starting number + page_search = re.search(r"^(\d+)", row) + page_count = page_search.group(1) if page_search else None + if page_count: + return page_count + return None + + +def resolve_languages(data: JsonDict) -> list[str]: + """Use openlibrary language code list to resolve iso-lang codes""" + result_languages = [] + for language_code in data: + result_languages.append( + languages.get(f"/languages/{language_code}", language_code) + ) + return result_languages + + +def join_subject_list(data: list[JsonDict]) -> list[str]: + """Join list of string list about subject topics as one list""" + return [" ".join(info) for info in data] + + +def describe_physical_format(formats: list[JsonDict]) -> str: + """Map if book is physical book, eBook or audiobook""" + found_format = "Hardcover" + # Map finnish finna formats to bookwyrm codes + format_mapping = { + "1/Book/Book/": "Hardcover", + "1/Book/AudioBook/": "AudiobookFormat", + "1/Book/eBook/": "EBook", + } + for format_to_check in formats: + format_value = format_to_check.get("value") + if not isinstance(format_value, str): + continue + if (mapping_match := format_mapping.get(format_value, None)) is not None: + found_format = mapping_match + return found_format + + +def parse_series_name(series: list[JsonDict]) -> str | None: + """Parse series name if given""" + for info in series: + if "name" in info: + return info.get("name") + return None + + +def parse_series_number(series: list[JsonDict]) -> str | None: + """Parse series number from additional info if given""" + for info in series: + if "additional" in info: + return info.get("additional") + return None + + +def retrieve_versions(book_id: str | None) -> list[JsonDict]: + """ + https://api.finna.fi/v1/search?id=anders.1946700&search=versions&view=& + + Search all editions/versions of the book that finna is aware of + """ + + if not book_id: + return [] + + request_parameters = { + "id": book_id, + "search": "versions", + "view": "", + "field[]": [ + "authors", + "cleanIsbn", + "edition", + "formats", + "id", + "images", + "isbns", + "languages", + "physicalDescriptions", + "publishers", + "recordPage", + "series", + "shortTitle", + "subjects", + "subTitle", + "summary", + "title", + "year", + ], + } + data = get_data( + url="https://api.finna.fi/api/v1/search", + params=request_parameters, # type: ignore[arg-type] + ) + result = data.get("records", []) + if isinstance(result, list): + return result + return [] + + +def get_first_author(data: JsonDict) -> str | None: + """Parse authors and return first one, usually the main author""" + authors = parse_authors(data) + if authors: + return authors[0] + return None + + +def parse_authors(data: JsonDict) -> list[str]: + """Search author info, they are given in SurName, FirstName style + return them also as FirstName SurName order""" + if author_keys := data.get("primary", None): + if author_keys: + # we search for 'kirjoittaja' role, if any found + tulos = list( + # Convert from 'Lewis, Michael' to 'Michael Lewis' + " ".join(reversed(author_key.split(", "))) + for author_key, author_info in author_keys.items() + if "kirjoittaja" in author_info.get("role", []) + ) + if tulos: + return tulos + # if not found, we search any role that is not specificly something + tulos = list( + " ".join(reversed(author_key.split(", "))) + for author_key, author_info in author_keys.items() + if "-" in author_info.get("role", []) + ) + return tulos + return [] + + +def pick_preferred_edition(options: list[JsonDict]) -> JsonDict | None: + """favor physical copies with covers in english""" + if not options: + return None + if len(options) == 1: + return options[0] + + # pick hardcodver book if present over eBook/audiobook + formats = ["1/Book/Book/"] + format_selection = [] + for edition in options: + for edition_format in edition.get("formats", []): + if edition_format.get("value") in formats: + format_selection.append(edition) + options = format_selection or options + + # Prefer Finnish/Swedish language editions if any found + language_list = ["fin", "swe"] + languages_selection = [] + for edition in options: + for edition_language in edition.get("languages", []): + if edition_language in language_list: + languages_selection.append(edition) + options = languages_selection or options + + options = [e for e in options if e.get("cleanIsbn")] or options + return options[0] diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py index 249f6b9ca5..53aade98d0 100644 --- a/bookwyrm/connectors/inventaire.py +++ b/bookwyrm/connectors/inventaire.py @@ -85,12 +85,15 @@ def get_book_data(self, remote_id: str) -> JsonDict: def parse_search_data( self, data: JsonDict, min_confidence: float ) -> Iterator[SearchResult]: + best_score = None for search_result in data.get("results", []): - images = search_result.get("image") - cover = f"{self.covers_url}/img/entities/{images[0]}" if images else None + image = search_result.get("image") + cover = f"{self.covers_url}{image}" if image else None # a deeply messy translation of inventaire's scores confidence = float(search_result.get("_score", 0.1)) - confidence = 0.1 if confidence < 150 else 0.999 + if best_score is None: + best_score = confidence + confidence = (confidence / best_score) - 0.001 if confidence < min_confidence: continue yield SearchResult( @@ -222,9 +225,10 @@ def resolve_keys(self, keys: Iterable[str]) -> list[str]: def get_description(self, links: JsonDict) -> str: """grab an extracted excerpt from wikipedia""" link = links.get("enwiki") - if not link: + if not link or not link.get("title"): return "" - url = f"{self.base_url}/api/data?action=wp-extract&lang=en&title={link}" + title = link.get("title") + url = f"{self.base_url}/api/data?action=wp-extract&lang=en&title={title}" try: data = get_data(url) except ConnectorException: diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 4dc6d6ac14..b6ecc4cd54 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -31,8 +31,12 @@ def __init__(self, identifier: str): Mapping("subtitle"), Mapping("description", formatter=get_description), Mapping("languages", formatter=get_languages), - Mapping("series", formatter=get_first), - Mapping("seriesNumber", remote_field="series_number"), + Mapping("series", formatter=parse_series), + Mapping( + "seriesNumber", + remote_field="series", + formatter=parse_series_number, + ), Mapping("subjects"), Mapping("subjectPlaces", remote_field="subject_places"), Mapping("isbn13", remote_field="isbn_13", formatter=get_first), @@ -186,11 +190,14 @@ def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]: key = self.books_url + search_result["key"] authors = search_result.get("authors") or [{"name": "Unknown"}] author_names = [author.get("name") for author in authors] + cover_obj = search_result.get("cover") + cover = cover_obj.get("medium") if cover_obj else "" yield SearchResult( title=search_result.get("title"), key=key, author=", ".join(author_names), connector=self, + cover=cover, year=search_result.get("publish_date"), ) @@ -323,3 +330,47 @@ def pick_default_edition(options: list[JsonDict]) -> Optional[JsonDict]: options = [e for e in options if e.get("isbn_13")] or options options = [e for e in options if e.get("ocaid")] or options return options[0] + + +def parse_series(data: list[str]) -> str | None: + """try to parse series name from different styles, + * 'series name, #1' + * 'title -- number' + * 'title, Book number' + * 'title (number)' + """ + if not data: + return None + series_title = data[0].strip() + for regex_to_try in [ + r"(.+)(?:, ?#\d+)$", + r"(.+)(?:-- ?\d+)$", + r"(.+)(?:, Book ?\d+)$", + r"(.+)(?: \(\d+\))$", + ]: + if series_match := re.search(regex_to_try, series_title): + series_name = series_match.group(1).strip() + return series_name + return series_title + + +def parse_series_number(data: list[str]) -> str | None: + """try to parse series number from different styles, + * 'series name, #1' + * 'title -- number' + * 'title, Book number' + * 'title (number)' + """ + if not data: + return None + series_title = data[0].strip() + for regex_to_try in [ + r"(.+)#(\d+)$", + r"(.+) -- (\d+)$", + r"(.+), Book (\d+)$", + r"(.+)\((\d+)\)", + ]: + if series_match := re.search(regex_to_try, series_title): + series_number = series_match.group(2) + return series_number + return None diff --git a/bookwyrm/connectors/settings.py b/bookwyrm/connectors/settings.py index 927e39b265..4ef149a21d 100644 --- a/bookwyrm/connectors/settings.py +++ b/bookwyrm/connectors/settings.py @@ -1,3 +1,8 @@ """ settings book data connectors """ -CONNECTORS = ["openlibrary", "inventaire", "bookwyrm_connector"] +CONNECTORS = [ + "openlibrary", + "inventaire", + "bookwyrm_connector", + "finna", +] diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py index 0047bfce11..bec704a2c7 100644 --- a/bookwyrm/context_processors.py +++ b/bookwyrm/context_processors.py @@ -2,7 +2,7 @@ from bookwyrm import models, settings -def site_settings(request): # pylint: disable=unused-argument +def site_settings(request): """include the custom info about the site""" request_protocol = "https://" if not request.is_secure(): diff --git a/bookwyrm/forms/admin.py b/bookwyrm/forms/admin.py index 72f50ccb87..783c78d50b 100644 --- a/bookwyrm/forms/admin.py +++ b/bookwyrm/forms/admin.py @@ -199,3 +199,8 @@ def save(self, request, *args, **kwargs): if not request.user.has_perm("bookwyrm.moderate_user"): raise PermissionDenied() return super().save(*args, **kwargs) + + +class ExportFileExpiryForm(forms.Form): + + hours = forms.IntegerField(min_value=1) diff --git a/bookwyrm/forms/books.py b/bookwyrm/forms/books.py index f73ce3f5a3..f9a110efc9 100644 --- a/bookwyrm/forms/books.py +++ b/bookwyrm/forms/books.py @@ -4,6 +4,7 @@ from file_resubmit.widgets import ResubmitImageWidget from bookwyrm import models +from bookwyrm.settings import DATA_UPLOAD_MAX_MEMORY_SIZE from .custom_form import CustomForm from .widgets import ArrayWidget, SelectDateWidget, Select @@ -16,6 +17,22 @@ class Meta: help_texts = {f: None for f in fields} +class ResubmitImageWidgetWithWarning(ResubmitImageWidget): + """Define template to use that shows warning on too big image""" + + template_name = "widgets/clearable_file_input_with_warning.html" + + def get_context(self, name, value, attrs): + context = super().get_context(name, value, attrs) + context["widget"]["attrs"].update( + { + "data-max-upload": DATA_UPLOAD_MAX_MEMORY_SIZE, + "max_mb": DATA_UPLOAD_MAX_MEMORY_SIZE >> 20, + } + ) + return context + + class EditionForm(CustomForm): class Meta: model = models.Edition @@ -40,6 +57,7 @@ class Meta: "openlibrary_key", "inventaire_id", "goodreads_key", + "finna_key", "oclc_number", "asin", "aasin", @@ -71,7 +89,9 @@ class Meta: "published_date": SelectDateWidget( attrs={"aria-describedby": "desc_published_date"} ), - "cover": ResubmitImageWidget(attrs={"aria-describedby": "desc_cover"}), + "cover": ResubmitImageWidgetWithWarning( + attrs={"aria-describedby": "desc_cover"} + ), "physical_format": Select( attrs={"aria-describedby": "desc_physical_format"} ), @@ -93,6 +113,7 @@ class Meta: "oclc_number": forms.TextInput( attrs={"aria-describedby": "desc_oclc_number"} ), + "finna_key": forms.TextInput(attrs={"aria-describedby": "desc_finna_key"}), "ASIN": forms.TextInput(attrs={"aria-describedby": "desc_ASIN"}), "AASIN": forms.TextInput(attrs={"aria-describedby": "desc_AASIN"}), "isfdb": forms.TextInput(attrs={"aria-describedby": "desc_isfdb"}), diff --git a/bookwyrm/forms/custom_form.py b/bookwyrm/forms/custom_form.py index c604deea42..6b425d216a 100644 --- a/bookwyrm/forms/custom_form.py +++ b/bookwyrm/forms/custom_form.py @@ -15,9 +15,9 @@ def __init__(self, *args, **kwargs): css_classes["number"] = "input" css_classes["checkbox"] = "checkbox" css_classes["textarea"] = "textarea" - # pylint: disable=super-with-arguments super().__init__(*args, **kwargs) for visible in self.visible_fields(): + input_type = "" if hasattr(visible.field.widget, "input_type"): input_type = visible.field.widget.input_type if isinstance(visible.field.widget, Textarea): diff --git a/bookwyrm/forms/edit_user.py b/bookwyrm/forms/edit_user.py index 9024972c33..018e5b16cd 100644 --- a/bookwyrm/forms/edit_user.py +++ b/bookwyrm/forms/edit_user.py @@ -18,6 +18,7 @@ class Meta: "email", "summary", "show_goal", + "show_ratings", "show_suggested_users", "manually_approves_followers", "default_post_privacy", @@ -111,7 +112,7 @@ def clean(self): self.add_error("confirm_password", _("Password does not match")) try: - validate_password(new_password) + validate_password(new_password, user=self.instance) except ValidationError as err: self.add_error("password", err) diff --git a/bookwyrm/forms/forms.py b/bookwyrm/forms/forms.py index 3d555f308d..0ecf3e3018 100644 --- a/bookwyrm/forms/forms.py +++ b/bookwyrm/forms/forms.py @@ -6,6 +6,7 @@ from bookwyrm import models from bookwyrm.models.user import FeedFilterChoices +from bookwyrm.models.fields import ClearableFileInputWithWarning from .custom_form import CustomForm # pylint: disable=missing-class-docstring @@ -22,11 +23,11 @@ class Meta: class ImportForm(forms.Form): - csv_file = forms.FileField() + csv_file = forms.FileField(widget=ClearableFileInputWithWarning) class ImportUserForm(forms.Form): - archive_file = forms.FileField() + archive_file = forms.FileField(widget=ClearableFileInputWithWarning) class ShelfForm(CustomForm): diff --git a/bookwyrm/forms/landing.py b/bookwyrm/forms/landing.py index 1da4fc4f11..290ee3f376 100644 --- a/bookwyrm/forms/landing.py +++ b/bookwyrm/forms/landing.py @@ -34,7 +34,6 @@ def infer_username(self): def add_invalid_password_error(self): """We don't want to be too specific about this""" - # pylint: disable=attribute-defined-outside-init self.non_field_errors = _("Username or password are incorrect") @@ -49,8 +48,15 @@ def clean(self): """Check if the username is taken""" cleaned_data = super().clean() localname = cleaned_data.get("localname").strip() + + # Create a temporary user instance for password validation + temp_user = self._meta.model( + localname=localname, + email=cleaned_data.get("email"), + ) + try: - validate_password(cleaned_data.get("password")) + validate_password(cleaned_data.get("password"), user=temp_user) except ValidationError as err: self.add_error("password", err) if models.User.objects.filter(localname=localname).first(): @@ -65,6 +71,10 @@ def clean(self): if email and models.User.objects.filter(email=email).exists(): self.add_error("email", _("A user with this email already exists.")) + email_domain = email.split("@")[-1] + if email and models.EmailBlocklist.objects.filter(domain=email_domain).exists(): + self.add_error("email", _("This email address cannot be registered.")) + class Meta: model = models.InviteRequest fields = ["email", "answer"] @@ -90,7 +100,7 @@ def clean(self): self.add_error("confirm_password", _("Password does not match")) try: - validate_password(new_password) + validate_password(new_password, user=self.instance) except ValidationError as err: self.add_error("password", err) diff --git a/bookwyrm/forms/links.py b/bookwyrm/forms/links.py index 5156d2578d..06de9a304d 100644 --- a/bookwyrm/forms/links.py +++ b/bookwyrm/forms/links.py @@ -30,22 +30,26 @@ def clean(self): if models.LinkDomain.objects.filter(domain=domain).exists(): status = models.LinkDomain.objects.get(domain=domain).status if status == "blocked": - # pylint: disable=line-too-long self.add_error( "url", _( - "This domain is blocked. Please contact your administrator if you think this is an error." + "This domain is blocked. " + "Please contact your administrator if you think " + "this is an error." ), ) - if ( - models.FileLink.objects.filter(url=url, book=book, filetype=filetype) - .exclude(pk=self.instance) - .exists() - ): - # pylint: disable=line-too-long - self.add_error( - "url", - _( - "This link with file type has already been added for this book. If it is not visible, the domain is still pending." - ), - ) + return + if current_links := models.FileLink.objects.filter( + url=url, book=book, filetype=filetype + ).all(): + for link in current_links: + if link == self.instance: + continue + self.add_error( + "url", + _( + "This link with file type has already been added for this book." + " If it is not visible, the domain is still pending." + ), + ) + break diff --git a/bookwyrm/forms/widgets.py b/bookwyrm/forms/widgets.py index ee9345aa03..001fdbec40 100644 --- a/bookwyrm/forms/widgets.py +++ b/bookwyrm/forms/widgets.py @@ -5,8 +5,6 @@ class ArrayWidget(forms.widgets.TextInput): """Inputs for postgres array fields""" - # pylint: disable=unused-argument - # pylint: disable=no-self-use def value_from_datadict(self, data, files, name): """get all values for this name""" return [i for i in data.getlist(name) if i] diff --git a/bookwyrm/importers/__init__.py b/bookwyrm/importers/__init__.py index 8e92872f25..497eebcf69 100644 --- a/bookwyrm/importers/__init__.py +++ b/bookwyrm/importers/__init__.py @@ -1,9 +1,10 @@ """ import classes """ from .importer import Importer -from .bookwyrm_import import BookwyrmImporter +from .bookwyrm_import import BookwyrmImporter, BookwyrmBooksImporter from .calibre_import import CalibreImporter from .goodreads_import import GoodreadsImporter from .librarything_import import LibrarythingImporter from .openlibrary_import import OpenLibraryImporter from .storygraph_import import StorygraphImporter +from .openreads_import import OpenReadsImporter diff --git a/bookwyrm/importers/bookwyrm_import.py b/bookwyrm/importers/bookwyrm_import.py index 206cd62197..d4fedb4f79 100644 --- a/bookwyrm/importers/bookwyrm_import.py +++ b/bookwyrm/importers/bookwyrm_import.py @@ -3,6 +3,7 @@ from bookwyrm.models import User from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob +from . import Importer class BookwyrmImporter: @@ -21,4 +22,33 @@ def process_import( job = BookwyrmImportJob.objects.create( user=user, archive_file=archive_file, required=required ) + return job + + def create_retry_job( + self, user: User, original_job: BookwyrmImportJob + ) -> BookwyrmImportJob: + """retry items that didn't import""" + + job = BookwyrmImportJob.objects.create( + user=user, + archive_file=original_job.archive_file, + required=original_job.required, + retry=True, + ) + + return job + + +class BookwyrmBooksImporter(Importer): + """ + Handle reading a csv from BookWyrm. + Goodreads is the default importer, we basically just use the same structure + But BookWyrm has additional attributes in the csv + """ + + service = "BookWyrm" + row_mappings_guesses = Importer.row_mappings_guesses + [ + ("shelf_name", ["shelf_name"]), + ("review_published", ["review_published"]), + ] diff --git a/bookwyrm/importers/goodreads_import.py b/bookwyrm/importers/goodreads_import.py index c0dc0ea283..8b3e94cbc7 100644 --- a/bookwyrm/importers/goodreads_import.py +++ b/bookwyrm/importers/goodreads_import.py @@ -1,4 +1,5 @@ """ handle reading a csv from goodreads """ +from typing import Optional from . import Importer @@ -7,3 +8,10 @@ class GoodreadsImporter(Importer): For a more complete example of overriding see librarything_import.py""" service = "Goodreads" + + def normalize_row( + self, entry: dict[str, str], mappings: dict[str, Optional[str]] + ) -> dict[str, Optional[str]]: + normalized = super().normalize_row(entry, mappings) + normalized["goodreads_key"] = normalized["id"] + return normalized diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 5b3192fa5b..d2a11d7f21 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -18,17 +18,26 @@ class Importer: row_mappings_guesses = [ ("id", ["id", "book id"]), ("title", ["title"]), - ("authors", ["author", "authors", "primary author"]), - ("isbn_10", ["isbn10", "isbn", "isbn/uid"]), - ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]), + ("authors", ["author_text", "author", "authors", "primary author"]), + ("isbn_10", ["isbn_10", "isbn10", "isbn", "isbn/uid"]), + ("isbn_13", ["isbn_13", "isbn13", "isbn", "isbns", "isbn/uid"]), ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]), - ("review_name", ["review name"]), - ("review_body", ["my review", "review"]), + ("review_name", ["review_name", "review name"]), + ("review_body", ["review_content", "my review", "review"]), ("rating", ["my rating", "rating", "star rating"]), - ("date_added", ["date added", "entry date", "added"]), - ("date_started", ["date started", "started"]), - ("date_finished", ["date finished", "last date read", "date read", "finished"]), + ( + "date_added", + ["shelf_date", "date_added", "date added", "entry date", "added"], + ), + ("date_started", ["start_date", "date started", "started"]), + ( + "date_finished", + ["finish_date", "date finished", "last date read", "date read", "finished"], + ), ] + + # TODO: stopped + date_fields = ["date_added", "date_started", "date_finished"] shelf_mapping_guesses = { "to-read": ["to-read", "want to read"], @@ -36,9 +45,14 @@ class Importer: "reading": ["currently-reading", "reading", "currently reading"], } - # pylint: disable=too-many-locals + # pylint: disable=too-many-arguments def create_job( - self, user: User, csv_file: Iterable[str], include_reviews: bool, privacy: str + self, + user: User, + csv_file: Iterable[str], + include_reviews: bool, + privacy: str, + create_shelves: bool = True, ) -> ImportJob: """check over a csv and creates a database entry for the job""" csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter) @@ -55,6 +69,7 @@ def create_job( job = ImportJob.objects.create( user=user, include_reviews=include_reviews, + create_shelves=create_shelves, privacy=privacy, mappings=mappings, source=self.service, @@ -114,7 +129,7 @@ def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]: shelf = [ s for (s, gs) in self.shelf_mapping_guesses.items() if shelf_name in gs ] - return shelf[0] if shelf else None + return shelf[0] if shelf else normalized_row.get("shelf") or None # pylint: disable=no-self-use def normalize_row( @@ -149,6 +164,7 @@ def create_retry_job( job = ImportJob.objects.create( user=user, include_reviews=original_job.include_reviews, + create_shelves=original_job.create_shelves, privacy=original_job.privacy, source=original_job.source, # TODO: allow users to adjust mappings diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py index 145657ba08..24a2626bf6 100644 --- a/bookwyrm/importers/librarything_import.py +++ b/bookwyrm/importers/librarything_import.py @@ -20,7 +20,7 @@ class LibrarythingImporter(Importer): def normalize_row( self, entry: dict[str, str], mappings: dict[str, Optional[str]] - ) -> dict[str, Optional[str]]: # pylint: disable=no-self-use + ) -> dict[str, Optional[str]]: """use the dataclass to create the formatted row of data""" normalized = { k: _remove_brackets(entry.get(v) if v else None) diff --git a/bookwyrm/importers/openreads_import.py b/bookwyrm/importers/openreads_import.py new file mode 100644 index 0000000000..e6e12c2ef1 --- /dev/null +++ b/bookwyrm/importers/openreads_import.py @@ -0,0 +1,60 @@ +""" handle reading a csv from openreads""" +from typing import Any, Optional +from datetime import datetime +from bookwyrm.models import Shelf + +from . import Importer + + +def parse_iso_timestamp(iso_date: str | None) -> None | str: + """Parse iso timestamp and return iso-formated date""" + if not iso_date: + return iso_date + return datetime.fromisoformat(iso_date).date().isoformat() + + +class OpenReadsImporter(Importer): + """csv downloads from OpenLibrary""" + + service = "OpenReads" + + def __init__(self, *args: Any, **kwargs: Any): + self.row_mappings_guesses.append(("openlibrary_key", ["olid"])) + self.row_mappings_guesses.append(("pages", ["pages"])) + self.row_mappings_guesses.append(("description", ["description"])) + self.row_mappings_guesses.append(("physical_format", ["book_format"])) + self.row_mappings_guesses.append(("published_date", ["publication_year"])) + super().__init__(*args, **kwargs) + + def normalize_row( + self, entry: dict[str, str], mappings: dict[str, Optional[str]] + ) -> dict[str, Optional[str]]: + normalized = {k: entry.get(v) if v else None for k, v in mappings.items()} + + reading_list = value.split(";") if (value := entry.get("readings")) else [] + if reading_list: + if reading_dates := reading_list[0].split("|"): + normalized["date_started"] = ( + parse_iso_timestamp(reading_dates[0]) or None + ) + normalized["date_finished"] = ( + parse_iso_timestamp(reading_dates[1]) or None + ) + if date_added := normalized.get("date_added"): + normalized["date_added"] = parse_iso_timestamp(date_added) + if read_status := entry.get("status"): + match read_status: + case "finished": + normalized["shelf"] = Shelf.READ_FINISHED + case "in_progress": + normalized["shelf"] = Shelf.READING + case "abandoned": + normalized["shelf"] = Shelf.STOPPED_READING + return normalized + + def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]: + if normalized_row["date_finished"]: + return Shelf.READ_FINISHED + if normalized_row["date_started"]: + return Shelf.READING + return Shelf.TO_READ diff --git a/bookwyrm/lists_stream.py b/bookwyrm/lists_stream.py index 148b81a78b..479eacb193 100644 --- a/bookwyrm/lists_stream.py +++ b/bookwyrm/lists_stream.py @@ -18,7 +18,7 @@ def stream_id(self, user): # pylint: disable=no-self-use return f"{user}-lists" return f"{user.id}-lists" - def get_rank(self, obj): # pylint: disable=no-self-use + def get_rank(self, obj): """lists are sorted by updated date""" return obj.updated_date.timestamp() diff --git a/bookwyrm/management/commands/add_finna_connector.py b/bookwyrm/management/commands/add_finna_connector.py new file mode 100644 index 0000000000..6da28570a3 --- /dev/null +++ b/bookwyrm/management/commands/add_finna_connector.py @@ -0,0 +1,58 @@ +""" Add finna connector to connectors """ +from django.core.management.base import BaseCommand + +from bookwyrm import models + + +def enable_finna_connector(): + + models.Connector.objects.create( + identifier="api.finna.fi", + name="Finna API", + connector_file="finna", + base_url="https://www.finna.fi", + books_url="https://api.finna.fi/api/v1/record" "?id=", + covers_url="https://api.finna.fi", + search_url="https://api.finna.fi/api/v1/search?limit=20" + "&filter[]=format%3a%220%2fBook%2f%22" + "&field[]=title&field[]=recordPage&field[]=authors" + "&field[]=year&field[]=id&field[]=formats&field[]=images" + "&lookfor=", + isbn_search_url="https://api.finna.fi/api/v1/search?limit=1" + "&filter[]=format%3a%220%2fBook%2f%22" + "&field[]=title&field[]=recordPage&field[]=authors&field[]=year" + "&field[]=id&field[]=formats&field[]=images" + "&lookfor=isbn:", + ) + + +def remove_finna_connector(): + models.Connector.objects.filter(identifier="api.finna.fi").update( + active=False, deactivation_reason="Disabled by management command" + ) + print("Finna connector deactivated") + + +# pylint: disable=no-self-use +# pylint: disable=unused-argument +class Command(BaseCommand): + """command-line options""" + + help = "Setup Finna API connector" + + def add_arguments(self, parser): + """specify argument to remove connector""" + parser.add_argument( + "--deactivate", + action="store_true", + help="Deactivate the finna connector from config", + ) + + def handle(self, *args, **options): + """enable or remove connector""" + if options.get("deactivate"): + print("Deactivate finna connector config if one present") + remove_finna_connector() + else: + print("Adding Finna API connector to configuration") + enable_finna_connector() diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py index ef8aff0fb8..88941a653e 100644 --- a/bookwyrm/management/commands/initdb.py +++ b/bookwyrm/management/commands/initdb.py @@ -77,7 +77,7 @@ def init_permissions(): def init_connectors(): """access book data sources""" - models.Connector.objects.create( + models.Connector.objects.get_or_create( identifier="bookwyrm.social", name="Bookwyrm.social", connector_file="bookwyrm_connector", @@ -90,7 +90,7 @@ def init_connectors(): ) # pylint: disable=line-too-long - models.Connector.objects.create( + models.Connector.objects.get_or_create( identifier="inventaire.io", name="Inventaire", connector_file="inventaire", @@ -99,10 +99,10 @@ def init_connectors(): covers_url="https://inventaire.io", search_url="https://inventaire.io/api/search?types=works&types=works&search=", isbn_search_url="https://inventaire.io/api/entities?action=by-uris&uris=isbn%3A", - priority=1, + priority=3, ) - models.Connector.objects.create( + models.Connector.objects.get_or_create( identifier="openlibrary.org", name="OpenLibrary", connector_file="openlibrary", @@ -111,19 +111,20 @@ def init_connectors(): covers_url="https://covers.openlibrary.org", search_url="https://openlibrary.org/search?q=", isbn_search_url="https://openlibrary.org/api/books?jscmd=data&format=json&bibkeys=ISBN:", - priority=1, + priority=3, ) def init_settings(): """info about the instance""" group_editor = Group.objects.filter(name="editor").first() - models.SiteSettings.objects.create( - support_link="https://www.patreon.com/bookwyrm", - support_title="Patreon", - install_mode=True, - default_user_auth_group=group_editor, - ) + if not models.SiteSettings.objects.all().first(): + models.SiteSettings.objects.create( + support_link="https://www.patreon.com/bookwyrm", + support_title="Patreon", + install_mode=True, + default_user_auth_group=group_editor, + ) def init_link_domains(): @@ -136,7 +137,7 @@ def init_link_domains(): ("theanarchistlibrary.org", "The Anarchist Library"), ] for domain, name in domains: - models.LinkDomain.objects.create( + models.LinkDomain.objects.get_or_create( domain=domain, name=name, status="approved", diff --git a/bookwyrm/management/commands/show_duplicate_authors.py b/bookwyrm/management/commands/show_duplicate_authors.py new file mode 100644 index 0000000000..34e5da2b0e --- /dev/null +++ b/bookwyrm/management/commands/show_duplicate_authors.py @@ -0,0 +1,51 @@ +from django.core.management.base import BaseCommand +from django.db.models import Count +from bookwyrm import models + + +def find_duplicate_author_names(): + """Show authors that have same name""" + dupes = ( + models.Author.objects.values("name") + .annotate(Count("name")) + .filter(name__count__gt=1) + .exclude(name="") + .exclude(name__isnull=True) + .order_by("name__count") + ) + + for dupe in dupes: + value = dupe["name"] + print("----------") + objs = ( + models.Author.objects.filter(name=value) + .annotate(num_books=Count("book", distinct=True)) + .order_by("-num_books", "id") + ) + print( + "You could check if the following authors are actually the same and can be merged, (only checked based on name)" + ) + for obj in objs: + born = obj.born.year if obj.born else "" + died = obj.died.year if obj.died else "" + years = "" + if born or died: + years = f" ({born}-{died})" + print( + f"- {obj.remote_id}, {obj.name}{years} book editions found:{obj.num_books}" + ) + + +class Command(BaseCommand): + """Show all the authors that appear with same name, but different id""" + + help = "show authors with same name but different id" + + # pylint: disable=no-self-use,unused-argument + def handle(self, *args, **options): + """run deduplications""" + find_duplicate_author_names() + print("----------") + print( + "You should manually check each author id to determine if they are same author before thinking of merging" + ) diff --git a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py index f25bafe157..b35d6ca480 100644 --- a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py +++ b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py @@ -481,7 +481,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="notification", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( notification_type__in=[ "FAVORITE", "REPLY", @@ -496,7 +496,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="userblocks", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( _negated=True, user_subject=django.db.models.expressions.F("user_object"), ), @@ -506,7 +506,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="userfollowrequest", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( _negated=True, user_subject=django.db.models.expressions.F("user_object"), ), @@ -516,7 +516,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="userfollows", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( _negated=True, user_subject=django.db.models.expressions.F("user_object"), ), @@ -610,7 +610,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="connector", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( connector_file__in=bookwyrm.models.connector.ConnectorFiles ), name="connector_file_valid", @@ -841,7 +841,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="notification", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( notification_type__in=[ "FAVORITE", "REPLY", @@ -1099,7 +1099,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="notification", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( notification_type__in=[ "FAVORITE", "REPLY", @@ -1182,7 +1182,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="notification", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( notification_type__in=[ "FAVORITE", "REPLY", @@ -1644,7 +1644,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="notification", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( notification_type__in=[ "FAVORITE", "REPLY", diff --git a/bookwyrm/migrations/0045_auto_20210210_2114.py b/bookwyrm/migrations/0045_auto_20210210_2114.py index 22f33cf471..ebf3042ee1 100644 --- a/bookwyrm/migrations/0045_auto_20210210_2114.py +++ b/bookwyrm/migrations/0045_auto_20210210_2114.py @@ -90,7 +90,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="notification", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( notification_type__in=[ "FAVORITE", "REPLY", diff --git a/bookwyrm/migrations/0046_reviewrating.py b/bookwyrm/migrations/0046_reviewrating.py index 26f6f36a69..961075fe5a 100644 --- a/bookwyrm/migrations/0046_reviewrating.py +++ b/bookwyrm/migrations/0046_reviewrating.py @@ -4,7 +4,6 @@ from django.db import connection from django.db.models import Q import django.db.models.deletion -from psycopg2.extras import execute_values def convert_review_rating(app_registry, schema_editor): @@ -19,8 +18,7 @@ def convert_review_rating(app_registry, schema_editor): with connection.cursor() as cursor: values = [(r.id,) for r in reviews] - execute_values( - cursor, + cursor.executemany( """ INSERT INTO bookwyrm_reviewrating(review_ptr_id) VALUES %s""", diff --git a/bookwyrm/migrations/0049_auto_20210309_0156.py b/bookwyrm/migrations/0049_auto_20210309_0156.py index ae9d77a893..7dd0411f4e 100644 --- a/bookwyrm/migrations/0049_auto_20210309_0156.py +++ b/bookwyrm/migrations/0049_auto_20210309_0156.py @@ -104,7 +104,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="report", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( _negated=True, reporter=django.db.models.expressions.F("user") ), name="self_report", diff --git a/bookwyrm/migrations/0051_auto_20210316_1950.py b/bookwyrm/migrations/0051_auto_20210316_1950.py index 3caecbbe13..6ef5e2a122 100644 --- a/bookwyrm/migrations/0051_auto_20210316_1950.py +++ b/bookwyrm/migrations/0051_auto_20210316_1950.py @@ -46,7 +46,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="notification", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( notification_type__in=[ "FAVORITE", "REPLY", diff --git a/bookwyrm/migrations/0086_auto_20210827_1727.py b/bookwyrm/migrations/0086_auto_20210827_1727.py index ef6af206b8..4550628177 100644 --- a/bookwyrm/migrations/0086_auto_20210827_1727.py +++ b/bookwyrm/migrations/0086_auto_20210827_1727.py @@ -31,7 +31,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="readthrough", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( ("finish_date__gte", django.db.models.expressions.F("start_date")) ), name="chronology", diff --git a/bookwyrm/migrations/0107_auto_20211016_0639.py b/bookwyrm/migrations/0107_auto_20211016_0639.py index 61dffca348..bc27d3d804 100644 --- a/bookwyrm/migrations/0107_auto_20211016_0639.py +++ b/bookwyrm/migrations/0107_auto_20211016_0639.py @@ -767,7 +767,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="notification", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( ( "notification_type__in", [ diff --git a/bookwyrm/migrations/0112_auto_20211022_0844.py b/bookwyrm/migrations/0112_auto_20211022_0844.py index 246480b3a9..43e9d806cc 100644 --- a/bookwyrm/migrations/0112_auto_20211022_0844.py +++ b/bookwyrm/migrations/0112_auto_20211022_0844.py @@ -62,7 +62,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="notification", constraint=models.CheckConstraint( - check=models.Q( + condition=models.Q( ( "notification_type__in", [ diff --git a/bookwyrm/migrations/0189_importjob_create_shelves.py b/bookwyrm/migrations/0189_importjob_create_shelves.py new file mode 100644 index 0000000000..a1b1fc5128 --- /dev/null +++ b/bookwyrm/migrations/0189_importjob_create_shelves.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2023-11-25 05:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0188_theme_loads"), + ] + + operations = [ + migrations.AddField( + model_name="importjob", + name="create_shelves", + field=models.BooleanField(default=True), + ), + ] diff --git a/bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py b/bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py index 4fe41ec357..859cd50595 100644 --- a/bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py +++ b/bookwyrm/migrations/0201_alter_hashtag_name_alter_user_localname.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - dependencies = [ ("bookwyrm", "0200_alter_user_preferred_timezone"), ] @@ -25,6 +24,16 @@ class Migration(migrations.Migration): db_collation="case_insensitive", max_length=256 ), ), + migrations.AlterField( + model_name="user", + name="localname", + field=models.CharField( + max_length=255, + null=True, + unique=False, + validators=[bookwyrm.models.fields.validate_localname], + ), + ), migrations.AlterField( model_name="user", name="localname", diff --git a/bookwyrm/migrations/0207_merge_20240629_0626.py b/bookwyrm/migrations/0207_merge_20240629_0626.py new file mode 100644 index 0000000000..b5a1a45560 --- /dev/null +++ b/bookwyrm/migrations/0207_merge_20240629_0626.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.11 on 2024-06-29 06:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0189_importjob_create_shelves"), + ("bookwyrm", "0206_merge_20240415_1537"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0207_sqlparse_update.py b/bookwyrm/migrations/0207_sqlparse_update.py new file mode 100644 index 0000000000..95c46eba2f --- /dev/null +++ b/bookwyrm/migrations/0207_sqlparse_update.py @@ -0,0 +1,51 @@ +# Generated by Django 4.2.11 on 2024-07-27 18:18 + +from django.db import migrations, models +import pgtrigger.compiler +import pgtrigger.migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0206_merge_20240415_1537"), + ] + + operations = [ + pgtrigger.migrations.RemoveTrigger( + model_name="author", + name="reset_book_search_vector_on_author_edit", + ), + pgtrigger.migrations.RemoveTrigger( + model_name="book", + name="update_search_vector_on_book_edit", + ), + pgtrigger.migrations.AddTrigger( + model_name="author", + trigger=pgtrigger.compiler.Trigger( + name="reset_book_search_vector_on_author_edit", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;", + hash="4eeb17d1c9c53f543615bcae1234bd0260adefcc", + operation='UPDATE OF "name", "aliases"', + pgid="pgtrigger_reset_book_search_vector_on_author_edit_a50c7", + table="bookwyrm_author", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="book", + trigger=pgtrigger.compiler.Trigger( + name="update_search_vector_on_book_edit", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="WITH author_names AS (SELECT array_to_string(bookwyrm_author.name || bookwyrm_author.aliases, ' ') AS name_and_aliases FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id) SELECT setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(name_and_aliases), ' '), '')), 'C') FROM author_names) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D') INTO new.search_vector;RETURN NEW;", + hash="676d929ce95beff671544b6add09cf9360b6f299", + operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"', + pgid="pgtrigger_update_search_vector_on_book_edit_bec58", + table="bookwyrm_book", + when="BEFORE", + ), + ), + ), + ] diff --git a/bookwyrm/migrations/0208_merge_0207_merge_20240629_0626_0207_sqlparse_update.py b/bookwyrm/migrations/0208_merge_0207_merge_20240629_0626_0207_sqlparse_update.py new file mode 100644 index 0000000000..24ef28e047 --- /dev/null +++ b/bookwyrm/migrations/0208_merge_0207_merge_20240629_0626_0207_sqlparse_update.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.11 on 2024-07-28 11:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0207_merge_20240629_0626"), + ("bookwyrm", "0207_sqlparse_update"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0209_user_show_ratings.py b/bookwyrm/migrations/0209_user_show_ratings.py new file mode 100644 index 0000000000..94e074407e --- /dev/null +++ b/bookwyrm/migrations/0209_user_show_ratings.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.15 on 2024-08-24 01:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0208_merge_0207_merge_20240629_0626_0207_sqlparse_update"), + ] + + operations = [ + migrations.AddField( + model_name="user", + name="show_ratings", + field=models.BooleanField(default=True), + ), + ] diff --git a/bookwyrm/migrations/0210_alter_connector_connector_file.py b/bookwyrm/migrations/0210_alter_connector_connector_file.py new file mode 100644 index 0000000000..8811378d7d --- /dev/null +++ b/bookwyrm/migrations/0210_alter_connector_connector_file.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.17 on 2025-02-02 20:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0209_user_show_ratings"), + ] + + operations = [ + migrations.AlterField( + model_name="connector", + name="connector_file", + field=models.CharField( + choices=[ + ("openlibrary", "Openlibrary"), + ("inventaire", "Inventaire"), + ("bookwyrm_connector", "Bookwyrm Connector"), + ("finna", "Finna"), + ], + max_length=255, + ), + ), + ] diff --git a/bookwyrm/migrations/0211_author_finna_key_book_finna_key.py b/bookwyrm/migrations/0211_author_finna_key_book_finna_key.py new file mode 100644 index 0000000000..78483345d5 --- /dev/null +++ b/bookwyrm/migrations/0211_author_finna_key_book_finna_key.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.17 on 2025-02-08 16:14 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0210_alter_connector_connector_file"), + ] + + operations = [ + migrations.AddField( + model_name="author", + name="finna_key", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), + ), + migrations.AddField( + model_name="book", + name="finna_key", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), + ), + ] diff --git a/bookwyrm/migrations/0212_userrelationshipimport_and_more.py b/bookwyrm/migrations/0212_userrelationshipimport_and_more.py new file mode 100644 index 0000000000..ef088057f2 --- /dev/null +++ b/bookwyrm/migrations/0212_userrelationshipimport_and_more.py @@ -0,0 +1,151 @@ +# Generated by Django 4.2.20 on 2025-03-28 07:37 + +import bookwyrm.models.fields +import django.contrib.postgres.fields +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0211_author_finna_key_book_finna_key"), + ] + + operations = [ + migrations.CreateModel( + name="UserRelationshipImport", + fields=[ + ( + "childjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.childjob", + ), + ), + ( + "relationship", + bookwyrm.models.fields.CharField( + choices=[("follow", "Follow"), ("block", "Block")], + max_length=10, + null=True, + ), + ), + ( + "remote_id", + bookwyrm.models.fields.RemoteIdField( + max_length=255, + null=True, + validators=[bookwyrm.models.fields.validate_remote_id], + ), + ), + ], + options={ + "abstract": False, + }, + bases=("bookwyrm.childjob",), + ), + migrations.RemoveField( + model_name="bookwyrmexportjob", + name="json_completed", + ), + migrations.AddField( + model_name="bookwyrmimportjob", + name="retry", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="childjob", + name="fail_reason", + field=models.TextField(null=True), + ), + migrations.AddField( + model_name="parentjob", + name="fail_reason", + field=models.TextField(null=True), + ), + migrations.AlterField( + model_name="bookwyrmimportjob", + name="required", + field=django.contrib.postgres.fields.ArrayField( + base_field=bookwyrm.models.fields.CharField(blank=True, max_length=50), + blank=True, + size=None, + ), + ), + migrations.CreateModel( + name="UserImportPost", + fields=[ + ( + "childjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.childjob", + ), + ), + ("json", models.JSONField()), + ( + "status_type", + bookwyrm.models.fields.CharField( + choices=[ + ("comment", "Comment"), + ("review", "Review"), + ("quote", "Quotation"), + ], + default="comment", + max_length=10, + null=True, + ), + ), + ( + "book", + bookwyrm.models.fields.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="bookwyrm.edition", + ), + ), + ], + options={ + "abstract": False, + }, + bases=("bookwyrm.childjob",), + ), + migrations.CreateModel( + name="UserImportBook", + fields=[ + ( + "childjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.childjob", + ), + ), + ("book_data", models.JSONField()), + ( + "book", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="bookwyrm.book", + ), + ), + ], + options={ + "abstract": False, + }, + bases=("bookwyrm.childjob",), + ), + ] diff --git a/bookwyrm/migrations/0213_alter_user_preferred_timezone.py b/bookwyrm/migrations/0213_alter_user_preferred_timezone.py new file mode 100644 index 0000000000..93048c1067 --- /dev/null +++ b/bookwyrm/migrations/0213_alter_user_preferred_timezone.py @@ -0,0 +1,634 @@ +# Generated by Django 4.2.20 on 2025-03-31 15:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0212_userrelationshipimport_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="preferred_timezone", + field=models.CharField( + choices=[ + ("Africa/Abidjan", "Africa/Abidjan"), + ("Africa/Accra", "Africa/Accra"), + ("Africa/Addis_Ababa", "Africa/Addis_Ababa"), + ("Africa/Algiers", "Africa/Algiers"), + ("Africa/Asmara", "Africa/Asmara"), + ("Africa/Asmera", "Africa/Asmera"), + ("Africa/Bamako", "Africa/Bamako"), + ("Africa/Bangui", "Africa/Bangui"), + ("Africa/Banjul", "Africa/Banjul"), + ("Africa/Bissau", "Africa/Bissau"), + ("Africa/Blantyre", "Africa/Blantyre"), + ("Africa/Brazzaville", "Africa/Brazzaville"), + ("Africa/Bujumbura", "Africa/Bujumbura"), + ("Africa/Cairo", "Africa/Cairo"), + ("Africa/Casablanca", "Africa/Casablanca"), + ("Africa/Ceuta", "Africa/Ceuta"), + ("Africa/Conakry", "Africa/Conakry"), + ("Africa/Dakar", "Africa/Dakar"), + ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"), + ("Africa/Djibouti", "Africa/Djibouti"), + ("Africa/Douala", "Africa/Douala"), + ("Africa/El_Aaiun", "Africa/El_Aaiun"), + ("Africa/Freetown", "Africa/Freetown"), + ("Africa/Gaborone", "Africa/Gaborone"), + ("Africa/Harare", "Africa/Harare"), + ("Africa/Johannesburg", "Africa/Johannesburg"), + ("Africa/Juba", "Africa/Juba"), + ("Africa/Kampala", "Africa/Kampala"), + ("Africa/Khartoum", "Africa/Khartoum"), + ("Africa/Kigali", "Africa/Kigali"), + ("Africa/Kinshasa", "Africa/Kinshasa"), + ("Africa/Lagos", "Africa/Lagos"), + ("Africa/Libreville", "Africa/Libreville"), + ("Africa/Lome", "Africa/Lome"), + ("Africa/Luanda", "Africa/Luanda"), + ("Africa/Lubumbashi", "Africa/Lubumbashi"), + ("Africa/Lusaka", "Africa/Lusaka"), + ("Africa/Malabo", "Africa/Malabo"), + ("Africa/Maputo", "Africa/Maputo"), + ("Africa/Maseru", "Africa/Maseru"), + ("Africa/Mbabane", "Africa/Mbabane"), + ("Africa/Mogadishu", "Africa/Mogadishu"), + ("Africa/Monrovia", "Africa/Monrovia"), + ("Africa/Nairobi", "Africa/Nairobi"), + ("Africa/Ndjamena", "Africa/Ndjamena"), + ("Africa/Niamey", "Africa/Niamey"), + ("Africa/Nouakchott", "Africa/Nouakchott"), + ("Africa/Ouagadougou", "Africa/Ouagadougou"), + ("Africa/Porto-Novo", "Africa/Porto-Novo"), + ("Africa/Sao_Tome", "Africa/Sao_Tome"), + ("Africa/Timbuktu", "Africa/Timbuktu"), + ("Africa/Tripoli", "Africa/Tripoli"), + ("Africa/Tunis", "Africa/Tunis"), + ("Africa/Windhoek", "Africa/Windhoek"), + ("America/Adak", "America/Adak"), + ("America/Anchorage", "America/Anchorage"), + ("America/Anguilla", "America/Anguilla"), + ("America/Antigua", "America/Antigua"), + ("America/Araguaina", "America/Araguaina"), + ( + "America/Argentina/Buenos_Aires", + "America/Argentina/Buenos_Aires", + ), + ("America/Argentina/Catamarca", "America/Argentina/Catamarca"), + ( + "America/Argentina/ComodRivadavia", + "America/Argentina/ComodRivadavia", + ), + ("America/Argentina/Cordoba", "America/Argentina/Cordoba"), + ("America/Argentina/Jujuy", "America/Argentina/Jujuy"), + ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"), + ("America/Argentina/Mendoza", "America/Argentina/Mendoza"), + ( + "America/Argentina/Rio_Gallegos", + "America/Argentina/Rio_Gallegos", + ), + ("America/Argentina/Salta", "America/Argentina/Salta"), + ("America/Argentina/San_Juan", "America/Argentina/San_Juan"), + ("America/Argentina/San_Luis", "America/Argentina/San_Luis"), + ("America/Argentina/Tucuman", "America/Argentina/Tucuman"), + ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"), + ("America/Aruba", "America/Aruba"), + ("America/Asuncion", "America/Asuncion"), + ("America/Atikokan", "America/Atikokan"), + ("America/Atka", "America/Atka"), + ("America/Bahia", "America/Bahia"), + ("America/Bahia_Banderas", "America/Bahia_Banderas"), + ("America/Barbados", "America/Barbados"), + ("America/Belem", "America/Belem"), + ("America/Belize", "America/Belize"), + ("America/Blanc-Sablon", "America/Blanc-Sablon"), + ("America/Boa_Vista", "America/Boa_Vista"), + ("America/Bogota", "America/Bogota"), + ("America/Boise", "America/Boise"), + ("America/Buenos_Aires", "America/Buenos_Aires"), + ("America/Cambridge_Bay", "America/Cambridge_Bay"), + ("America/Campo_Grande", "America/Campo_Grande"), + ("America/Cancun", "America/Cancun"), + ("America/Caracas", "America/Caracas"), + ("America/Catamarca", "America/Catamarca"), + ("America/Cayenne", "America/Cayenne"), + ("America/Cayman", "America/Cayman"), + ("America/Chicago", "America/Chicago"), + ("America/Chihuahua", "America/Chihuahua"), + ("America/Ciudad_Juarez", "America/Ciudad_Juarez"), + ("America/Coral_Harbour", "America/Coral_Harbour"), + ("America/Cordoba", "America/Cordoba"), + ("America/Costa_Rica", "America/Costa_Rica"), + ("America/Coyhaique", "America/Coyhaique"), + ("America/Creston", "America/Creston"), + ("America/Cuiaba", "America/Cuiaba"), + ("America/Curacao", "America/Curacao"), + ("America/Danmarkshavn", "America/Danmarkshavn"), + ("America/Dawson", "America/Dawson"), + ("America/Dawson_Creek", "America/Dawson_Creek"), + ("America/Denver", "America/Denver"), + ("America/Detroit", "America/Detroit"), + ("America/Dominica", "America/Dominica"), + ("America/Edmonton", "America/Edmonton"), + ("America/Eirunepe", "America/Eirunepe"), + ("America/El_Salvador", "America/El_Salvador"), + ("America/Ensenada", "America/Ensenada"), + ("America/Fort_Nelson", "America/Fort_Nelson"), + ("America/Fort_Wayne", "America/Fort_Wayne"), + ("America/Fortaleza", "America/Fortaleza"), + ("America/Glace_Bay", "America/Glace_Bay"), + ("America/Godthab", "America/Godthab"), + ("America/Goose_Bay", "America/Goose_Bay"), + ("America/Grand_Turk", "America/Grand_Turk"), + ("America/Grenada", "America/Grenada"), + ("America/Guadeloupe", "America/Guadeloupe"), + ("America/Guatemala", "America/Guatemala"), + ("America/Guayaquil", "America/Guayaquil"), + ("America/Guyana", "America/Guyana"), + ("America/Halifax", "America/Halifax"), + ("America/Havana", "America/Havana"), + ("America/Hermosillo", "America/Hermosillo"), + ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"), + ("America/Indiana/Knox", "America/Indiana/Knox"), + ("America/Indiana/Marengo", "America/Indiana/Marengo"), + ("America/Indiana/Petersburg", "America/Indiana/Petersburg"), + ("America/Indiana/Tell_City", "America/Indiana/Tell_City"), + ("America/Indiana/Vevay", "America/Indiana/Vevay"), + ("America/Indiana/Vincennes", "America/Indiana/Vincennes"), + ("America/Indiana/Winamac", "America/Indiana/Winamac"), + ("America/Indianapolis", "America/Indianapolis"), + ("America/Inuvik", "America/Inuvik"), + ("America/Iqaluit", "America/Iqaluit"), + ("America/Jamaica", "America/Jamaica"), + ("America/Jujuy", "America/Jujuy"), + ("America/Juneau", "America/Juneau"), + ("America/Kentucky/Louisville", "America/Kentucky/Louisville"), + ("America/Kentucky/Monticello", "America/Kentucky/Monticello"), + ("America/Knox_IN", "America/Knox_IN"), + ("America/Kralendijk", "America/Kralendijk"), + ("America/La_Paz", "America/La_Paz"), + ("America/Lima", "America/Lima"), + ("America/Los_Angeles", "America/Los_Angeles"), + ("America/Louisville", "America/Louisville"), + ("America/Lower_Princes", "America/Lower_Princes"), + ("America/Maceio", "America/Maceio"), + ("America/Managua", "America/Managua"), + ("America/Manaus", "America/Manaus"), + ("America/Marigot", "America/Marigot"), + ("America/Martinique", "America/Martinique"), + ("America/Matamoros", "America/Matamoros"), + ("America/Mazatlan", "America/Mazatlan"), + ("America/Mendoza", "America/Mendoza"), + ("America/Menominee", "America/Menominee"), + ("America/Merida", "America/Merida"), + ("America/Metlakatla", "America/Metlakatla"), + ("America/Mexico_City", "America/Mexico_City"), + ("America/Miquelon", "America/Miquelon"), + ("America/Moncton", "America/Moncton"), + ("America/Monterrey", "America/Monterrey"), + ("America/Montevideo", "America/Montevideo"), + ("America/Montreal", "America/Montreal"), + ("America/Montserrat", "America/Montserrat"), + ("America/Nassau", "America/Nassau"), + ("America/New_York", "America/New_York"), + ("America/Nipigon", "America/Nipigon"), + ("America/Nome", "America/Nome"), + ("America/Noronha", "America/Noronha"), + ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"), + ("America/North_Dakota/Center", "America/North_Dakota/Center"), + ( + "America/North_Dakota/New_Salem", + "America/North_Dakota/New_Salem", + ), + ("America/Nuuk", "America/Nuuk"), + ("America/Ojinaga", "America/Ojinaga"), + ("America/Panama", "America/Panama"), + ("America/Pangnirtung", "America/Pangnirtung"), + ("America/Paramaribo", "America/Paramaribo"), + ("America/Phoenix", "America/Phoenix"), + ("America/Port-au-Prince", "America/Port-au-Prince"), + ("America/Port_of_Spain", "America/Port_of_Spain"), + ("America/Porto_Acre", "America/Porto_Acre"), + ("America/Porto_Velho", "America/Porto_Velho"), + ("America/Puerto_Rico", "America/Puerto_Rico"), + ("America/Punta_Arenas", "America/Punta_Arenas"), + ("America/Rainy_River", "America/Rainy_River"), + ("America/Rankin_Inlet", "America/Rankin_Inlet"), + ("America/Recife", "America/Recife"), + ("America/Regina", "America/Regina"), + ("America/Resolute", "America/Resolute"), + ("America/Rio_Branco", "America/Rio_Branco"), + ("America/Rosario", "America/Rosario"), + ("America/Santa_Isabel", "America/Santa_Isabel"), + ("America/Santarem", "America/Santarem"), + ("America/Santiago", "America/Santiago"), + ("America/Santo_Domingo", "America/Santo_Domingo"), + ("America/Sao_Paulo", "America/Sao_Paulo"), + ("America/Scoresbysund", "America/Scoresbysund"), + ("America/Shiprock", "America/Shiprock"), + ("America/Sitka", "America/Sitka"), + ("America/St_Barthelemy", "America/St_Barthelemy"), + ("America/St_Johns", "America/St_Johns"), + ("America/St_Kitts", "America/St_Kitts"), + ("America/St_Lucia", "America/St_Lucia"), + ("America/St_Thomas", "America/St_Thomas"), + ("America/St_Vincent", "America/St_Vincent"), + ("America/Swift_Current", "America/Swift_Current"), + ("America/Tegucigalpa", "America/Tegucigalpa"), + ("America/Thule", "America/Thule"), + ("America/Thunder_Bay", "America/Thunder_Bay"), + ("America/Tijuana", "America/Tijuana"), + ("America/Toronto", "America/Toronto"), + ("America/Tortola", "America/Tortola"), + ("America/Vancouver", "America/Vancouver"), + ("America/Virgin", "America/Virgin"), + ("America/Whitehorse", "America/Whitehorse"), + ("America/Winnipeg", "America/Winnipeg"), + ("America/Yakutat", "America/Yakutat"), + ("America/Yellowknife", "America/Yellowknife"), + ("Antarctica/Casey", "Antarctica/Casey"), + ("Antarctica/Davis", "Antarctica/Davis"), + ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"), + ("Antarctica/Macquarie", "Antarctica/Macquarie"), + ("Antarctica/Mawson", "Antarctica/Mawson"), + ("Antarctica/McMurdo", "Antarctica/McMurdo"), + ("Antarctica/Palmer", "Antarctica/Palmer"), + ("Antarctica/Rothera", "Antarctica/Rothera"), + ("Antarctica/South_Pole", "Antarctica/South_Pole"), + ("Antarctica/Syowa", "Antarctica/Syowa"), + ("Antarctica/Troll", "Antarctica/Troll"), + ("Antarctica/Vostok", "Antarctica/Vostok"), + ("Arctic/Longyearbyen", "Arctic/Longyearbyen"), + ("Asia/Aden", "Asia/Aden"), + ("Asia/Almaty", "Asia/Almaty"), + ("Asia/Amman", "Asia/Amman"), + ("Asia/Anadyr", "Asia/Anadyr"), + ("Asia/Aqtau", "Asia/Aqtau"), + ("Asia/Aqtobe", "Asia/Aqtobe"), + ("Asia/Ashgabat", "Asia/Ashgabat"), + ("Asia/Ashkhabad", "Asia/Ashkhabad"), + ("Asia/Atyrau", "Asia/Atyrau"), + ("Asia/Baghdad", "Asia/Baghdad"), + ("Asia/Bahrain", "Asia/Bahrain"), + ("Asia/Baku", "Asia/Baku"), + ("Asia/Bangkok", "Asia/Bangkok"), + ("Asia/Barnaul", "Asia/Barnaul"), + ("Asia/Beirut", "Asia/Beirut"), + ("Asia/Bishkek", "Asia/Bishkek"), + ("Asia/Brunei", "Asia/Brunei"), + ("Asia/Calcutta", "Asia/Calcutta"), + ("Asia/Chita", "Asia/Chita"), + ("Asia/Choibalsan", "Asia/Choibalsan"), + ("Asia/Chongqing", "Asia/Chongqing"), + ("Asia/Chungking", "Asia/Chungking"), + ("Asia/Colombo", "Asia/Colombo"), + ("Asia/Dacca", "Asia/Dacca"), + ("Asia/Damascus", "Asia/Damascus"), + ("Asia/Dhaka", "Asia/Dhaka"), + ("Asia/Dili", "Asia/Dili"), + ("Asia/Dubai", "Asia/Dubai"), + ("Asia/Dushanbe", "Asia/Dushanbe"), + ("Asia/Famagusta", "Asia/Famagusta"), + ("Asia/Gaza", "Asia/Gaza"), + ("Asia/Harbin", "Asia/Harbin"), + ("Asia/Hebron", "Asia/Hebron"), + ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"), + ("Asia/Hong_Kong", "Asia/Hong_Kong"), + ("Asia/Hovd", "Asia/Hovd"), + ("Asia/Irkutsk", "Asia/Irkutsk"), + ("Asia/Istanbul", "Asia/Istanbul"), + ("Asia/Jakarta", "Asia/Jakarta"), + ("Asia/Jayapura", "Asia/Jayapura"), + ("Asia/Jerusalem", "Asia/Jerusalem"), + ("Asia/Kabul", "Asia/Kabul"), + ("Asia/Kamchatka", "Asia/Kamchatka"), + ("Asia/Karachi", "Asia/Karachi"), + ("Asia/Kashgar", "Asia/Kashgar"), + ("Asia/Kathmandu", "Asia/Kathmandu"), + ("Asia/Katmandu", "Asia/Katmandu"), + ("Asia/Khandyga", "Asia/Khandyga"), + ("Asia/Kolkata", "Asia/Kolkata"), + ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"), + ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"), + ("Asia/Kuching", "Asia/Kuching"), + ("Asia/Kuwait", "Asia/Kuwait"), + ("Asia/Macao", "Asia/Macao"), + ("Asia/Macau", "Asia/Macau"), + ("Asia/Magadan", "Asia/Magadan"), + ("Asia/Makassar", "Asia/Makassar"), + ("Asia/Manila", "Asia/Manila"), + ("Asia/Muscat", "Asia/Muscat"), + ("Asia/Nicosia", "Asia/Nicosia"), + ("Asia/Novokuznetsk", "Asia/Novokuznetsk"), + ("Asia/Novosibirsk", "Asia/Novosibirsk"), + ("Asia/Omsk", "Asia/Omsk"), + ("Asia/Oral", "Asia/Oral"), + ("Asia/Phnom_Penh", "Asia/Phnom_Penh"), + ("Asia/Pontianak", "Asia/Pontianak"), + ("Asia/Pyongyang", "Asia/Pyongyang"), + ("Asia/Qatar", "Asia/Qatar"), + ("Asia/Qostanay", "Asia/Qostanay"), + ("Asia/Qyzylorda", "Asia/Qyzylorda"), + ("Asia/Rangoon", "Asia/Rangoon"), + ("Asia/Riyadh", "Asia/Riyadh"), + ("Asia/Saigon", "Asia/Saigon"), + ("Asia/Sakhalin", "Asia/Sakhalin"), + ("Asia/Samarkand", "Asia/Samarkand"), + ("Asia/Seoul", "Asia/Seoul"), + ("Asia/Shanghai", "Asia/Shanghai"), + ("Asia/Singapore", "Asia/Singapore"), + ("Asia/Srednekolymsk", "Asia/Srednekolymsk"), + ("Asia/Taipei", "Asia/Taipei"), + ("Asia/Tashkent", "Asia/Tashkent"), + ("Asia/Tbilisi", "Asia/Tbilisi"), + ("Asia/Tehran", "Asia/Tehran"), + ("Asia/Tel_Aviv", "Asia/Tel_Aviv"), + ("Asia/Thimbu", "Asia/Thimbu"), + ("Asia/Thimphu", "Asia/Thimphu"), + ("Asia/Tokyo", "Asia/Tokyo"), + ("Asia/Tomsk", "Asia/Tomsk"), + ("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"), + ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"), + ("Asia/Ulan_Bator", "Asia/Ulan_Bator"), + ("Asia/Urumqi", "Asia/Urumqi"), + ("Asia/Ust-Nera", "Asia/Ust-Nera"), + ("Asia/Vientiane", "Asia/Vientiane"), + ("Asia/Vladivostok", "Asia/Vladivostok"), + ("Asia/Yakutsk", "Asia/Yakutsk"), + ("Asia/Yangon", "Asia/Yangon"), + ("Asia/Yekaterinburg", "Asia/Yekaterinburg"), + ("Asia/Yerevan", "Asia/Yerevan"), + ("Atlantic/Azores", "Atlantic/Azores"), + ("Atlantic/Bermuda", "Atlantic/Bermuda"), + ("Atlantic/Canary", "Atlantic/Canary"), + ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"), + ("Atlantic/Faeroe", "Atlantic/Faeroe"), + ("Atlantic/Faroe", "Atlantic/Faroe"), + ("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"), + ("Atlantic/Madeira", "Atlantic/Madeira"), + ("Atlantic/Reykjavik", "Atlantic/Reykjavik"), + ("Atlantic/South_Georgia", "Atlantic/South_Georgia"), + ("Atlantic/St_Helena", "Atlantic/St_Helena"), + ("Atlantic/Stanley", "Atlantic/Stanley"), + ("Australia/ACT", "Australia/ACT"), + ("Australia/Adelaide", "Australia/Adelaide"), + ("Australia/Brisbane", "Australia/Brisbane"), + ("Australia/Broken_Hill", "Australia/Broken_Hill"), + ("Australia/Canberra", "Australia/Canberra"), + ("Australia/Currie", "Australia/Currie"), + ("Australia/Darwin", "Australia/Darwin"), + ("Australia/Eucla", "Australia/Eucla"), + ("Australia/Hobart", "Australia/Hobart"), + ("Australia/LHI", "Australia/LHI"), + ("Australia/Lindeman", "Australia/Lindeman"), + ("Australia/Lord_Howe", "Australia/Lord_Howe"), + ("Australia/Melbourne", "Australia/Melbourne"), + ("Australia/NSW", "Australia/NSW"), + ("Australia/North", "Australia/North"), + ("Australia/Perth", "Australia/Perth"), + ("Australia/Queensland", "Australia/Queensland"), + ("Australia/South", "Australia/South"), + ("Australia/Sydney", "Australia/Sydney"), + ("Australia/Tasmania", "Australia/Tasmania"), + ("Australia/Victoria", "Australia/Victoria"), + ("Australia/West", "Australia/West"), + ("Australia/Yancowinna", "Australia/Yancowinna"), + ("Brazil/Acre", "Brazil/Acre"), + ("Brazil/DeNoronha", "Brazil/DeNoronha"), + ("Brazil/East", "Brazil/East"), + ("Brazil/West", "Brazil/West"), + ("CET", "CET"), + ("CST6CDT", "CST6CDT"), + ("Canada/Atlantic", "Canada/Atlantic"), + ("Canada/Central", "Canada/Central"), + ("Canada/Eastern", "Canada/Eastern"), + ("Canada/Mountain", "Canada/Mountain"), + ("Canada/Newfoundland", "Canada/Newfoundland"), + ("Canada/Pacific", "Canada/Pacific"), + ("Canada/Saskatchewan", "Canada/Saskatchewan"), + ("Canada/Yukon", "Canada/Yukon"), + ("Chile/Continental", "Chile/Continental"), + ("Chile/EasterIsland", "Chile/EasterIsland"), + ("Cuba", "Cuba"), + ("EET", "EET"), + ("EST", "EST"), + ("EST5EDT", "EST5EDT"), + ("Egypt", "Egypt"), + ("Eire", "Eire"), + ("Etc/GMT", "Etc/GMT"), + ("Etc/GMT+0", "Etc/GMT+0"), + ("Etc/GMT+1", "Etc/GMT+1"), + ("Etc/GMT+10", "Etc/GMT+10"), + ("Etc/GMT+11", "Etc/GMT+11"), + ("Etc/GMT+12", "Etc/GMT+12"), + ("Etc/GMT+2", "Etc/GMT+2"), + ("Etc/GMT+3", "Etc/GMT+3"), + ("Etc/GMT+4", "Etc/GMT+4"), + ("Etc/GMT+5", "Etc/GMT+5"), + ("Etc/GMT+6", "Etc/GMT+6"), + ("Etc/GMT+7", "Etc/GMT+7"), + ("Etc/GMT+8", "Etc/GMT+8"), + ("Etc/GMT+9", "Etc/GMT+9"), + ("Etc/GMT-0", "Etc/GMT-0"), + ("Etc/GMT-1", "Etc/GMT-1"), + ("Etc/GMT-10", "Etc/GMT-10"), + ("Etc/GMT-11", "Etc/GMT-11"), + ("Etc/GMT-12", "Etc/GMT-12"), + ("Etc/GMT-13", "Etc/GMT-13"), + ("Etc/GMT-14", "Etc/GMT-14"), + ("Etc/GMT-2", "Etc/GMT-2"), + ("Etc/GMT-3", "Etc/GMT-3"), + ("Etc/GMT-4", "Etc/GMT-4"), + ("Etc/GMT-5", "Etc/GMT-5"), + ("Etc/GMT-6", "Etc/GMT-6"), + ("Etc/GMT-7", "Etc/GMT-7"), + ("Etc/GMT-8", "Etc/GMT-8"), + ("Etc/GMT-9", "Etc/GMT-9"), + ("Etc/GMT0", "Etc/GMT0"), + ("Etc/Greenwich", "Etc/Greenwich"), + ("Etc/UCT", "Etc/UCT"), + ("Etc/UTC", "Etc/UTC"), + ("Etc/Universal", "Etc/Universal"), + ("Etc/Zulu", "Etc/Zulu"), + ("Europe/Amsterdam", "Europe/Amsterdam"), + ("Europe/Andorra", "Europe/Andorra"), + ("Europe/Astrakhan", "Europe/Astrakhan"), + ("Europe/Athens", "Europe/Athens"), + ("Europe/Belfast", "Europe/Belfast"), + ("Europe/Belgrade", "Europe/Belgrade"), + ("Europe/Berlin", "Europe/Berlin"), + ("Europe/Bratislava", "Europe/Bratislava"), + ("Europe/Brussels", "Europe/Brussels"), + ("Europe/Bucharest", "Europe/Bucharest"), + ("Europe/Budapest", "Europe/Budapest"), + ("Europe/Busingen", "Europe/Busingen"), + ("Europe/Chisinau", "Europe/Chisinau"), + ("Europe/Copenhagen", "Europe/Copenhagen"), + ("Europe/Dublin", "Europe/Dublin"), + ("Europe/Gibraltar", "Europe/Gibraltar"), + ("Europe/Guernsey", "Europe/Guernsey"), + ("Europe/Helsinki", "Europe/Helsinki"), + ("Europe/Isle_of_Man", "Europe/Isle_of_Man"), + ("Europe/Istanbul", "Europe/Istanbul"), + ("Europe/Jersey", "Europe/Jersey"), + ("Europe/Kaliningrad", "Europe/Kaliningrad"), + ("Europe/Kiev", "Europe/Kiev"), + ("Europe/Kirov", "Europe/Kirov"), + ("Europe/Kyiv", "Europe/Kyiv"), + ("Europe/Lisbon", "Europe/Lisbon"), + ("Europe/Ljubljana", "Europe/Ljubljana"), + ("Europe/London", "Europe/London"), + ("Europe/Luxembourg", "Europe/Luxembourg"), + ("Europe/Madrid", "Europe/Madrid"), + ("Europe/Malta", "Europe/Malta"), + ("Europe/Mariehamn", "Europe/Mariehamn"), + ("Europe/Minsk", "Europe/Minsk"), + ("Europe/Monaco", "Europe/Monaco"), + ("Europe/Moscow", "Europe/Moscow"), + ("Europe/Nicosia", "Europe/Nicosia"), + ("Europe/Oslo", "Europe/Oslo"), + ("Europe/Paris", "Europe/Paris"), + ("Europe/Podgorica", "Europe/Podgorica"), + ("Europe/Prague", "Europe/Prague"), + ("Europe/Riga", "Europe/Riga"), + ("Europe/Rome", "Europe/Rome"), + ("Europe/Samara", "Europe/Samara"), + ("Europe/San_Marino", "Europe/San_Marino"), + ("Europe/Sarajevo", "Europe/Sarajevo"), + ("Europe/Saratov", "Europe/Saratov"), + ("Europe/Simferopol", "Europe/Simferopol"), + ("Europe/Skopje", "Europe/Skopje"), + ("Europe/Sofia", "Europe/Sofia"), + ("Europe/Stockholm", "Europe/Stockholm"), + ("Europe/Tallinn", "Europe/Tallinn"), + ("Europe/Tirane", "Europe/Tirane"), + ("Europe/Tiraspol", "Europe/Tiraspol"), + ("Europe/Ulyanovsk", "Europe/Ulyanovsk"), + ("Europe/Uzhgorod", "Europe/Uzhgorod"), + ("Europe/Vaduz", "Europe/Vaduz"), + ("Europe/Vatican", "Europe/Vatican"), + ("Europe/Vienna", "Europe/Vienna"), + ("Europe/Vilnius", "Europe/Vilnius"), + ("Europe/Volgograd", "Europe/Volgograd"), + ("Europe/Warsaw", "Europe/Warsaw"), + ("Europe/Zagreb", "Europe/Zagreb"), + ("Europe/Zaporozhye", "Europe/Zaporozhye"), + ("Europe/Zurich", "Europe/Zurich"), + ("Factory", "Factory"), + ("GB", "GB"), + ("GB-Eire", "GB-Eire"), + ("GMT", "GMT"), + ("GMT+0", "GMT+0"), + ("GMT-0", "GMT-0"), + ("GMT0", "GMT0"), + ("Greenwich", "Greenwich"), + ("HST", "HST"), + ("Hongkong", "Hongkong"), + ("Iceland", "Iceland"), + ("Indian/Antananarivo", "Indian/Antananarivo"), + ("Indian/Chagos", "Indian/Chagos"), + ("Indian/Christmas", "Indian/Christmas"), + ("Indian/Cocos", "Indian/Cocos"), + ("Indian/Comoro", "Indian/Comoro"), + ("Indian/Kerguelen", "Indian/Kerguelen"), + ("Indian/Mahe", "Indian/Mahe"), + ("Indian/Maldives", "Indian/Maldives"), + ("Indian/Mauritius", "Indian/Mauritius"), + ("Indian/Mayotte", "Indian/Mayotte"), + ("Indian/Reunion", "Indian/Reunion"), + ("Iran", "Iran"), + ("Israel", "Israel"), + ("Jamaica", "Jamaica"), + ("Japan", "Japan"), + ("Kwajalein", "Kwajalein"), + ("Libya", "Libya"), + ("MET", "MET"), + ("MST", "MST"), + ("MST7MDT", "MST7MDT"), + ("Mexico/BajaNorte", "Mexico/BajaNorte"), + ("Mexico/BajaSur", "Mexico/BajaSur"), + ("Mexico/General", "Mexico/General"), + ("NZ", "NZ"), + ("NZ-CHAT", "NZ-CHAT"), + ("Navajo", "Navajo"), + ("PRC", "PRC"), + ("PST8PDT", "PST8PDT"), + ("Pacific/Apia", "Pacific/Apia"), + ("Pacific/Auckland", "Pacific/Auckland"), + ("Pacific/Bougainville", "Pacific/Bougainville"), + ("Pacific/Chatham", "Pacific/Chatham"), + ("Pacific/Chuuk", "Pacific/Chuuk"), + ("Pacific/Easter", "Pacific/Easter"), + ("Pacific/Efate", "Pacific/Efate"), + ("Pacific/Enderbury", "Pacific/Enderbury"), + ("Pacific/Fakaofo", "Pacific/Fakaofo"), + ("Pacific/Fiji", "Pacific/Fiji"), + ("Pacific/Funafuti", "Pacific/Funafuti"), + ("Pacific/Galapagos", "Pacific/Galapagos"), + ("Pacific/Gambier", "Pacific/Gambier"), + ("Pacific/Guadalcanal", "Pacific/Guadalcanal"), + ("Pacific/Guam", "Pacific/Guam"), + ("Pacific/Honolulu", "Pacific/Honolulu"), + ("Pacific/Johnston", "Pacific/Johnston"), + ("Pacific/Kanton", "Pacific/Kanton"), + ("Pacific/Kiritimati", "Pacific/Kiritimati"), + ("Pacific/Kosrae", "Pacific/Kosrae"), + ("Pacific/Kwajalein", "Pacific/Kwajalein"), + ("Pacific/Majuro", "Pacific/Majuro"), + ("Pacific/Marquesas", "Pacific/Marquesas"), + ("Pacific/Midway", "Pacific/Midway"), + ("Pacific/Nauru", "Pacific/Nauru"), + ("Pacific/Niue", "Pacific/Niue"), + ("Pacific/Norfolk", "Pacific/Norfolk"), + ("Pacific/Noumea", "Pacific/Noumea"), + ("Pacific/Pago_Pago", "Pacific/Pago_Pago"), + ("Pacific/Palau", "Pacific/Palau"), + ("Pacific/Pitcairn", "Pacific/Pitcairn"), + ("Pacific/Pohnpei", "Pacific/Pohnpei"), + ("Pacific/Ponape", "Pacific/Ponape"), + ("Pacific/Port_Moresby", "Pacific/Port_Moresby"), + ("Pacific/Rarotonga", "Pacific/Rarotonga"), + ("Pacific/Saipan", "Pacific/Saipan"), + ("Pacific/Samoa", "Pacific/Samoa"), + ("Pacific/Tahiti", "Pacific/Tahiti"), + ("Pacific/Tarawa", "Pacific/Tarawa"), + ("Pacific/Tongatapu", "Pacific/Tongatapu"), + ("Pacific/Truk", "Pacific/Truk"), + ("Pacific/Wake", "Pacific/Wake"), + ("Pacific/Wallis", "Pacific/Wallis"), + ("Pacific/Yap", "Pacific/Yap"), + ("Poland", "Poland"), + ("Portugal", "Portugal"), + ("ROC", "ROC"), + ("ROK", "ROK"), + ("Singapore", "Singapore"), + ("Turkey", "Turkey"), + ("UCT", "UCT"), + ("US/Alaska", "US/Alaska"), + ("US/Aleutian", "US/Aleutian"), + ("US/Arizona", "US/Arizona"), + ("US/Central", "US/Central"), + ("US/East-Indiana", "US/East-Indiana"), + ("US/Eastern", "US/Eastern"), + ("US/Hawaii", "US/Hawaii"), + ("US/Indiana-Starke", "US/Indiana-Starke"), + ("US/Michigan", "US/Michigan"), + ("US/Mountain", "US/Mountain"), + ("US/Pacific", "US/Pacific"), + ("US/Samoa", "US/Samoa"), + ("UTC", "UTC"), + ("Universal", "Universal"), + ("W-SU", "W-SU"), + ("WET", "WET"), + ("Zulu", "Zulu"), + ("localtime", "localtime"), + ], + default="UTC", + max_length=255, + ), + ), + ] diff --git a/bookwyrm/migrations/0214_alter_edition_isbn_10_alter_edition_isbn_13.py b/bookwyrm/migrations/0214_alter_edition_isbn_10_alter_edition_isbn_13.py new file mode 100644 index 0000000000..68463548a8 --- /dev/null +++ b/bookwyrm/migrations/0214_alter_edition_isbn_10_alter_edition_isbn_13.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.20 on 2025-05-02 18:17 + +import bookwyrm.models.book +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0213_alter_user_preferred_timezone"), + ] + + operations = [ + migrations.AlterField( + model_name="edition", + name="isbn_10", + field=bookwyrm.models.fields.CharField( + blank=True, + max_length=255, + null=True, + validators=[bookwyrm.models.book.validate_isbn10], + ), + ), + migrations.AlterField( + model_name="edition", + name="isbn_13", + field=bookwyrm.models.fields.CharField( + blank=True, + max_length=255, + null=True, + validators=[bookwyrm.models.book.validate_isbn13], + ), + ), + ] diff --git a/bookwyrm/migrations/0215_cleanupuserexportfilesjob_and_more.py b/bookwyrm/migrations/0215_cleanupuserexportfilesjob_and_more.py new file mode 100644 index 0000000000..550b67d044 --- /dev/null +++ b/bookwyrm/migrations/0215_cleanupuserexportfilesjob_and_more.py @@ -0,0 +1,42 @@ +# Generated by Django 4.2.22 on 2025-08-16 01:38 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0214_alter_edition_isbn_10_alter_edition_isbn_13"), + ] + + operations = [ + migrations.CreateModel( + name="CleanUpUserExportFilesJob", + fields=[ + ( + "parentjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.parentjob", + ), + ), + ("expiry_date", models.DateTimeField()), + ("tasks", models.IntegerField(default=0)), + ("completed_tasks", models.IntegerField(default=0)), + ], + options={ + "abstract": False, + }, + bases=("bookwyrm.parentjob",), + ), + migrations.AddField( + model_name="sitesettings", + name="export_files_lifetime_hours", + field=models.IntegerField(default=72), + ), + ] diff --git a/bookwyrm/migrations/0215_rename_userrelationshipimport_userimportrelationship_and_more.py b/bookwyrm/migrations/0215_rename_userrelationshipimport_userimportrelationship_and_more.py new file mode 100644 index 0000000000..fec58ec5e5 --- /dev/null +++ b/bookwyrm/migrations/0215_rename_userrelationshipimport_userimportrelationship_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.22 on 2025-07-27 06:03 + +import bookwyrm.models.bookwyrm_import_job +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0214_alter_edition_isbn_10_alter_edition_isbn_13"), + ] + + operations = [ + migrations.RenameModel( + old_name="UserRelationshipImport", + new_name="UserImportRelationship", + ), + migrations.AlterField( + model_name="bookwyrmimportjob", + name="archive_file", + field=models.FileField( + blank=True, + null=True, + storage=bookwyrm.models.bookwyrm_import_job.select_exports_storage, + upload_to="", + ), + ), + ] diff --git a/bookwyrm/migrations/0216_alter_edition_shelves_alter_list_books_and_more.py b/bookwyrm/migrations/0216_alter_edition_shelves_alter_list_books_and_more.py new file mode 100644 index 0000000000..14177599f6 --- /dev/null +++ b/bookwyrm/migrations/0216_alter_edition_shelves_alter_list_books_and_more.py @@ -0,0 +1,93 @@ +# Generated by Django 5.2.3 on 2025-06-24 08:38 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ( + "bookwyrm", + "0215_rename_userrelationshipimport_userimportrelationship_and_more", + ), + ] + + operations = [ + migrations.AlterField( + model_name="edition", + name="shelves", + field=models.ManyToManyField( + through="bookwyrm.ShelfBook", + through_fields=("book", "shelf"), + to="bookwyrm.shelf", + ), + ), + migrations.AlterField( + model_name="list", + name="books", + field=models.ManyToManyField( + through="bookwyrm.ListItem", + through_fields=("book_list", "book"), + to="bookwyrm.edition", + ), + ), + migrations.AlterField( + model_name="shelf", + name="books", + field=models.ManyToManyField( + through="bookwyrm.ShelfBook", + through_fields=("shelf", "book"), + to="bookwyrm.edition", + ), + ), + migrations.AlterField( + model_name="status", + name="favorites", + field=models.ManyToManyField( + related_name="user_favorites", + through="bookwyrm.Favorite", + through_fields=("status", "user"), + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="user", + name="blocks", + field=models.ManyToManyField( + related_name="blocked_by", + through="bookwyrm.UserBlocks", + through_fields=("user_subject", "user_object"), + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="user", + name="favorites", + field=models.ManyToManyField( + related_name="favorite_statuses", + through="bookwyrm.Favorite", + through_fields=("user", "status"), + to="bookwyrm.status", + ), + ), + migrations.AlterField( + model_name="user", + name="follow_requests", + field=models.ManyToManyField( + related_name="follower_requests", + through="bookwyrm.UserFollowRequest", + through_fields=("user_subject", "user_object"), + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="user", + name="followers", + field=models.ManyToManyField( + related_name="following", + through="bookwyrm.UserFollows", + through_fields=("user_object", "user_subject"), + to=settings.AUTH_USER_MODEL, + ), + ), + ] diff --git a/bookwyrm/migrations/0217_merge_20250816_0749.py b/bookwyrm/migrations/0217_merge_20250816_0749.py new file mode 100644 index 0000000000..ae2fa96f78 --- /dev/null +++ b/bookwyrm/migrations/0217_merge_20250816_0749.py @@ -0,0 +1,13 @@ +# Generated by Django 5.2.3 on 2025-08-16 07:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0215_cleanupuserexportfilesjob_and_more"), + ("bookwyrm", "0216_alter_edition_shelves_alter_list_books_and_more"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0217_usersession.py b/bookwyrm/migrations/0217_usersession.py new file mode 100644 index 0000000000..f470145eb1 --- /dev/null +++ b/bookwyrm/migrations/0217_usersession.py @@ -0,0 +1,42 @@ +# Generated by Django 5.2.3 on 2025-08-13 00:07 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0216_alter_edition_shelves_alter_list_books_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="UserSession", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("session_key", models.CharField(max_length=40)), + ("created_date", models.DateTimeField(auto_now_add=True)), + ("operating_system", models.CharField(max_length=50)), + ("browser_type", models.CharField(max_length=50)), + ("ip_address", models.CharField(max_length=45)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="sessions", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + ] diff --git a/bookwyrm/migrations/0218_merge_0217_merge_20250816_0749_0217_usersession.py b/bookwyrm/migrations/0218_merge_0217_merge_20250816_0749_0217_usersession.py new file mode 100644 index 0000000000..12dec50729 --- /dev/null +++ b/bookwyrm/migrations/0218_merge_0217_merge_20250816_0749_0217_usersession.py @@ -0,0 +1,13 @@ +# Generated by Django 5.2.3 on 2025-09-08 18:40 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0217_merge_20250816_0749"), + ("bookwyrm", "0217_usersession"), + ] + + operations = [] diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py index 6bb99c7f25..9c93d3b1ac 100644 --- a/bookwyrm/models/__init__.py +++ b/bookwyrm/models/__init__.py @@ -25,8 +25,15 @@ from .group import Group, GroupMember, GroupMemberInvitation +from .housekeeping import CleanUpUserExportFilesJob, start_export_deletions + from .import_job import ImportJob, ImportItem -from .bookwyrm_import_job import BookwyrmImportJob +from .bookwyrm_import_job import ( + BookwyrmImportJob, + UserImportBook, + UserImportPost, + import_book_task, +) from .bookwyrm_export_job import BookwyrmExportJob from .move import MoveUser @@ -40,6 +47,8 @@ from .hashtag import Hashtag +from .session import UserSession, create_user_session + cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass) activity_models = { c[1].activity_serializer.__name__: c[1] diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index 06ef373e68..71f7b0ed90 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -31,7 +31,7 @@ PropertyField = namedtuple("PropertyField", ("set_activity_from_field")) -# pylint: disable=invalid-name + def set_activity_from_property_field(activity, obj, field): """assign a model property value to the activity json""" activity[field[1]] = getattr(obj, field[0]) @@ -129,7 +129,20 @@ def find_existing(cls, data): def broadcast(self, activity, sender, software=None, queue=BROADCAST): """send out an activity""" + + # if we're posting about ShelfBooks, set a delay to give the base activity + # time to add the book on remote servers first to avoid race conditions + countdown = ( + 10 + if ( + isinstance(activity, object) + and not isinstance(activity["object"], str) + and activity["object"].get("type", None) in ["GeneratedNote", "Comment"] + ) + else 0 + ) broadcast_task.apply_async( + countdown=countdown, args=( sender.id, json.dumps(activity, cls=activitypub.ActivityEncoder), @@ -227,6 +240,7 @@ def save( return try: + # TODO: here is where we might use an ActivityPub extension instead # do we have a "pure" activitypub version of this for mastodon? if software != "bookwyrm" and hasattr(self, "pure_content"): pure_activity = self.to_create_activity(user, pure=True) diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index ca13d95538..814e59a733 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -62,7 +62,7 @@ def local_path(self): name = self.name if name: - slug = slugify(name) + slug = slugify(name, allow_unicode=True) local = f"{local}/s/{slug}" return local diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 4ff377dbbb..164f8e9322 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -8,6 +8,7 @@ from django.contrib.postgres.search import SearchVectorField from django.contrib.postgres.indexes import GinIndex from django.core.cache import cache +from django.core.exceptions import ValidationError from django.db import models, transaction from django.db.models import Prefetch, ManyToManyField from django.dispatch import receiver @@ -41,6 +42,9 @@ class BookDataModel(ObjectMixin, BookWyrmModel): openlibrary_key = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) + finna_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True + ) inventaire_id = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) @@ -91,6 +95,11 @@ def isfdb_link(self): """generate the url from the isfdb id""" return f"https://www.isfdb.org/cgi-bin/title.cgi?{self.isfdb}" + @property + def finna_link(self): + """generate the url from the finna key""" + return f"http://finna.fi/Record/{self.finna_key}" + class Meta: """can't initialize this model, that wouldn't make sense""" @@ -133,7 +142,6 @@ def merge_into(self, canonical: Self, dry_run=False) -> Dict[str, Any]: related_models = [ (r.remote_field.name, r.related_model) for r in self._meta.related_objects ] - # pylint: disable=protected-access for related_field, related_model in related_models: # Skip the ManyToMany fields that aren’t auto-created. These # should have a corresponding OneToMany field in the model for @@ -334,11 +342,35 @@ def get_remote_id(self): """editions and works both use "book" instead of model_name""" return f"{BASE_URL}/book/{self.id}" - def guess_sort_title(self): + def guess_sort_title(self, user=None): """Get a best-guess sort title for the current book""" + + if self.languages not in ([], None): + lang_codes = set( + k + for (k, v) in LANGUAGE_ARTICLES.items() + for language in tuple(self.languages) + if language.lower() in v["variants"] + ) + + elif user and user.preferred_language: + lang_codes = set( + k + for (k, v) in LANGUAGE_ARTICLES.items() + if user.preferred_language.lower() in v["variants"] + ) + + else: + lang_codes = set( + k + for (k, v) in LANGUAGE_ARTICLES.items() + if DEFAULT_LANGUAGE.lower() in v["variants"] + ) + articles = chain( - *(LANGUAGE_ARTICLES.get(language, ()) for language in tuple(self.languages)) + *(LANGUAGE_ARTICLES[language].get("articles") for language in lang_codes) ) + return re.sub(f'^{" |^".join(articles)} ', "", str(self.title).lower()) def __repr__(self): @@ -446,15 +478,107 @@ def to_edition_list(self, **kwargs): ] +def validate_isbn10(maybe_isbn: str) -> None: + """Check if isbn10 mathes some expectations""" + + if not (normalized_isbn := normalize_isbn(maybe_isbn)): + raise ValidationError( + _("%(value)s doesn't look like an ISBN"), params={"value": maybe_isbn} + ) + + normalized_isbn = normalized_isbn.zfill(10) + # len should be 10 with poddible 0 in front + if len(normalized_isbn) != 10: + raise ValidationError( + _("%(value)s doesn't look like an ISBN"), params={"value": maybe_isbn} + ) + + # Last character can be X for checksum mark + if not normalized_isbn.upper()[:-1].isnumeric(): + raise ValidationError( + _("%(value)s doesn't look like an ISBN"), params={"value": maybe_isbn} + ) + + if (isbn13_version := isbn_10_to_13(normalized_isbn)) is None: + raise ValidationError( + _("%(value)s doesn't look like an ISBN"), params={"value": maybe_isbn} + ) + + if (checksum_version := isbn_13_to_10(isbn13_version)) is None: + raise ValidationError( + _("%(value)s doesn't look like an ISBN"), params={"value": maybe_isbn} + ) + + if checksum_version != normalized_isbn: + raise ValidationError( + _( + "%(value)s doesn't have correct ISBN checksum, " + "we expected %(check_version)s" + ), + params={"value": maybe_isbn, "check_version": checksum_version}, + ) + + +def validate_isbn13(maybe_isbn: str) -> None: + """Check if isbn13 mathes some expectations""" + + if maybe_isbn[:3] not in ["978", "979"]: + raise ValidationError( + _("%(value)s doesn't look like an ISBN"), params={"value": maybe_isbn} + ) + + normalized_isbn = normalize_isbn(maybe_isbn) + if len(normalized_isbn) != 13: + raise ValidationError( + _("%(value)s doesn't look like an ISBN"), params={"value": maybe_isbn} + ) + + if not normalized_isbn.isnumeric(): + raise ValidationError( + _("%(value)s doesn't look like an ISBN"), params={"value": maybe_isbn} + ) + + if (isbn10_version := isbn_13_to_10(normalized_isbn)) is None: + raise ValidationError( + _("%(value)s doesn't look like an ISBN"), params={"value": maybe_isbn} + ) + + if (checksum_version := isbn_10_to_13(isbn10_version)) is None: + raise ValidationError( + _("%(value)s doesn't look like an ISBN"), params={"value": maybe_isbn} + ) + + # We might have 978 or 979 prefix, so ignore that on comparing + if checksum_version[3:] != normalized_isbn[3:]: + raise ValidationError( + _( + "%(value)s doesn't have correct ISBN checksum, " + "we expected %(check_version)s" + ), + params={ + "value": maybe_isbn, + "check_version": maybe_isbn[:3] + checksum_version[3:], + }, + ) + + class Edition(Book): """an edition of a book""" # these identifiers only apply to editions, not works isbn_10 = fields.CharField( - max_length=255, blank=True, null=True, deduplication_field=True + max_length=255, + blank=True, + null=True, + deduplication_field=True, + validators=[validate_isbn10], ) isbn_13 = fields.CharField( - max_length=255, blank=True, null=True, deduplication_field=True + max_length=255, + blank=True, + null=True, + deduplication_field=True, + validators=[validate_isbn13], ) oclc_number = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True @@ -617,7 +741,7 @@ def isbn_10_to_13(isbn_10): def isbn_13_to_10(isbn_13): """convert isbn 13 to 10, if possible""" - if isbn_13[:3] != "978": + if isbn_13[:3] not in ["978", "979"]: return None isbn_13 = re.sub(r"[^0-9X]", "", isbn_13) diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index f355c86a4a..8fbb659ad0 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -6,8 +6,7 @@ from boto3.session import Session as BotoSession from s3_tar import S3Tar -from django.db.models import BooleanField, FileField, JSONField -from django.db.models import Q +from django.db.models import FileField, JSONField from django.core.serializers.json import DjangoJSONEncoder from django.core.files.base import ContentFile from django.core.files.storage import storages @@ -18,7 +17,7 @@ from bookwyrm.models import Review, Comment, Quotation from bookwyrm.models import Edition from bookwyrm.models import UserFollows, User, UserBlocks -from bookwyrm.models.job import ParentJob +from bookwyrm.models.job import ParentJob, ParentTask from bookwyrm.tasks import app, IMPORTS from bookwyrm.utils.tar import BookwyrmTarFile @@ -28,7 +27,7 @@ class BookwyrmAwsSession(BotoSession): """a boto session that always uses settings.AWS_S3_ENDPOINT_URL""" - def client(self, *args, **kwargs): # pylint: disable=arguments-differ + def client(self, *args, **kwargs): kwargs["endpoint_url"] = settings.AWS_S3_ENDPOINT_URL return super().client("s3", *args, **kwargs) @@ -43,38 +42,41 @@ class BookwyrmExportJob(ParentJob): export_data = FileField(null=True, storage=select_exports_storage) export_json = JSONField(null=True, encoder=DjangoJSONEncoder) - json_completed = BooleanField(default=False) def start_job(self): """schedule the first task""" - task = create_export_json_task.delay(job_id=self.id) - self.task_id = task.id - self.save(update_fields=["task_id"]) + self.set_status("active") + create_export_json_task.delay(job_id=self.id) -@app.task(queue=IMPORTS) -def create_export_json_task(job_id): +@app.task(queue=IMPORTS, base=ParentTask) +def create_export_json_task(**kwargs): """create the JSON data for the export""" - job = BookwyrmExportJob.objects.get(id=job_id) - + job = BookwyrmExportJob.objects.get(id=kwargs["job_id"]) # don't start the job if it was stopped from the UI - if job.complete: + if job.status == "stopped": return try: - job.set_status("active") - - # generate JSON structure - job.export_json = export_json(job.user) + # generate JSON + data = export_user(job.user) + data["settings"] = export_settings(job.user) + data["goals"] = export_goals(job.user) + data["books"] = export_books(job.user) + data["saved_lists"] = export_saved_lists(job.user) + data["follows"] = export_follows(job.user) + data["blocks"] = export_blocks(job.user) + job.export_json = data job.save(update_fields=["export_json"]) - # create archive in separate task + # trigger task to create tar file create_archive_task.delay(job_id=job.id) + except Exception as err: # pylint: disable=broad-except logger.exception( - "create_export_json_task for %s failed with error: %s", job, err + "create_export_json_task for job %s failed with error: %s", job.id, err ) job.set_status("failed") @@ -95,23 +97,21 @@ def add_file_to_s3_tar(s3_tar: S3Tar, storage, file, directory=""): ) -@app.task(queue=IMPORTS) -def create_archive_task(job_id): +@app.task(queue=IMPORTS, base=ParentTask) +def create_archive_task(**kwargs): """create the archive containing the JSON file and additional files""" - job = BookwyrmExportJob.objects.get(id=job_id) + job = BookwyrmExportJob.objects.get(id=kwargs["job_id"]) # don't start the job if it was stopped from the UI - if job.complete: + if job.status == "stopped": return try: export_task_id = str(job.task_id) archive_filename = f"{export_task_id}.tar.gz" export_json_bytes = DjangoJSONEncoder().encode(job.export_json).encode("utf-8") - user = job.user - editions = get_books_for_user(user) if settings.USE_S3: # Storage for writing temporary files @@ -134,18 +134,12 @@ def create_archive_task(job_id): os.path.join(exports_storage.location, export_json_tmp_file) ) - # Add images to TAR + # Add avatar to TAR images_storage = storages["default"] if user.avatar: add_file_to_s3_tar(s3_tar, images_storage, user.avatar) - for edition in editions: - if edition.cover: - add_file_to_s3_tar( - s3_tar, images_storage, edition.cover, directory="images" - ) - # Create archive and store file name s3_tar.tar() job.export_data = archive_filename @@ -165,30 +159,17 @@ def create_archive_task(job_id): if user.avatar: tar.add_image(user.avatar) - for edition in editions: - if edition.cover: - tar.add_image(edition.cover, directory="images") job.save(update_fields=["export_data"]) - job.set_status("completed") + job.complete_job() except Exception as err: # pylint: disable=broad-except - logger.exception("create_archive_task for %s failed with error: %s", job, err) + logger.exception( + "create_archive_task for job %s failed with error: %s", job.id, err + ) job.set_status("failed") -def export_json(user: User): - """create export JSON""" - data = export_user(user) # in the root of the JSON structure - data["settings"] = export_settings(user) - data["goals"] = export_goals(user) - data["books"] = export_books(user) - data["saved_lists"] = export_saved_lists(user) - data["follows"] = export_follows(user) - data["blocks"] = export_blocks(user) - return data - - def export_user(user: User): """export user data""" data = user.to_activity() @@ -288,14 +269,14 @@ def export_book(user: User, edition: Edition): for status in ["comments", "quotations", "reviews"]: data[status] = [] - comments = Comment.objects.filter(user=user, book=edition).all() + comments = Comment.objects.filter(user=user, book=edition, deleted=False).all() for status in comments: obj = status.to_activity() obj["progress"] = status.progress obj["progress_mode"] = status.progress_mode data["comments"].append(obj) - quotes = Quotation.objects.filter(user=user, book=edition).all() + quotes = Quotation.objects.filter(user=user, book=edition, deleted=False).all() for status in quotes: obj = status.to_activity() obj["position"] = status.position @@ -303,7 +284,7 @@ def export_book(user: User, edition: Edition): obj["position_mode"] = status.position_mode data["quotations"].append(obj) - reviews = Review.objects.filter(user=user, book=edition).all() + reviews = Review.objects.filter(user=user, book=edition, deleted=False).all() data["reviews"] = [status.to_activity() for status in reviews] # readthroughs can't be serialized to activity @@ -315,19 +296,26 @@ def export_book(user: User, edition: Edition): def get_books_for_user(user): - """Get all the books and editions related to a user""" - - editions = ( - Edition.objects.select_related("parent_work") - .filter( - Q(shelves__user=user) - | Q(readthrough__user=user) - | Q(review__user=user) - | Q(list__user=user) - | Q(comment__user=user) - | Q(quotation__user=user) - ) - .distinct() + """ + Get all the books and editions related to a user. + We use union() instead of Q objects because it creates + multiple simple queries instead of a complex DB query + that can time out. + """ + + shelf_eds = Edition.objects.select_related("parent_work").filter(shelves__user=user) + rt_eds = Edition.objects.select_related("parent_work").filter( + readthrough__user=user + ) + review_eds = Edition.objects.select_related("parent_work").filter(review__user=user) + list_eds = Edition.objects.select_related("parent_work").filter(list__user=user) + comment_eds = Edition.objects.select_related("parent_work").filter( + comment__user=user, comment__deleted=False ) + quote_eds = Edition.objects.select_related("parent_work").filter( + quotation__user=user, quotation__deleted=False + ) + + editions = shelf_eds.union(rt_eds, review_eds, list_eds, comment_eds, quote_eds) return editions diff --git a/bookwyrm/models/bookwyrm_import_job.py b/bookwyrm/models/bookwyrm_import_job.py index 5229430eb0..d679be2928 100644 --- a/bookwyrm/models/bookwyrm_import_job.py +++ b/bookwyrm/models/bookwyrm_import_job.py @@ -2,43 +2,251 @@ import json import logging - -from django.db.models import FileField, JSONField, CharField +import math +from urllib.parse import urlparse + +from botocore.exceptions import EndpointConnectionError +import requests + +from django.apps import apps +from django.db.models import ( + BooleanField, + ForeignKey, + FileField, + JSONField, + TextChoices, + PROTECT, + SET_NULL, +) +from django.core.files.storage import storages from django.utils import timezone from django.utils.html import strip_tags +from django.utils.translation import gettext_lazy as _ from django.contrib.postgres.fields import ArrayField as DjangoArrayField -from bookwyrm import activitypub -from bookwyrm import models +from bookwyrm import activitypub, models, settings +from bookwyrm.connectors import connector_manager from bookwyrm.tasks import app, IMPORTS -from bookwyrm.models.job import ParentJob, ParentTask, SubTask +from bookwyrm.models.job import Job, ParentJob, ChildJob, ParentTask, SubTask from bookwyrm.utils.tar import BookwyrmTarFile logger = logging.getLogger(__name__) +def select_exports_storage(): + """callable to allow for dependency on runtime configuration""" + return storages["exports"] + + class BookwyrmImportJob(ParentJob): """entry for a specific request for importing a bookwyrm user backup""" - archive_file = FileField(null=True, blank=True) + archive_file = FileField(null=True, blank=True, storage=select_exports_storage) import_data = JSONField(null=True) - required = DjangoArrayField(CharField(max_length=50, blank=True), blank=True) + required = DjangoArrayField( + models.fields.CharField(max_length=50, blank=True), blank=True + ) + retry = BooleanField(default=False) + + def start_job(self): + """Start the job""" + start_import_task.delay(job_id=self.id) + + @property + def book_tasks(self): + """How many import book tasks are there?""" + return UserImportBook.objects.filter(parent_job=self).all() + + @property + def status_tasks(self): + """How many import status tasks are there?""" + return UserImportPost.objects.filter(parent_job=self).all() + + @property + def relationship_tasks(self): + """How many import relationship tasks are there?""" + return UserImportRelationship.objects.filter(parent_job=self).all() + + @property + def item_count(self): + """How many total tasks are there?""" + return self.book_tasks.count() + self.status_tasks.count() + + @property + def pending_item_count(self): + """How many tasks are incomplete?""" + status = BookwyrmImportJob.Status + book_tasks = self.book_tasks.filter( + status__in=[status.PENDING, status.ACTIVE] + ).count() + + status_tasks = self.status_tasks.filter( + status__in=[status.PENDING, status.ACTIVE] + ).count() + + relationship_tasks = self.relationship_tasks.filter( + status__in=[status.PENDING, status.ACTIVE] + ).count() + + return book_tasks + status_tasks + relationship_tasks + + @property + def percent_complete(self): + """How far along?""" + item_count = self.item_count + if not item_count: + return 0 + return math.floor((item_count - self.pending_item_count) / item_count * 100) + + def complete_job(self): + """Report that the job has completed and stop pending children.""" + + super().complete_job() + + # delete the import file + self.archive_file.delete(save=True) + + def notify_child_job_complete(self): + """let the job know when the items get work done""" + + if self.complete: + return + + self.updated_date = timezone.now() + self.save(update_fields=["updated_date"]) + + if not self.complete and self.has_completed: + self.complete_job() + + +class UserImportBook(ChildJob): + """ChildJob to import each book. + Equivalent to ImportItem when importing a csv file of books""" + + book = ForeignKey(models.Book, on_delete=SET_NULL, null=True, blank=True) + book_data = JSONField(null=False) + + def start_job(self, origin_is_ok=False): + """Start the job""" + import_book_task.delay( + child_id=self.id, origin_is_ok=origin_is_ok, job_type="UserImportBook" + ) + + def complete_job(self): + """Report to BookwyrmImportJob that the job has completed. + Do not use super() here because the parent class will + complete the job over the top of us and then you will be sad.""" + + Job.complete_job(self) # don't notify ParentJob + parent = BookwyrmImportJob.objects.get(id=self.parent_job.id) + parent.notify_child_job_complete() + + +class UserImportPost(ChildJob): + """ChildJob for comments, quotes, and reviews""" + + class StatusType(TextChoices): + """Possible status types.""" + + COMMENT = "comment", _("Comment") + REVIEW = "review", _("Review") + QUOTE = "quote", _("Quotation") + + json = JSONField(null=False) + book = models.fields.ForeignKey( + "Edition", on_delete=PROTECT, activitypub_field="inReplyToBook" + ) + status_type = models.fields.CharField( + max_length=10, choices=StatusType.choices, default=StatusType.COMMENT, null=True + ) + + def start_job(self): + """Start the job""" + upsert_status_task.delay(child_id=self.id, job_type="UserImportPost") + + def complete_job(self): + """Report to BookwyrmImportJob that the job has completed.""" + + Job.complete_job(self) # don't notify ParentJob + parent = BookwyrmImportJob.objects.get(id=self.parent_job.id) + parent.notify_child_job_complete() + + +class UserImportRelationship(ChildJob): + """ChildJob for follows and blocks""" + + class RelationshipType(TextChoices): + """Possible relationship types.""" + + FOLLOW = "follow", _("Follow") + BLOCK = "block", _("Block") + + relationship = models.fields.CharField( + max_length=10, choices=RelationshipType.choices, null=True + ) + remote_id = models.fields.RemoteIdField(null=True, unique=False) def start_job(self): """Start the job""" - start_import_task.delay(job_id=self.id, no_children=True) + import_user_relationship_task.delay( + child_id=self.id, job_type="UserImportRelationship" + ) + + def complete_job(self): + """Report to BookwyrmImportJob that the job has completed.""" + + Job.complete_job(self) # don't notify ParentJob + parent = BookwyrmImportJob.objects.get(id=self.parent_job.id) + parent.notify_child_job_complete() + + +class ImportUserTask(ParentTask): + """A task for a user import job""" + + def before_start(self, task_id, args, kwargs): + """Handler called before the task starts.""" + job = BookwyrmImportJob.objects.get(id=kwargs["job_id"]) + job.task_id = task_id + job.save(update_fields=["task_id"]) + + +class UserImportSubTask(SubTask): + """Makes sure we refer to the correct child job and call + subclass methods instead of methods on ChildJob & ParentJob""" + + def before_start(self, task_id, args, kwargs): + """Handler called before the task starts.""" + + model = apps.get_model(f'bookwyrm.{kwargs["job_type"]}', require_ready=True) + child_job = model.objects.get(id=kwargs["child_id"]) + child_job.task_id = task_id + child_job.save(update_fields=["task_id"]) + child_job.set_status(ChildJob.Status.ACTIVE) + def on_success(self, retval, task_id, args, kwargs): + """Run by the worker if the task executes successfully""" -@app.task(queue=IMPORTS, base=ParentTask) + # we want to complete our own UserImportBook job, not ChildJob + model = apps.get_model(f'bookwyrm.{kwargs["job_type"]}', require_ready=True) + subtask = model.objects.get(id=kwargs["child_id"]) + subtask.complete_job() + + +# pylint: disable=too-many-branches +@app.task(queue=IMPORTS, base=ImportUserTask) def start_import_task(**kwargs): - """trigger the child import tasks for each user data""" + """trigger the child import tasks for each user data + We always import the books even if not assigning + them to shelves, lists etc""" job = BookwyrmImportJob.objects.get(id=kwargs["job_id"]) - archive_file = job.archive_file + archive_file = job.bookwyrmimportjob.archive_file - # don't start the job if it was stopped from the UI - if job.complete: + if job.status == "stopped": return + job.status = "active" + job.save(update_fields=["status"]) + try: archive_file.open("rb") with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: @@ -56,135 +264,198 @@ def start_import_task(**kwargs): if "include_saved_lists" in job.required: upsert_saved_lists(job.user, job.import_data.get("saved_lists", [])) if "include_follows" in job.required: - upsert_follows(job.user, job.import_data.get("follows", [])) + for remote_id in job.import_data.get("follows", []): + UserImportRelationship.objects.create( + parent_job=job, remote_id=remote_id, relationship="follow" + ) if "include_blocks" in job.required: - upsert_user_blocks(job.user, job.import_data.get("blocks", [])) - - process_books(job, tar) - - job.set_status("complete") - archive_file.close() - - except Exception as err: # pylint: disable=broad-except - logger.exception("User Import Job %s Failed with error: %s", job.id, err) - job.set_status("failed") - - -def process_books(job, tar): - """ - Process user import data related to books - We always import the books even if not assigning - them to shelves, lists etc - """ - - books = job.import_data.get("books") + for remote_id in job.import_data.get("blocks", []): + UserImportRelationship.objects.create( + parent_job=job, remote_id=remote_id, relationship="block" + ) + + for item in UserImportRelationship.objects.filter(parent_job=job).all(): + item.start_job() + + try: + url_parts = urlparse(job.import_data.get("id")) + url = f"{url_parts.scheme}://{url_parts.netloc}" + # Check https://example.com to see if the instance is still online + # If not, we don't bother trying to pull book data from it. + resp = requests.head( + url, + headers={ + "User-Agent": settings.USER_AGENT, + }, + timeout=settings.QUERY_TIMEOUT, + ) - for data in books: - book = get_or_create_edition(data, tar) + origin_is_ok = resp.ok - if "include_shelves" in job.required: - upsert_shelves(book, job.user, data) + except ( + EndpointConnectionError, + requests.exceptions.ConnectionError, + ConnectionRefusedError, + ): - if "include_readthroughs" in job.required: - upsert_readthroughs(data.get("readthroughs"), job.user, book.id) + origin_is_ok = False - if "include_comments" in job.required: - upsert_statuses( - job.user, models.Comment, data.get("comments"), book.remote_id - ) - if "include_quotations" in job.required: - upsert_statuses( - job.user, models.Quotation, data.get("quotations"), book.remote_id - ) + for data in job.import_data.get("books"): + book_job = UserImportBook.objects.create(parent_job=job, book_data=data) + book_job.start_job(origin_is_ok=origin_is_ok) - if "include_reviews" in job.required: - upsert_statuses( - job.user, models.Review, data.get("reviews"), book.remote_id - ) + archive_file.close() - if "include_lists" in job.required: - upsert_lists(job.user, data.get("lists"), book.id) + except Exception as err: # pylint: disable=broad-except + logger.error( + "User Import Job %s Failed with error: %s", job.id, err, exc_info=True + ) + job.set_status("failed") -def get_or_create_edition(book_data, tar): - """Take a JSON string of work and edition data, - find or create the edition and work in the database and - return an edition instance""" +def create_book_from_json(book_data): + """create a book from the JSON in the import file + as a last resort if we can't find the book + in this instance or in the source instance""" edition = book_data.get("edition") - existing = models.Edition.find_existing(edition) - if existing: - return existing - + work = book_data.get("work") # make sure we have the authors in the local DB # replace the old author ids in the edition JSON edition["authors"] = [] + work["authors"] = [] for author in book_data.get("authors"): - parsed_author = activitypub.parse(author) - instance = parsed_author.to_model( - model=models.Author, save=True, overwrite=True + instance = activitypub.parse(author).to_model( + model=models.Author, save=True, overwrite=False ) edition["authors"].append(instance.remote_id) - - # we will add the cover later from the tar - # don't try to load it from the old server - cover = edition.get("cover", {}) - cover_path = cover.get("url", None) - edition["cover"] = {} + work["authors"].append(instance.remote_id) # first we need the parent work to exist - work = book_data.get("work") work["editions"] = [] - parsed_work = activitypub.parse(work) - work_instance = parsed_work.to_model(model=models.Work, save=True, overwrite=True) + work_instance = activitypub.parse(work).to_model( + model=models.Work, save=True, overwrite=False + ) # now we have a work we can add it to the edition # and create the edition model instance edition["work"] = work_instance.remote_id - parsed_edition = activitypub.parse(edition) - book = parsed_edition.to_model(model=models.Edition, save=True, overwrite=True) - - # set the cover image from the tar - if cover_path: - tar.write_image_to_file(cover_path, book.cover) + book = activitypub.parse(edition).to_model( + model=models.Edition, save=True, overwrite=False + ) return book -def upsert_readthroughs(data, user, book_id): - """Take a JSON string of readthroughs and - find or create the instances in the database""" +@app.task(queue=IMPORTS, base=UserImportSubTask) +def import_book_task(**kwargs): # pylint: disable=too-many-branches + """Take work and edition data, + find or create the edition and work in the database""" - for read_through in data: + task = UserImportBook.objects.get(id=kwargs["child_id"]) + job = task.parent_job + book_data = task.book_data - obj = {} - keys = [ - "progress_mode", - "start_date", - "finish_date", - "stopped_date", - "is_active", - ] - for key in keys: - obj[key] = read_through[key] - obj["user_id"] = user.id - obj["book_id"] = book_id + if task.complete or job.status == "stopped": + return - existing = models.ReadThrough.objects.filter(**obj).first() - if not existing: - models.ReadThrough.objects.create(**obj) + try: + edition = book_data.get("edition") + book = models.Edition.find_existing(edition) + if not book: + + if kwargs["origin_is_ok"]: + # try importing from the instance the user is coming from + remote_id = edition.get("id") + connector = connector_manager.get_or_create_connector(remote_id) + book = connector.get_or_create_book(remote_id) + + if not book: + # import directly from the import JSON + # this will not create a cover image + book = create_book_from_json(book_data) + + task.book = book + task.save(update_fields=["book"]) + required = task.parent_job.bookwyrmimportjob.required + + if "include_shelves" in required: + upsert_shelves(task.parent_job.user, book, book_data.get("shelves")) + + if "include_readthroughs" in required: + upsert_readthroughs( + task.parent_job.user, book.id, book_data.get("readthroughs") + ) + if "include_lists" in required: + upsert_lists(task.parent_job.user, book.id, book_data.get("lists")) -def upsert_statuses(user, cls, data, book_remote_id): - """Take a JSON string of a status and - find or create the instances in the database""" + except Exception as err: # pylint: disable=broad-except + logger.error( + "Book Import Task %s for Job %s Failed with error: %s", task.id, job.id, err + ) + task.fail_reason = _("Unknown error importing book") + task.save(update_fields=["fail_reason"]) + task.set_status("failed") + + # Now import statuses + # These are also subtasks so that we can isolate anything that fails + if "include_comments" in job.bookwyrmimportjob.required: + for status in book_data.get("comments"): + UserImportPost.objects.create( + parent_job=task.parent_job, + json=status, + book=book, + status_type=UserImportPost.StatusType.COMMENT, + ) - for status in data: - if is_alias( - user, status["attributedTo"] - ): # don't let l33t hax0rs steal other people's posts - # update ids and remove replies + if "include_quotations" in job.bookwyrmimportjob.required: + for status in book_data.get("quotations"): + UserImportPost.objects.create( + parent_job=task.parent_job, + json=status, + book=book, + status_type=UserImportPost.StatusType.QUOTE, + ) + + if "include_reviews" in job.bookwyrmimportjob.required: + for status in book_data.get("reviews"): + UserImportPost.objects.create( + parent_job=task.parent_job, + json=status, + book=book, + status_type=UserImportPost.StatusType.REVIEW, + ) + + for item in UserImportPost.objects.filter(parent_job=job).all(): + item.start_job() + + task.complete_job() + + +@app.task(queue=IMPORTS, base=UserImportSubTask) +def upsert_status_task(**kwargs): + """Find or create book statuses""" + + task = UserImportPost.objects.get(id=kwargs["child_id"]) + job = task.parent_job + user = job.user + status = task.json + status_class = ( + models.Review + if task.status_type == "review" + else models.Quotation + if task.status_type == "quote" + else models.Comment + ) + + if task.complete or job.status == "stopped": + return + + try: + # only add statuses if this is the same user + if is_alias(user, status.get("attributedTo", False)): status["attributedTo"] = user.remote_id status["to"] = update_followers_address(user, status["to"]) status["cc"] = update_followers_address(user, status["cc"]) @@ -193,13 +464,15 @@ def upsert_statuses(user, cls, data, book_remote_id): ] = ( {} ) # this parses incorrectly but we can't set it without knowing the new id - status["inReplyToBook"] = book_remote_id + status["inReplyToBook"] = task.book.remote_id parsed = activitypub.parse(status) if not status_already_exists( user, parsed ): # don't duplicate posts on multiple import - instance = parsed.to_model(model=cls, save=True, overwrite=True) + instance = parsed.to_model( + model=status_class, save=True, overwrite=True + ) for val in [ "progress", @@ -214,11 +487,52 @@ def upsert_statuses(user, cls, data, book_remote_id): instance.remote_id = instance.get_remote_id() # update the remote_id instance.save() # save and broadcast + task.complete_job() + else: - logger.warning("User does not have permission to import statuses") + logger.warning( + "User not authorized to import statuses, or status is tombstone" + ) + task.fail_reason = _("unauthorized") + task.save(update_fields=["fail_reason"]) + task.set_status("failed") + + except Exception as err: # pylint: disable=broad-except + logger.error("User Status Import Task %s Failed with error: %s", task.id, err) + task.fail_reason = _("Unknown error importing book status") + task.save(update_fields=["fail_reason"]) + task.set_status("failed") -def upsert_lists(user, lists, book_id): +def upsert_readthroughs(user, book_id, data): + """Take a JSON string of readthroughs and + find or create the instances in the database""" + + for read_through in data: + + obj = {} + keys = [ + "progress_mode", + "start_date", + "finish_date", + "stopped_date", + "is_active", + ] + for key in keys: + obj[key] = read_through[key] + obj["user_id"] = user.id + obj["book_id"] = book_id + + existing = models.ReadThrough.objects.filter(**obj).first() + if not existing: + models.ReadThrough.objects.create(**obj) + + +def upsert_lists( + user, + book_id, + lists, +): """Take a list of objects each containing a list and list item as AP objects @@ -244,21 +558,22 @@ def upsert_lists(user, lists, book_id): item = models.ListItem.objects.filter(book=book, book_list=booklist).exists() if not item: count = booklist.books.count() + notes = blist["list_item"].get("notes", "") + approved = blist["list_item"].get("approved", False) models.ListItem.objects.create( book=book, book_list=booklist, user=user, - notes=blist["list_item"]["notes"], - approved=blist["list_item"]["approved"], + notes=notes, + approved=approved, order=count + 1, ) -def upsert_shelves(book, user, book_data): +def upsert_shelves(user, book, shelves): """Take shelf JSON objects and create DB entries if they don't already exist""" - shelves = book_data["shelves"] for shelf in shelves: book_shelf = models.Shelf.objects.filter(name=shelf["name"], user=user).first() @@ -275,6 +590,10 @@ def upsert_shelves(book, user, book_data): ) +# user updates +############## + + def update_user_profile(user, tar, data): """update the user's profile from import data""" name = data.get("name", None) @@ -315,14 +634,6 @@ def update_user_settings(user, data): user.save(update_fields=update_fields) -@app.task(queue=IMPORTS, base=SubTask) -def update_user_settings_task(job_id): - """wrapper task for user's settings import""" - parent_job = BookwyrmImportJob.objects.get(id=job_id) - - return update_user_settings(parent_job.user, parent_job.import_data.get("user")) - - def update_goals(user, data): """update the user's goals from import data""" @@ -340,14 +651,6 @@ def update_goals(user, data): models.AnnualGoal.objects.create(**goal) -@app.task(queue=IMPORTS, base=SubTask) -def update_goals_task(job_id): - """wrapper task for user's goals import""" - parent_job = BookwyrmImportJob.objects.get(id=job_id) - - return update_goals(parent_job.user, parent_job.import_data.get("goals")) - - def upsert_saved_lists(user, values): """Take a list of remote ids and add as saved lists""" @@ -357,68 +660,90 @@ def upsert_saved_lists(user, values): user.saved_lists.add(book_list) -@app.task(queue=IMPORTS, base=SubTask) -def upsert_saved_lists_task(job_id): - """wrapper task for user's saved lists import""" - parent_job = BookwyrmImportJob.objects.get(id=job_id) - - return upsert_saved_lists( - parent_job.user, parent_job.import_data.get("saved_lists") - ) - - -def upsert_follows(user, values): - """Take a list of remote ids and add as follows""" - - for remote_id in values: - followee = activitypub.resolve_remote_id(remote_id, models.User) - if followee: - (follow_request, created,) = models.UserFollowRequest.objects.get_or_create( - user_subject=user, - user_object=followee, - ) - - if not created: - # this request probably failed to connect with the remote - # and should save to trigger a re-broadcast - follow_request.save() - +@app.task(queue=IMPORTS, base=UserImportSubTask) +def import_user_relationship_task(**kwargs): + """import a user follow or block from an import file""" -@app.task(queue=IMPORTS, base=SubTask) -def upsert_follows_task(job_id): - """wrapper task for user's follows import""" - parent_job = BookwyrmImportJob.objects.get(id=job_id) + task = UserImportRelationship.objects.get(id=kwargs["child_id"]) + job = task.parent_job - return upsert_follows(parent_job.user, parent_job.import_data.get("follows")) + try: + if task.relationship == "follow": + + followee = activitypub.resolve_remote_id(task.remote_id, models.User) + if followee: + ( + follow_request, + created, + ) = models.UserFollowRequest.objects.get_or_create( + user_subject=job.user, + user_object=followee, + ) + if not created: + # this request probably failed to connect with the remote + # and should save to trigger a re-broadcast + follow_request.save() -def upsert_user_blocks(user, user_ids): - """block users""" + task.complete_job() - for user_id in user_ids: - user_object = activitypub.resolve_remote_id(user_id, models.User) - if user_object: - exists = models.UserBlocks.objects.filter( - user_subject=user, user_object=user_object - ).exists() - if not exists: - models.UserBlocks.objects.create( - user_subject=user, user_object=user_object + else: + logger.error( + "Could not resolve import user %s follow task %s", + task.remote_id, + task.id, + ) + task.fail_reason = _("connection_error") + task.save(update_fields=["fail_reason"]) + task.set_status("failed") + + elif task.relationship == "block": + + user_object = activitypub.resolve_remote_id(task.remote_id, models.User) + if user_object: + exists = models.UserBlocks.objects.filter( + user_subject=job.user, user_object=user_object + ).exists() + if not exists: + models.UserBlocks.objects.create( + user_subject=job.user, user_object=user_object + ) + # remove the blocked users's lists from the groups + models.List.remove_from_group(job.user, user_object) + # remove the blocked user from all blocker's owned groups + models.GroupMember.remove(job.user, user_object) + + task.complete_job() + + else: + logger.error( + "Could not resolve user %s block task %s", task.remote_id, task.id ) - # remove the blocked users's lists from the groups - models.List.remove_from_group(user, user_object) - # remove the blocked user from all blocker's owned groups - models.GroupMember.remove(user, user_object) + task.fail_reason = _("connection_error") + task.save(update_fields=["fail_reason"]) + task.set_status("failed") + else: + logger.error( + "Invalid relationship type %s specified in user import task %s", + task.relationship, + task.id, + ) + task.fail_reason = _("invalid_relationship") + task.save(update_fields=["fail_reason"]) + task.set_status("failed") -@app.task(queue=IMPORTS, base=SubTask) -def upsert_user_blocks_task(job_id): - """wrapper task for user's blocks import""" - parent_job = BookwyrmImportJob.objects.get(id=job_id) + except Exception as err: # pylint: disable=broad-except + logger.error( + "User Import Relationship Task %s Failed with error: %s", task.id, err + ) + task.fail_reason = _("Unkown error importing relationship") + task.save(update_fields=["fail_reason"]) + task.set_status("failed") - return upsert_user_blocks( - parent_job.user, parent_job.import_data.get("blocked_users") - ) + +# utilities +########### def update_followers_address(user, field): @@ -433,19 +758,21 @@ def update_followers_address(user, field): def is_alias(user, remote_id): - """check that the user is listed as movedTo or also_known_as - in the remote user's profile""" + """check that the user is listed as moved_to + or also_known_as in the remote user's profile""" + + if not remote_id: + return False remote_user = activitypub.resolve_remote_id( remote_id=remote_id, model=models.User, save=False ) if remote_user: - - if remote_user.moved_to: + if getattr(remote_user, "moved_to", None) is not None: return user.remote_id == remote_user.moved_to - if remote_user.also_known_as: + if hasattr(remote_user, "also_known_as"): return user in remote_user.also_known_as.all() return False diff --git a/bookwyrm/models/connector.py b/bookwyrm/models/connector.py index f4b5be04c0..d62f97f02e 100644 --- a/bookwyrm/models/connector.py +++ b/bookwyrm/models/connector.py @@ -1,4 +1,6 @@ """ manages interfaces with external sources of book data """ +from typing import Optional + from django.db import models from bookwyrm.connectors.settings import CONNECTORS @@ -29,3 +31,34 @@ class Connector(BookWyrmModel): def __str__(self): return f"{self.identifier} ({self.id})" + + def deactivate(self, reason: Optional[str] = None) -> None: + """Make an active connector inactive. We do not delete connectors + because they have books and authors associated with them.""" + + self.active = False + self.deactivation_reason = reason + self.save(update_fields=["active", "deactivation_reason"]) + + def activate(self) -> None: + """Make an inactive connector active again""" + + self.active = True + self.deactivation_reason = None + self.save(update_fields=["active", "deactivation_reason"]) + + def change_priority(self, priority: int) -> None: + """Change the priority value for a connector + This determines the order they appear in book search""" + + self.priority = priority + self.save(update_fields=["priority"]) + + def update(self) -> None: + """Update the settings for this connector. e.g. if the + API endpoints change.""" + + # example + # if self.identifier == "openlibrary.org": + # self.isbn_search_url = "https://openlibrary.org/search.json?isbn=" + # self.save(update_fields=["isbn_search_url"]) diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 6643bdc193..3cf455603e 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -25,7 +25,7 @@ PartialDateModel, from_partial_isoformat, ) -from bookwyrm.settings import MEDIA_FULL_URL +from bookwyrm.settings import MEDIA_FULL_URL, DATA_UPLOAD_MAX_MEMORY_SIZE def validate_remote_id(value): @@ -86,7 +86,9 @@ def set_field_from_activity( raise value = getattr(data, "actor") formatted = self.field_from_activity( - value, allow_external_connections=allow_external_connections + value, + allow_external_connections=allow_external_connections, + trigger=instance, ) if formatted is None or formatted is MISSING or formatted == {}: return False @@ -128,7 +130,7 @@ def field_to_activity(self, value): return value # pylint: disable=unused-argument - def field_from_activity(self, value, allow_external_connections=True): + def field_from_activity(self, value, allow_external_connections=True, trigger=None): """formatter to convert activitypub into a model value""" if value and hasattr(self, "activitypub_wrapper"): value = value.get(self.activitypub_wrapper) @@ -150,7 +152,9 @@ def __init__(self, *args, load_remote=True, **kwargs): self.load_remote = load_remote super().__init__(*args, **kwargs) - def field_from_activity(self, value, allow_external_connections=True): + def field_from_activity(self, value, allow_external_connections=True, trigger=None): + """trigger: the object that triggered this deserialization. + For example the Edition for which self is the parent Work""" if not value: return None @@ -160,7 +164,7 @@ def field_from_activity(self, value, allow_external_connections=True): # only look in the local database return related_model.find_existing(value.serialize()) # this is an activitypub object, which we can deserialize - return value.to_model(model=related_model) + return value.to_model(model=related_model, trigger=trigger) try: # make sure the value looks like a remote id validate_remote_id(value) @@ -193,8 +197,7 @@ class UsernameField(ActivitypubFieldMixin, models.CharField): def __init__(self, activitypub_field="preferredUsername", **kwargs): self.activitypub_field = activitypub_field - # I don't totally know why pylint is mad at this, but it makes it work - super(ActivitypubFieldMixin, self).__init__( # pylint: disable=bad-super-call + super(ActivitypubFieldMixin, self).__init__( _("username"), max_length=150, unique=True, @@ -234,7 +237,6 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField): def __init__(self, *args, **kwargs): super().__init__(*args, max_length=255, choices=PrivacyLevels, default="public") - # pylint: disable=invalid-name def set_field_from_activity( self, instance, data, overwrite=True, allow_external_connections=True ): @@ -276,7 +278,6 @@ def set_activity_from_field(self, activity, instance): if hasattr(instance, "mention_users"): mentions = [u.remote_id for u in instance.mention_users.all()] # this is a link to the followers list - # pylint: disable=protected-access followers = instance.user.followers_url if instance.privacy == "public": activity["to"] = [self.public] @@ -292,7 +293,10 @@ def set_activity_from_field(self, activity, instance): activity["cc"] = [] -class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey): +class ForeignKey( # pylint: disable=abstract-method + ActivitypubRelatedFieldMixin, + models.ForeignKey, +): """activitypub-aware foreign key field""" def field_to_activity(self, value): @@ -301,7 +305,9 @@ def field_to_activity(self, value): return value.remote_id -class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField): +class OneToOneField( # pylint: disable=abstract-method + ActivitypubRelatedFieldMixin, models.OneToOneField +): """activitypub-aware foreign key field""" def field_to_activity(self, value): @@ -310,7 +316,9 @@ def field_to_activity(self, value): return value.to_activity() -class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): +class ManyToManyField( # pylint: disable=abstract-method + ActivitypubFieldMixin, models.ManyToManyField +): """activitypub-aware many to many field""" def __init__(self, *args, link_only=False, **kwargs): @@ -339,7 +347,7 @@ def field_to_activity(self, value): return f"{value.instance.remote_id}/{self.name}" return [i.remote_id for i in value.all()] - def field_from_activity(self, value, allow_external_connections=True): + def field_from_activity(self, value, allow_external_connections=True, trigger=None): if value is None or value is MISSING: return None if not isinstance(value, list): @@ -361,7 +369,7 @@ def field_from_activity(self, value, allow_external_connections=True): return items -class TagField(ManyToManyField): +class TagField(ManyToManyField): # pylint: disable=abstract-method """special case of many to many that uses Tags""" def __init__(self, *args, **kwargs): @@ -389,7 +397,7 @@ def field_to_activity(self, value): ) return tags - def field_from_activity(self, value, allow_external_connections=True): + def field_from_activity(self, value, allow_external_connections=True, trigger=None): if not isinstance(value, list): # GoToSocial DMs and single-user mentions are # sent as objects, not as an array of objects @@ -430,6 +438,16 @@ class ClearableFileInputWithWarning(ClearableFileInput): template_name = "widgets/clearable_file_input_with_warning.html" + def get_context(self, name, value, attrs): + context = super().get_context(name, value, attrs) + context["widget"]["attrs"].update( + { + "data-max-upload": DATA_UPLOAD_MAX_MEMORY_SIZE, + "max_mb": DATA_UPLOAD_MAX_MEMORY_SIZE >> 20, + } + ) + return context + class CustomImageField(DjangoImageField): """overwrites image field for form""" @@ -444,7 +462,7 @@ def __init__(self, *args, alt_field=None, **kwargs): self.alt_field = alt_field super().__init__(*args, **kwargs) - # pylint: disable=arguments-differ,arguments-renamed,too-many-arguments + # pylint: disable=arguments-renamed,too-many-arguments def set_field_from_activity( self, instance, data, save=True, overwrite=True, allow_external_connections=True ): @@ -484,7 +502,7 @@ def field_to_activity(self, value, alt=None): return activitypub.Image(url=url, name=alt) - def field_from_activity(self, value, allow_external_connections=True): + def field_from_activity(self, value, allow_external_connections=True, trigger=None): image_slug = value # when it's an inline image (User avatar/icon, Book cover), it's a json # blob, but when it's an attached image, it's just a url @@ -541,7 +559,7 @@ def field_to_activity(self, value): return None return value.isoformat() - def field_from_activity(self, value, allow_external_connections=True): + def field_from_activity(self, value, allow_external_connections=True, trigger=None): missing_fields = datetime(1970, 1, 1) # "2022-10" => "2022-10-01" try: date_value = dateutil.parser.parse(value, default=missing_fields) @@ -559,7 +577,7 @@ class PartialDateField(ActivitypubFieldMixin, PartialDateModel): def field_to_activity(self, value) -> str: return value.partial_isoformat() if value else None - def field_from_activity(self, value, allow_external_connections=True): + def field_from_activity(self, value, allow_external_connections=True, trigger=None): # pylint: disable=no-else-return try: return from_partial_isoformat(value) @@ -587,7 +605,7 @@ def field_from_activity(self, value, allow_external_connections=True): class HtmlField(ActivitypubFieldMixin, models.TextField): """a text field for storing html""" - def field_from_activity(self, value, allow_external_connections=True): + def field_from_activity(self, value, allow_external_connections=True, trigger=None): if not value or value == MISSING: return None return clean(value) diff --git a/bookwyrm/models/housekeeping.py b/bookwyrm/models/housekeeping.py new file mode 100644 index 0000000000..e89887785e --- /dev/null +++ b/bookwyrm/models/housekeeping.py @@ -0,0 +1,97 @@ +""" cleanup tasks """ +import math +from datetime import datetime, timedelta, timezone + +from django.db.models import DateTimeField, IntegerField + +from bookwyrm.tasks import app, MISC +from bookwyrm import models +from bookwyrm.models.job import ParentJob, ParentTask + + +class CleanUpUserExportFilesJob(ParentJob): + """A job to clean up old import and export files""" + + expiry_date = DateTimeField() + tasks = IntegerField(default=0) + completed_tasks = IntegerField(default=0) + + @property + def percent_complete(self): + """How far along?""" + + if not self.tasks: + return 0 + return math.floor(self.completed_tasks / self.tasks * 100) + + def start_job(self): + """schedule the tasks""" + + self.set_status("active") + + export_jobs = models.BookwyrmExportJob.objects.filter( + complete=True, updated_date__lt=self.expiry_date + ) + + import_jobs = models.BookwyrmImportJob.objects.filter( + complete=True, updated_date__lt=self.expiry_date + ) + + for export in export_jobs: + if export.export_data.name: + self.tasks += 1 + self.save(update_fields=["tasks"]) + delete_user_export_file_task.delay(job_id=self.id, export_id=export.id) + + for job in import_jobs: + if job.archive_file.name: + self.tasks += 1 + self.save(update_fields=["tasks"]) + delete_user_export_file_task.delay(job_id=self.id, import_id=job.id) + + if self.tasks == 0: + self.complete_job() + + +class CleanUpExportsTask(ParentTask): + """Task to delete expired user export files""" + + # pylint: disable=too-many-arguments, unused-argument, no-self-use + def after_return(self, status, retval, task_id, args, kwargs, einfo): + """Handler called after the task returns""" + + job = CleanUpUserExportFilesJob.objects.get(id=kwargs["job_id"]) + job.completed_tasks += 1 + job.save(update_fields=["completed_tasks"]) + + if job.completed_tasks == job.tasks: + job.complete_job() + + +@app.task(queue=MISC, base=CleanUpExportsTask) +def delete_user_export_file_task(**kwargs): + """A task to delete a specific export/import file""" + + if kwargs.get("import_id"): + file = models.BookwyrmImportJob.objects.get(id=kwargs["import_id"]) + file.archive_file.delete() + + else: + export_id = kwargs.get("export_id") + if export_id: + file = models.BookwyrmExportJob.objects.get(id=export_id) + file.export_data.delete() + + +@app.task(queue=MISC) +def start_export_deletions(**kwargs): + """trigger the job from scheduler""" + + user = models.User.objects.get(id=kwargs["user"]) + site = models.SiteSettings.objects.get() + hours = site.export_files_lifetime_hours + + expiry_date = datetime.now(timezone.utc) - timedelta(hours=hours) + job = CleanUpUserExportFilesJob.objects.create(user=user, expiry_date=expiry_date) + + job.start_job() diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index f5d86ad2e8..5a6ba3f512 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -4,6 +4,7 @@ import re import dateutil.parser +from django.core.exceptions import ObjectDoesNotExist from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -59,6 +60,7 @@ class ImportJob(models.Model): created_date = models.DateTimeField(default=timezone.now) updated_date = models.DateTimeField(default=timezone.now) include_reviews: bool = models.BooleanField(default=True) + create_shelves: bool = models.BooleanField(default=True) mappings = models.JSONField() source = models.CharField(max_length=100) privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels) @@ -245,11 +247,26 @@ def shelf(self): """the goodreads shelf field""" return self.normalized_data.get("shelf") + @property + def shelf_name(self): + """the goodreads shelf field""" + return self.normalized_data.get("shelf_name") + @property def review(self): """a user-written review, to be imported with the book data""" return self.normalized_data.get("review_body") + @property + def review_name(self): + """a user-written review name, to be imported with the book data""" + return self.normalized_data.get("review_name") + + @property + def review_published(self): + """date the review was published - included in BookWyrm export csv""" + return self.normalized_data.get("review_published", None) + @property def rating(self): """x/5 star rating for a book""" @@ -352,7 +369,7 @@ def import_item_task(item_id): try: item.resolve() - except Exception as err: # pylint: disable=broad-except + except Exception as err: item.fail_reason = _("Error loading book") item.save() item.update_job() @@ -368,7 +385,7 @@ def import_item_task(item_id): item.update_job() -def handle_imported_book(item): +def handle_imported_book(item): # pylint: disable=too-many-branches """process a csv and then post about it""" job = item.job if job.complete: @@ -385,13 +402,31 @@ def handle_imported_book(item): item.book = item.book.edition existing_shelf = ShelfBook.objects.filter(book=item.book, user=user).exists() + if job.create_shelves and item.shelf and not existing_shelf: + # shelve the book if it hasn't been shelved already - # shelve the book if it hasn't been shelved already - if item.shelf and not existing_shelf: - desired_shelf = Shelf.objects.get(identifier=item.shelf, user=user) shelved_date = item.date_added or timezone.now() + shelfname = getattr(item, "shelf_name", item.shelf) + + try: + shelf = Shelf.objects.get(name=shelfname, user=user) + except ObjectDoesNotExist: + try: + shelf = Shelf.objects.get(identifier=item.shelf, user=user) + except ObjectDoesNotExist: + + shelf = Shelf.objects.create( + user=user, + identifier=item.shelf, + name=shelfname, + privacy=job.privacy, + ) + ShelfBook( - book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date + book=item.book, + shelf=shelf, + user=user, + shelved_date=shelved_date, ).save(priority=IMPORT_TRIGGERED) for read in item.reads: @@ -408,19 +443,25 @@ def handle_imported_book(item): read.save() if job.include_reviews and (item.rating or item.review) and not item.linked_review: - # we don't know the publication date of the review, - # but "now" is a bad guess - published_date_guess = item.date_read or item.date_added + # we don't necessarily know the publication date of the review, + # but "now" is a bad guess unless we have no choice + + published_date_guess = ( + item.review_published or item.date_read or item.date_added or timezone.now() + ) if item.review: + # pylint: disable=consider-using-f-string review_title = "Review of {!r} on {!r}".format( item.book.title, job.source, ) + review_name = getattr(item, "review_name", review_title) + review = Review.objects.filter( user=user, book=item.book, - name=review_title, + name=review_name, rating=item.rating, published_date=published_date_guess, ).first() @@ -428,7 +469,7 @@ def handle_imported_book(item): review = Review( user=user, book=item.book, - name=review_title, + name=review_name, content=item.review, rating=item.rating, published_date=published_date_guess, diff --git a/bookwyrm/models/job.py b/bookwyrm/models/job.py index 5a26535718..ff4895dc60 100644 --- a/bookwyrm/models/job.py +++ b/bookwyrm/models/job.py @@ -29,6 +29,7 @@ class Status(models.TextChoices): status = models.CharField( max_length=50, choices=Status.choices, default=Status.PENDING, null=True ) + fail_reason = models.TextField(null=True) class Meta: """Make it abstract""" @@ -133,7 +134,8 @@ def __terminate_pending_child_jobs(self): tasks = self.pending_child_jobs.filter(task_id__isnull=False).values_list( "task_id", flat=True ) - app.control.revoke(list(tasks)) + tasklist = [str(task) for task in list(tasks)] + app.control.revoke(tasklist) self.pending_child_jobs.update(status=self.Status.STOPPED) @@ -208,7 +210,7 @@ def before_start( job.task_id = task_id job.save(update_fields=["task_id"]) - if kwargs["no_children"]: + if kwargs.get("no_children"): job.set_status(ChildJob.Status.ACTIVE) def on_success( @@ -233,7 +235,7 @@ def on_success( None: The return value of this handler is ignored. """ - if kwargs["no_children"]: + if kwargs.get("no_children"): job = ParentJob.objects.get(id=kwargs["job_id"]) job.complete_job() @@ -247,7 +249,7 @@ class SubTask(app.Task): """ def before_start( - self, task_id, *args, **kwargs + self, task_id, args, kwargs ): # pylint: disable=no-self-use, unused-argument """Handler called before the task starts. Override. @@ -271,7 +273,7 @@ def before_start( child_job.set_status(ChildJob.Status.ACTIVE) def on_success( - self, retval, task_id, *args, **kwargs + self, retval, task_id, args, kwargs ): # pylint: disable=no-self-use, unused-argument """Run by the worker if the task executes successfully. Override. diff --git a/bookwyrm/models/readthrough.py b/bookwyrm/models/readthrough.py index 7700b4a87d..670b35821f 100644 --- a/bookwyrm/models/readthrough.py +++ b/bookwyrm/models/readthrough.py @@ -59,7 +59,7 @@ class Meta: constraints = [ models.CheckConstraint( - check=Q(finish_date__gte=F("start_date")), name="chronology" + condition=Q(finish_date__gte=F("start_date")), name="chronology" ) ] ordering = ("-start_date",) diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index 745ff78b67..c4344d812c 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -57,7 +57,7 @@ class Meta: fields=["user_subject", "user_object"], name="%(class)s_unique" ), models.CheckConstraint( - check=~models.Q(user_subject=models.F("user_object")), + condition=~models.Q(user_subject=models.F("user_object")), name="%(class)s_no_self", ), ] @@ -135,7 +135,7 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship): status = "follow_request" activity_serializer = activitypub.Follow - def save(self, *args, broadcast=True, **kwargs): # pylint: disable=arguments-differ + def save(self, *args, broadcast=True, **kwargs): """make sure the follow or block relationship doesn't already exist""" # if there's a request for a follow that already exists, accept it # without changing the local database state diff --git a/bookwyrm/models/session.py b/bookwyrm/models/session.py new file mode 100644 index 0000000000..3de199d299 --- /dev/null +++ b/bookwyrm/models/session.py @@ -0,0 +1,53 @@ +""" functions for managing user sessions """ +from importlib import import_module +import ua_parser + +from django.conf import settings +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from bookwyrm.models import User + +SessionStore = import_module(settings.SESSION_ENGINE).SessionStore + + +class UserSession(models.Model): + """A session for a logged-in user. We only use this model to save info about + a logged-in session when the user first logs in, so users can remove sessions + e.g. for devices they have lost.""" + + user = models.ForeignKey("User", on_delete=models.CASCADE, related_name="sessions") + session_key = models.CharField(max_length=40) + created_date = models.DateTimeField(auto_now_add=True) + operating_system = models.CharField(max_length=50) + browser_type = models.CharField(max_length=50) + # account for full IPv4-as-IPv6 strings, in case we need to + ip_address = models.CharField(max_length=45) + + def logout(self): + """log out the session and delete the UserSession""" + + s = SessionStore(session_key=self.session_key) + s.delete() + self.delete() + + +def create_user_session( + user_id: int, session_key: str, ip_address: str, agent_string: str = "" +): + """create a session object""" + + user = User.objects.get(id=user_id) + parsed = ua_parser.parse(agent_string) + unknown = _("Unknown") + system = getattr(parsed.os, "family", str(unknown)) + browser = getattr(parsed.user_agent, "family", str(unknown)) + + sess = UserSession( + user=user, + session_key=session_key, + operating_system=system, + browser_type=browser, + ip_address=ip_address, + ) + sess.save() diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index 6c2a73422b..31e77e7e2b 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -103,6 +103,7 @@ class SiteSettings(SiteModel): import_limit_reset = models.IntegerField(default=0) user_exports_enabled = models.BooleanField(default=False) user_import_time_limit = models.IntegerField(default=48) + export_files_lifetime_hours = models.IntegerField(default=72) field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"]) diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 9dc60e6477..2b357ebd22 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -98,7 +98,7 @@ def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): self.thread_id = self.id super().save(broadcast=False, update_fields=["thread_id"]) - def delete(self, *args, **kwargs): # pylint: disable=unused-argument + def delete(self, *args, **kwargs): """ "delete" a status""" if hasattr(self, "boosted_status"): # okay but if it's a boost really delete it @@ -213,7 +213,7 @@ def to_replies(self, **kwargs): **kwargs, ).serialize() - def to_activity_dataclass(self, pure=False): # pylint: disable=arguments-differ + def to_activity_dataclass(self, pure=False): """return tombstone if the status is deleted""" if self.deleted: return activitypub.Tombstone( diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index ed5c59e9c8..2f411fb5ad 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -1,5 +1,6 @@ """ database schema for user data """ import datetime +from importlib import import_module import re import zoneinfo from typing import Optional, Iterable @@ -7,6 +8,7 @@ from uuid import uuid4 from django.apps import apps +from django.conf import settings from django.contrib.auth.models import AbstractUser from django.contrib.postgres.fields import ArrayField as DjangoArrayField from django.core.exceptions import PermissionDenied, ObjectDoesNotExist @@ -31,6 +33,7 @@ from .federated_server import FederatedServer from . import fields +SessionStore = import_module(settings.SESSION_ENGINE).SessionStore FeedFilterChoices = [ ("review", _("Reviews")), @@ -141,7 +144,6 @@ class User(OrderedCollectionPageMixin, AbstractUser): hide_follows = fields.BooleanField(default=False) # migration fields - moved_to = fields.RemoteIdField( null=True, unique=False, activitypub_field="movedTo", deduplication_field=False ) @@ -158,6 +160,7 @@ class User(OrderedCollectionPageMixin, AbstractUser): show_suggested_users = models.BooleanField(default=True) discoverable = fields.BooleanField(default=False) show_guided_tour = models.BooleanField(default=True) + show_ratings = models.BooleanField(default=True) # feed options feed_status_types = DjangoArrayField( @@ -305,9 +308,10 @@ def to_following_activity(self, **kwargs): return self.to_ordered_collection( self.following.order_by("-updated_date").all(), remote_id=remote_id, + collection_only=True, id_only=True, **kwargs, - ) + ).serialize() def to_followers_activity(self, **kwargs): """activitypub followers list""" @@ -315,9 +319,10 @@ def to_followers_activity(self, **kwargs): return self.to_ordered_collection( self.followers.order_by("-updated_date").all(), remote_id=remote_id, + collection_only=True, id_only=True, **kwargs, - ) + ).serialize() def to_activity(self, **kwargs): """override default AP serializer to add context object @@ -409,7 +414,6 @@ def save(self, *args, update_fields: Optional[Iterable[str]] = None, **kwargs): def delete(self, *args, **kwargs): """We don't actually delete the database entry""" - # pylint: disable=attribute-defined-outside-init self.is_active = False self.allow_reactivation = False self.is_deleted = True @@ -452,7 +456,6 @@ def erase_user_statuses(self, broadcast=True): def deactivate(self): """Disable the user but allow them to reactivate""" - # pylint: disable=attribute-defined-outside-init self.is_active = False self.deactivation_reason = "self_deactivation" self.allow_reactivation = True @@ -460,7 +463,6 @@ def deactivate(self): def reactivate(self): """Now you want to come back, huh?""" - # pylint: disable=attribute-defined-outside-init if not self.allow_reactivation: return self.is_active = True @@ -512,6 +514,15 @@ def raise_not_editable(self, viewer): return raise PermissionDenied() + def refresh_user_sessions(self): + """Check sessions still exist + We delete them on logout but not when sessions expire""" + + cache_session = SessionStore() + for sess in self.sessions.all(): + if not cache_session.exists(session_key=sess.session_key): + sess.delete() + class KeyPair(ActivitypubMixin, BookWyrmModel): """public and private keys for a user""" diff --git a/bookwyrm/preview_images.py b/bookwyrm/preview_images.py index a213490abf..66d2e2d4b9 100644 --- a/bookwyrm/preview_images.py +++ b/bookwyrm/preview_images.py @@ -420,7 +420,6 @@ def save_and_cleanup(image, instance=None): return True -# pylint: disable=invalid-name @app.task(queue=IMAGES) def generate_site_preview_image_task(): """generate preview_image for the website""" @@ -445,7 +444,6 @@ def generate_site_preview_image_task(): save_and_cleanup(image, instance=site) -# pylint: disable=invalid-name @app.task(queue=IMAGES) def generate_edition_preview_image_task(book_id): """generate preview_image for a book""" diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index c2b9c8e859..008e183843 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -1,4 +1,5 @@ -""" bookwyrm settings and configuration """ +"""bookwyrm settings and configuration""" + import os from typing import AnyStr @@ -29,9 +30,7 @@ PAGE_LENGTH = env.int("PAGE_LENGTH", 15) DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English") -# TODO: extend maximum age to 1 year once termination of active sessions -# is implemented (see bookwyrm-social#2278, bookwyrm-social#3082). -SESSION_COOKIE_AGE = env.int("SESSION_COOKIE_AGE", 3600 * 24 * 30) # 1 month +SESSION_COOKIE_AGE = env.int("SESSION_COOKIE_AGE", 3600 * 24 * 365) # One year ...ish JS_CACHE = "8a89cad7" @@ -81,12 +80,14 @@ # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = env.bool("DEBUG", True) -USE_HTTPS = env.bool("USE_HTTPS", not DEBUG) +DEBUG = env.bool("DEBUG", False) # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env("SECRET_KEY") -if not DEBUG and SECRET_KEY == "7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr": +SECRET_KEY = env("SECRET_KEY", None) +if not DEBUG and SECRET_KEY in [ + None, + "7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr", +]: raise ImproperlyConfigured("You must change the SECRET_KEY env variable") ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", ["*"]) @@ -274,7 +275,7 @@ DATABASES = { "default": { - "ENGINE": "django.db.backends.postgresql_psycopg2", + "ENGINE": "django.db.backends.postgresql", "NAME": env("POSTGRES_DB", "bookwyrm"), "USER": env("POSTGRES_USER", "bookwyrm"), "PASSWORD": env("POSTGRES_PASSWORD", "bookwyrm"), @@ -293,15 +294,15 @@ AUTH_PASSWORD_VALIDATORS = [ { "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", - 'OPTIONS': { - 'max_similarity': .9, - } + "OPTIONS": { + "max_similarity": 0.9, + }, }, { "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", - 'OPTIONS': { - 'min_length': 16, - } + "OPTIONS": { + "min_length": 16, + }, }, { "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", @@ -342,8 +343,14 @@ ] LANGUAGE_ARTICLES = { - "English": {"the", "a", "an"}, - "Español (Spanish)": {"un", "una", "unos", "unas", "el", "la", "los", "las"}, + "en-us": { + "variants": ["english", "anglais", "inglés", "englanti"], + "articles": {"the", "a", "an"}, + }, + "es-es": { + "variants": ["spanish", "español", "espagnol", "espanja"], + "articles": {"un", "una", "unos", "unas", "el", "la", "los", "las"}, + }, } TIME_ZONE = "UTC" @@ -359,18 +366,24 @@ PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) CSP_ADDITIONAL_HOSTS = env.list("CSP_ADDITIONAL_HOSTS", []) +PORT = env.int("PORT", 80) -PROTOCOL = "http" -if USE_HTTPS: +if DOMAIN == "localhost": + # only run insecurely when testing on localhost + PROTOCOL = "http" + SESSION_COOKIE_SECURE = False + CSRF_COOKIE_SECURE = False + NETLOC = f"{DOMAIN}:{PORT}" +else: + # if we are not running on localhost, everything should be using https + # PORT should only be used to pass traffic to a reverse-proxy, not exposed externally + # so we don't need it here PROTOCOL = "https" SESSION_COOKIE_SECURE = True CSRF_COOKIE_SECURE = True - -PORT = env.int("PORT", 443 if USE_HTTPS else 80) -if (USE_HTTPS and PORT == 443) or (not USE_HTTPS and PORT == 80): NETLOC = DOMAIN -else: - NETLOC = f"{DOMAIN}:{PORT}" + SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + BASE_URL = f"{PROTOCOL}://{NETLOC}" CSRF_TRUSTED_ORIGINS = [BASE_URL] @@ -380,7 +393,6 @@ USE_S3 = env.bool("USE_S3", False) USE_AZURE = env.bool("USE_AZURE", False) -S3_SIGNED_URL_EXPIRY = env.int("S3_SIGNED_URL_EXPIRY", 900) if USE_S3: # AWS settings @@ -390,7 +402,7 @@ AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN", None) AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME", "") AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL", None) - AWS_DEFAULT_ACL = "public-read" + AWS_DEFAULT_ACL = env("AWS_DEFAULT_ACL", "public-read") AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"} AWS_S3_URL_PROTOCOL = env("AWS_S3_URL_PROTOCOL", f"{PROTOCOL}:") # Storages @@ -399,7 +411,7 @@ "BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": { "location": "images", - "default_acl": "public-read", + "default_acl": AWS_DEFAULT_ACL, "file_overwrite": False, }, }, @@ -407,15 +419,14 @@ "BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": { "location": "static", - "default_acl": "public-read", + "default_acl": AWS_DEFAULT_ACL, }, }, - "exports": { + "sass_processor": { "BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": { - "location": "images", - "default_acl": None, - "file_overwrite": False, + "location": "static", + "default_acl": AWS_DEFAULT_ACL, }, }, } @@ -461,9 +472,6 @@ "location": "static", }, }, - "exports": { - "BACKEND": None, # not implemented yet - }, } # Azure Static settings STATIC_LOCATION = "static" @@ -489,12 +497,6 @@ "staticfiles": { "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", }, - "exports": { - "BACKEND": "django.core.files.storage.FileSystemStorage", - "OPTIONS": { - "location": "exports", - }, - }, } # Static settings STATIC_URL = "/static/" @@ -506,6 +508,39 @@ CSP_DEFAULT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS CSP_SCRIPT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS +# storage of user export and import files +USE_S3_FOR_EXPORTS = env.bool("USE_S3_FOR_EXPORTS", False) + +# Must use a different bucket for exports +# This ensures we can secure use import/export files +# for S3 services without ACL (e.g. Backblaze B2 or Cloudflare R2) +S3_SIGNED_URL_EXPIRY = env.int("S3_SIGNED_URL_EXPIRY", 900) +if USE_S3_FOR_EXPORTS: + STORAGES["exports"] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + "location": "exports", + "default_acl": "private", + "file_overwrite": False, + "object_parameters": {"CacheControl": "max-age=86400"}, + "access_key": env("EXPORTS_ACCESS_KEY_ID", env("AWS_ACCESS_KEY_ID")), + "secret_key": env( + "EXPORTS_SECRET_ACCESS_KEY", env("AWS_SECRET_ACCESS_KEY") + ), + "region_name": env("EXPORTS_S3_REGION_NAME", env("AWS_S3_REGION_NAME")), + "endpoint_url": env("EXPORTS_S3_ENDPOINT_URL", env("AWS_S3_ENDPOINT_URL")), + "custom_domain": env("EXPORTS_S3_CUSTOM_DOMAIN", None), + "bucket_name": env("EXPORTS_STORAGE_BUCKET_NAME"), + }, + } +else: + STORAGES["exports"] = { + "BACKEND": "django.core.files.storage.FileSystemStorage", + "OPTIONS": { + "location": "exports", + }, + } + CSP_INCLUDE_NONCE_IN = ["script-src"] OTEL_EXPORTER_OTLP_ENDPOINT = env("OTEL_EXPORTER_OTLP_ENDPOINT", None) @@ -516,10 +551,6 @@ TWO_FACTOR_LOGIN_MAX_SECONDS = env.int("TWO_FACTOR_LOGIN_MAX_SECONDS", 60) TWO_FACTOR_LOGIN_VALIDITY_WINDOW = env.int("TWO_FACTOR_LOGIN_VALIDITY_WINDOW", 2) -HTTP_X_FORWARDED_PROTO = env.bool("SECURE_PROXY_SSL_HEADER", False) -if HTTP_X_FORWARDED_PROTO: - SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") - # Instance Actor for signing GET requests to "secure mode" # Mastodon servers. # Do not change this setting unless you already have an existing diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py index 08780b7317..f59367b51e 100644 --- a/bookwyrm/signatures.py +++ b/bookwyrm/signatures.py @@ -6,7 +6,7 @@ from Crypto import Random from Crypto.PublicKey import RSA -from Crypto.Signature import pkcs1_15 # pylint: disable=no-name-in-module +from Crypto.Signature import pkcs1_15 from Crypto.Hash import SHA256 MAX_SIGNATURE_AGE = 300 @@ -84,7 +84,6 @@ def __init__(self, key_id, headers, signature): self.headers = headers self.signature = signature - # pylint: disable=invalid-name @classmethod def parse(cls, request): """extract and parse a signature from an http request""" diff --git a/bookwyrm/static/js/bookwyrm.js b/bookwyrm/static/js/bookwyrm.js index a2351a98cb..3088165d6f 100644 --- a/bookwyrm/static/js/bookwyrm.js +++ b/bookwyrm/static/js/bookwyrm.js @@ -3,7 +3,6 @@ let BookWyrm = new (class { constructor() { - this.MAX_FILE_SIZE_BYTES = 10 * 1000000; this.initOnDOMLoaded(); this.initRecurringTasks(); this.initEventListeners(); @@ -14,6 +13,10 @@ let BookWyrm = new (class { .querySelectorAll("[data-controls]") .forEach((button) => button.addEventListener("click", this.toggleAction.bind(this))); + document + .querySelectorAll("[data-disappear]") + .forEach((button) => button.addEventListener("click", this.hideSelf.bind(this))); + document .querySelectorAll(".interaction") .forEach((button) => button.addEventListener("submit", this.interact.bind(this))); @@ -181,6 +184,18 @@ let BookWyrm = new (class { this.addRemoveClass(visible, "is-hidden", true); } + /** + * Hide the element you just clicked + * + * @param {Event} event + * @return {undefined} + */ + hideSelf(event) { + let trigger = event.currentTarget; + + this.addRemoveClass(trigger, "is-hidden", true); + } + /** * Execute actions on targets based on triggers. * @@ -380,13 +395,14 @@ let BookWyrm = new (class { } disableIfTooLarge(eventOrElement) { - const { addRemoveClass, MAX_FILE_SIZE_BYTES } = this; + const { addRemoveClass } = this; const element = eventOrElement.currentTarget || eventOrElement; + const limit = element.dataset.maxUpload; const submits = element.form.querySelectorAll('[type="submit"]'); const warns = element.parentElement.querySelectorAll(".file-too-big"); const isTooBig = - element.files && element.files[0] && element.files[0].size > MAX_FILE_SIZE_BYTES; + element.files && limit && element.files[0] && element.files[0].size > limit; if (isTooBig) { submits.forEach((submitter) => (submitter.disabled = true)); diff --git a/bookwyrm/suggested_users.py b/bookwyrm/suggested_users.py index a13ee97fd0..3e1cf17bde 100644 --- a/bookwyrm/suggested_users.py +++ b/bookwyrm/suggested_users.py @@ -34,7 +34,6 @@ def store_id(self, user): # pylint: disable=no-self-use def get_counts_from_rank(self, rank): # pylint: disable=no-self-use """calculate mutuals count and shared books count from rank""" - # pylint: disable=c-extension-no-member return { "mutuals": math.floor(rank), # "shared_books": int(1 / (-1 * (rank % 1 - 1))) - 1, @@ -128,7 +127,6 @@ def get_annotated_users(viewer, *args, **kwargs): ), distinct=True, ), - # pylint: disable=line-too-long # shared_books=Count( # "shelfbook", # filter=Q( @@ -202,7 +200,7 @@ def update_suggestions_on_unfollow(sender, instance, **kwargs): @receiver(signals.post_save, sender=models.User) -# pylint: disable=unused-argument, too-many-arguments +# pylint: disable=unused-argument def update_user(sender, instance, created, update_fields=None, **kwargs): """an updated user, neat""" # a new user is found, create suggestions for them diff --git a/bookwyrm/telemetry/open_telemetry.py b/bookwyrm/telemetry/open_telemetry.py index 2a0168ff3b..f18b8b38c1 100644 --- a/bookwyrm/telemetry/open_telemetry.py +++ b/bookwyrm/telemetry/open_telemetry.py @@ -23,9 +23,9 @@ def instrumentDjango() -> None: def instrumentPostgres() -> None: - from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor + from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor - Psycopg2Instrumentor().instrument() + PsycopgInstrumentor().instrument() def instrumentCelery() -> None: diff --git a/bookwyrm/templates/author/author.html b/bookwyrm/templates/author/author.html index e24a77dcd4..043f3fa54d 100644 --- a/bookwyrm/templates/author/author.html +++ b/bookwyrm/templates/author/author.html @@ -28,7 +28,7 @@
+ + {% trans "View on Finna" %} + + + {% if request.user.is_authenticated and perms.bookwyrm.edit_book %} + + {% include "book/sync_modal.html" with source="api.finna.fi" source_name="Finna" id="finna_sync" %} + {% endif %} +
+ {% endif %}{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}
-{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}
+{% blocktrans with hours=next_available.1 %}Currently you are allowed to import one user every {{ hours }} hours.{% endblocktrans %}
+{% blocktrans with next_time=next_available.0 %}You will next be able to import a user file at {{ next_time }}{% endblocktrans %}
+ {% if recent_avg_hours %} + {% blocktrans trimmed with hours=recent_avg_hours|floatformat:0|intcomma %} + On average, recent imports have taken {{ hours }} hours. + {% endblocktrans %} + {% else %} + {% blocktrans trimmed with minutes=recent_avg_minutes|floatformat:0|intcomma %} + On average, recent imports have taken {{ minutes }} minutes. + {% endblocktrans %} + {% endif %} +
+