diff --git a/api/config/api_urls.py b/api/config/api_urls.py index cdcc74391..dc5ef22a3 100644 --- a/api/config/api_urls.py +++ b/api/config/api_urls.py @@ -28,6 +28,7 @@ router.register( r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks" ) router.register(r"mutations", common_views.MutationViewSet, "mutations") +router.register(r"attachments", common_views.AttachmentViewSet, "attachments") v1_patterns = router.urls subsonic_router = routers.SimpleRouter(trailing_slash=False) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index e5ac5a344..1ec11e7ff 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -392,6 +392,11 @@ MEDIA_ROOT = env("MEDIA_ROOT", default=str(APPS_DIR("media"))) # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url MEDIA_URL = env("MEDIA_URL", default="/media/") FILE_UPLOAD_PERMISSIONS = 0o644 + +ATTACHMENTS_UNATTACHED_PRUNE_DELAY = env.int( + "ATTACHMENTS_UNATTACHED_PRUNE_DELAY", default=3600 * 24 +) + # URL Configuration # ------------------------------------------------------------------------------ ROOT_URLCONF = "config.urls" @@ -558,6 +563,11 @@ CELERY_BROKER_URL = env( CELERY_TASK_DEFAULT_RATE_LIMIT = 1 CELERY_TASK_TIME_LIMIT = 300 CELERY_BEAT_SCHEDULE = { + "common.prune_unattached_attachments": { + "task": "common.prune_unattached_attachments", + "schedule": crontab(minute="0", hour="*"), + "options": {"expires": 60 * 60}, + }, "federation.clean_music_cache": { "task": "federation.clean_music_cache", "schedule": crontab(minute="0", hour="*/2"), @@ -856,6 +866,7 @@ ACCOUNT_USERNAME_BLACKLIST = [ ] + env.list("ACCOUNT_USERNAME_BLACKLIST", default=[]) EXTERNAL_REQUESTS_VERIFY_SSL = env.bool("EXTERNAL_REQUESTS_VERIFY_SSL", default=True) +EXTERNAL_REQUESTS_TIMEOUT = env.int("EXTERNAL_REQUESTS_TIMEOUT", default=5) # XXX: deprecated, see #186 API_AUTHENTICATION_REQUIRED = env.bool("API_AUTHENTICATION_REQUIRED", True) @@ -878,7 +889,11 @@ VERSATILEIMAGEFIELD_RENDITION_KEY_SETS = { ("square_crop", "crop__400x400"), ("medium_square_crop", "crop__200x200"), ("small_square_crop", "crop__50x50"), - ] + ], + "attachment_square": [ + ("original", "url"), + ("medium_square_crop", "crop__200x200"), + ], } VERSATILEIMAGEFIELD_SETTINGS = {"create_images_on_demand": False} RSA_KEY_SIZE = 2048 diff --git a/api/funkwhale_api/common/admin.py b/api/funkwhale_api/common/admin.py index 3ec6f1f44..39a112531 100644 --- a/api/funkwhale_api/common/admin.py +++ b/api/funkwhale_api/common/admin.py @@ -45,3 +45,20 @@ class MutationAdmin(ModelAdmin): search_fields = ["created_by__preferred_username"] list_filter = ["type", "is_approved", "is_applied"] actions = [apply] + + +@register(models.Attachment) +class AttachmentAdmin(ModelAdmin): + list_display = [ + "uuid", + "actor", + "url", + "file", + "size", + "mimetype", + "creation_date", + "last_fetch_date", + ] + list_select_related = True + search_fields = ["actor__domain__name"] + list_filter = ["mimetype"] diff --git a/api/funkwhale_api/common/factories.py b/api/funkwhale_api/common/factories.py index 6919f9c37..85a441e85 100644 --- a/api/funkwhale_api/common/factories.py +++ b/api/funkwhale_api/common/factories.py @@ -23,3 +23,14 @@ class MutationFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): return self.target = extracted self.save() + + +@registry.register +class AttachmentFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): + url = factory.Faker("federation_url") + uuid = factory.Faker("uuid4") + actor = factory.SubFactory(federation_factories.ActorFactory) + file = factory.django.ImageField() + + class Meta: + model = "common.Attachment" diff --git a/api/funkwhale_api/common/middleware.py b/api/funkwhale_api/common/middleware.py index d8573baaa..3b9fc17e5 100644 --- a/api/funkwhale_api/common/middleware.py +++ b/api/funkwhale_api/common/middleware.py @@ -1,6 +1,5 @@ import html import io -import requests import time import xml.sax.saxutils @@ -11,6 +10,7 @@ from django import urls from rest_framework import views from . import preferences +from . import session from . import throttling from . import utils @@ -76,10 +76,7 @@ def get_spa_html(spa_url): if cached: return cached - response = requests.get( - utils.join_url(spa_url, "index.html"), - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, - ) + response = session.get_session().get(utils.join_url(spa_url, "index.html"),) response.raise_for_status() content = response.text caches["local"].set(cache_key, content, settings.FUNKWHALE_SPA_HTML_CACHE_DURATION) diff --git a/api/funkwhale_api/common/migrations/0004_auto_20191111_1338.py b/api/funkwhale_api/common/migrations/0004_auto_20191111_1338.py new file mode 100644 index 000000000..f35c9e11f --- /dev/null +++ b/api/funkwhale_api/common/migrations/0004_auto_20191111_1338.py @@ -0,0 +1,35 @@ +# Generated by Django 2.2.6 on 2019-11-11 13:38 + +import django.contrib.postgres.fields.jsonb +import django.core.serializers.json +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import funkwhale_api.common.models +import funkwhale_api.common.validators +import uuid +import versatileimagefield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0003_cit_extension'), + ] + + operations = [ + migrations.CreateModel( + name='Attachment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.URLField(max_length=500, unique=True, null=True)), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, unique=True)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('last_fetch_date', models.DateTimeField(blank=True, null=True)), + ('size', models.IntegerField(blank=True, null=True)), + ('mimetype', models.CharField(blank=True, max_length=200, null=True)), + ('file', versatileimagefield.fields.VersatileImageField(max_length=255, upload_to=funkwhale_api.common.models.get_file_path, validators=[funkwhale_api.common.validators.ImageDimensionsValidator(min_height=50, min_width=50), funkwhale_api.common.validators.FileValidator(allowed_extensions=['png', 'jpg', 'jpeg'], max_size=5242880)])), + ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='attachments', to='federation.Actor', null=True)), + ], + ), + ] diff --git a/api/funkwhale_api/common/models.py b/api/funkwhale_api/common/models.py index 9fd1a3c76..f764bf251 100644 --- a/api/funkwhale_api/common/models.py +++ b/api/funkwhale_api/common/models.py @@ -1,4 +1,6 @@ import uuid +import magic +import mimetypes from django.contrib.postgres.fields import JSONField from django.contrib.contenttypes.fields import GenericForeignKey @@ -9,11 +11,18 @@ from django.db import connections, models, transaction from django.db.models import Lookup from django.db.models.fields import Field from django.db.models.sql.compiler import SQLCompiler +from django.dispatch import receiver from django.utils import timezone from django.urls import reverse +from versatileimagefield.fields import VersatileImageField +from versatileimagefield.image_warmer import VersatileImageFieldWarmer + from funkwhale_api.federation import utils as federation_utils +from . import utils +from . import validators + @Field.register_lookup class NotEqual(Lookup): @@ -150,3 +159,102 @@ class Mutation(models.Model): self.applied_date = timezone.now() self.save(update_fields=["is_applied", "applied_date", "previous_state"]) return previous_state + + +def get_file_path(instance, filename): + return utils.ChunkedPath("attachments")(instance, filename) + + +class AttachmentQuerySet(models.QuerySet): + def attached(self, include=True): + related_fields = ["covered_album"] + query = None + for field in related_fields: + field_query = ~models.Q(**{field: None}) + query = query | field_query if query else field_query + + if include is False: + query = ~query + + return self.filter(query) + + +class Attachment(models.Model): + # Remote URL where the attachment can be fetched + url = models.URLField(max_length=500, unique=True, null=True) + uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4) + # Actor associated with the attachment + actor = models.ForeignKey( + "federation.Actor", + related_name="attachments", + on_delete=models.CASCADE, + null=True, + ) + creation_date = models.DateTimeField(default=timezone.now) + last_fetch_date = models.DateTimeField(null=True, blank=True) + # File size + size = models.IntegerField(null=True, blank=True) + mimetype = models.CharField(null=True, blank=True, max_length=200) + + file = VersatileImageField( + upload_to=get_file_path, + max_length=255, + validators=[ + validators.ImageDimensionsValidator(min_width=50, min_height=50), + validators.FileValidator( + allowed_extensions=["png", "jpg", "jpeg"], max_size=1024 * 1024 * 5, + ), + ], + ) + + objects = AttachmentQuerySet.as_manager() + + def save(self, **kwargs): + if self.file and not self.size: + self.size = self.file.size + + if self.file and not self.mimetype: + self.mimetype = self.guess_mimetype() + + return super().save() + + @property + def is_local(self): + return federation_utils.is_local(self.fid) + + def guess_mimetype(self): + f = self.file + b = min(1000000, f.size) + t = magic.from_buffer(f.read(b), mime=True) + if not t.startswith("image/"): + # failure, we try guessing by extension + mt, _ = mimetypes.guess_type(f.name) + if mt: + t = mt + return t + + @property + def download_url_original(self): + if self.file: + return federation_utils.full_url(self.file.url) + proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid}) + return federation_utils.full_url(proxy_url + "?next=original") + + @property + def download_url_medium_square_crop(self): + if self.file: + return federation_utils.full_url(self.file.crop["200x200"].url) + proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid}) + return federation_utils.full_url(proxy_url + "?next=medium_square_crop") + + +@receiver(models.signals.post_save, sender=Attachment) +def warm_attachment_thumbnails(sender, instance, **kwargs): + if not instance.file or not settings.CREATE_IMAGE_THUMBNAILS: + return + warmer = VersatileImageFieldWarmer( + instance_or_queryset=instance, + rendition_key_set="attachment_square", + image_attr="file", + ) + num_created, failed_to_create = warmer.warm() diff --git a/api/funkwhale_api/common/scripts/create_image_variations.py b/api/funkwhale_api/common/scripts/create_image_variations.py index 5e941ce1f..31bf0269c 100644 --- a/api/funkwhale_api/common/scripts/create_image_variations.py +++ b/api/funkwhale_api/common/scripts/create_image_variations.py @@ -4,11 +4,16 @@ Compute different sizes of image used for Album covers and User avatars from versatileimagefield.image_warmer import VersatileImageFieldWarmer +from funkwhale_api.common.models import Attachment from funkwhale_api.music.models import Album from funkwhale_api.users.models import User -MODELS = [(Album, "cover", "square"), (User, "avatar", "square")] +MODELS = [ + (Album, "cover", "square"), + (User, "avatar", "square"), + (Attachment, "file", "attachment_square"), +] def main(command, **kwargs): diff --git a/api/funkwhale_api/common/serializers.py b/api/funkwhale_api/common/serializers.py index 59b513f37..3f128c18d 100644 --- a/api/funkwhale_api/common/serializers.py +++ b/api/funkwhale_api/common/serializers.py @@ -272,3 +272,38 @@ class APIMutationSerializer(serializers.ModelSerializer): if value not in self.context["registry"]: raise serializers.ValidationError("Invalid mutation type {}".format(value)) return value + + +class AttachmentSerializer(serializers.Serializer): + uuid = serializers.UUIDField(read_only=True) + size = serializers.IntegerField(read_only=True) + mimetype = serializers.CharField(read_only=True) + creation_date = serializers.DateTimeField(read_only=True) + file = StripExifImageField(write_only=True) + urls = serializers.SerializerMethodField() + + def get_urls(self, o): + urls = {} + urls["source"] = o.url + urls["original"] = o.download_url_original + urls["medium_square_crop"] = o.download_url_medium_square_crop + return urls + + def to_representation(self, o): + repr = super().to_representation(o) + # XXX: BACKWARD COMPATIBILITY + # having the attachment urls in a nested JSON obj is better, + # but we can't do this without breaking clients + # So we extract the urls and include these in the parent payload + repr.update({k: v for k, v in repr["urls"].items() if k != "source"}) + # also, our legacy images had lots of variations (400x400, 200x200, 50x50) + # but we removed some of these, so we emulate these by hand (by redirecting) + # to actual, existing attachment variations + repr["square_crop"] = repr["medium_square_crop"] + repr["small_square_crop"] = repr["medium_square_crop"] + return repr + + def create(self, validated_data): + return models.Attachment.objects.create( + file=validated_data["file"], actor=validated_data["actor"] + ) diff --git a/api/funkwhale_api/common/session.py b/api/funkwhale_api/common/session.py index 4d5d0bb60..23fd28306 100644 --- a/api/funkwhale_api/common/session.py +++ b/api/funkwhale_api/common/session.py @@ -4,6 +4,13 @@ from django.conf import settings import funkwhale_api +class FunkwhaleSession(requests.Session): + def request(self, *args, **kwargs): + kwargs.setdefault("verify", settings.EXTERNAL_REQUESTS_VERIFY_SSL) + kwargs.setdefault("timeout", settings.EXTERNAL_REQUESTS_TIMEOUT) + return super().request(*args, **kwargs) + + def get_user_agent(): return "python-requests (funkwhale/{}; +{})".format( funkwhale_api.__version__, settings.FUNKWHALE_URL @@ -11,6 +18,6 @@ def get_user_agent(): def get_session(): - s = requests.Session() + s = FunkwhaleSession() s.headers["User-Agent"] = get_user_agent() return s diff --git a/api/funkwhale_api/common/tasks.py b/api/funkwhale_api/common/tasks.py index 994b0bdff..c7deee7f5 100644 --- a/api/funkwhale_api/common/tasks.py +++ b/api/funkwhale_api/common/tasks.py @@ -1,14 +1,23 @@ +import datetime +import logging +import tempfile + +from django.conf import settings +from django.core.files import File from django.db import transaction from django.dispatch import receiver - +from django.utils import timezone from funkwhale_api.common import channels from funkwhale_api.taskapp import celery from . import models from . import serializers +from . import session from . import signals +logger = logging.getLogger(__name__) + @celery.app.task(name="common.apply_mutation") @transaction.atomic @@ -57,3 +66,35 @@ def broadcast_mutation_update(mutation, old_is_approved, new_is_approved, **kwar }, }, ) + + +def fetch_remote_attachment(attachment, filename=None, save=True): + if attachment.file: + # already there, no need to fetch + return + + s = session.get_session() + attachment.last_fetch_date = timezone.now() + with tempfile.TemporaryFile() as tf: + with s.get(attachment.url, timeout=5, stream=True) as r: + for chunk in r.iter_content(): + tf.write(chunk) + tf.seek(0) + attachment.file.save( + filename or attachment.url.split("/")[-1], File(tf), save=save + ) + + +@celery.app.task(name="common.prune_unattached_attachments") +def prune_unattached_attachments(): + limit = timezone.now() - datetime.timedelta( + seconds=settings.ATTACHMENTS_UNATTACHED_PRUNE_DELAY + ) + candidates = models.Attachment.objects.attached(False).filter( + creation_date__lte=limit + ) + + total = candidates.count() + logger.info("Deleting %s unattached attachments…", total) + result = candidates.delete() + logger.info("Deletion done: %s", result) diff --git a/api/funkwhale_api/common/views.py b/api/funkwhale_api/common/views.py index d197dbe90..1109589d0 100644 --- a/api/funkwhale_api/common/views.py +++ b/api/funkwhale_api/common/views.py @@ -11,6 +11,8 @@ from rest_framework import response from rest_framework import views from rest_framework import viewsets +from funkwhale_api.users.oauth import permissions as oauth_permissions + from . import filters from . import models from . import mutations @@ -140,3 +142,40 @@ class RateLimitView(views.APIView): "scopes": throttling.get_status(ident, time.time()), } return response.Response(data, status=200) + + +class AttachmentViewSet( + mixins.RetrieveModelMixin, + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + viewsets.GenericViewSet, +): + lookup_field = "uuid" + queryset = models.Attachment.objects.all() + serializer_class = serializers.AttachmentSerializer + permission_classes = [oauth_permissions.ScopePermission] + required_scope = "libraries" + anonymous_policy = "setting" + + @action(detail=True, methods=["get"]) + @transaction.atomic + def proxy(self, request, *args, **kwargs): + instance = self.get_object() + + size = request.GET.get("next", "original").lower() + if size not in ["original", "medium_square_crop"]: + size = "original" + + tasks.fetch_remote_attachment(instance) + data = self.serializer_class(instance).data + redirect = response.Response(status=302) + redirect["Location"] = data["urls"][size] + return redirect + + def perform_create(self, serializer): + return serializer.save(actor=self.request.user.actor) + + def perform_destroy(self, instance): + if instance.actor is None or instance.actor != self.request.user.actor: + raise exceptions.PermissionDenied() + instance.delete() diff --git a/api/funkwhale_api/favorites/views.py b/api/funkwhale_api/favorites/views.py index 7d1991bc6..3e1f33759 100644 --- a/api/funkwhale_api/favorites/views.py +++ b/api/funkwhale_api/favorites/views.py @@ -54,7 +54,9 @@ class TrackFavoriteViewSet( ) tracks = Track.objects.with_playable_uploads( music_utils.get_actor_from_request(self.request) - ).select_related("artist", "album__artist", "attributed_to") + ).select_related( + "artist", "album__artist", "attributed_to", "album__attachment_cover" + ) queryset = queryset.prefetch_related(Prefetch("track", queryset=tracks)) return queryset diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 95997f952..39161a9cb 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -14,10 +14,7 @@ logger = logging.getLogger(__name__) def get_actor_data(actor_url): response = session.get_session().get( - actor_url, - timeout=5, - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, - headers={"Accept": "application/activity+json"}, + actor_url, headers={"Accept": "application/activity+json"}, ) response.raise_for_status() try: diff --git a/api/funkwhale_api/federation/library.py b/api/funkwhale_api/federation/library.py index e7f8373fa..1fef43c22 100644 --- a/api/funkwhale_api/federation/library.py +++ b/api/funkwhale_api/federation/library.py @@ -1,5 +1,4 @@ import requests -from django.conf import settings from funkwhale_api.common import session @@ -12,8 +11,6 @@ def get_library_data(library_url, actor): response = session.get_session().get( library_url, auth=auth, - timeout=5, - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={"Content-Type": "application/activity+json"}, ) except requests.ConnectionError: @@ -35,11 +32,7 @@ def get_library_data(library_url, actor): def get_library_page(library, page_url, actor): auth = signing.get_auth(actor.private_key, actor.private_key_id) response = session.get_session().get( - page_url, - auth=auth, - timeout=5, - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, - headers={"Content-Type": "application/activity+json"}, + page_url, auth=auth, headers={"Content-Type": "application/activity+json"}, ) serializer = serializers.CollectionPageSerializer( data=response.json(), diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index fd52e17c8..12057253f 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -541,7 +541,6 @@ class LibraryTrack(models.Model): auth=auth, stream=True, timeout=20, - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={"Content-Type": "application/activity+json"}, ) with remote_response as r: diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 697c2acc9..5a62f4ce3 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -824,8 +824,8 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer): def get_tags_repr(self, instance): return [ - {"type": "Hashtag", "name": "#{}".format(tag)} - for tag in sorted(instance.tagged_items.values_list("tag__name", flat=True)) + {"type": "Hashtag", "name": "#{}".format(item.tag.name)} + for item in sorted(instance.tagged_items.all(), key=lambda i: i.tag.name) ] @@ -902,12 +902,11 @@ class AlbumSerializer(MusicEntitySerializer): else None, "tag": self.get_tags_repr(instance), } - if instance.cover: + if instance.attachment_cover: d["cover"] = { "type": "Link", - "href": utils.full_url(instance.cover.url), - "mediaType": mimetypes.guess_type(instance.cover_path)[0] - or "image/jpeg", + "href": instance.attachment_cover.download_url_original, + "mediaType": instance.attachment_cover.mimetype or "image/jpeg", } if self.context.get("include_ap_context", self.parent is None): d["@context"] = jsonld.get_default_context() diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index 163ac7788..6a03438c4 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -88,7 +88,7 @@ def dispatch_inbox(activity, call_handlers=True): context={ "activity": activity, "actor": activity.actor, - "inbox_items": activity.inbox_items.filter(is_read=False), + "inbox_items": activity.inbox_items.filter(is_read=False).order_by("id"), }, call_handlers=call_handlers, ) @@ -142,8 +142,6 @@ def deliver_to_remote(delivery): auth=auth, json=delivery.activity.payload, url=delivery.inbox_url, - timeout=5, - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={"Content-Type": "application/activity+json"}, ) logger.debug("Remote answered with %s", response.status_code) @@ -163,9 +161,7 @@ def deliver_to_remote(delivery): def fetch_nodeinfo(domain_name): s = session.get_session() wellknown_url = "https://{}/.well-known/nodeinfo".format(domain_name) - response = s.get( - url=wellknown_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL - ) + response = s.get(url=wellknown_url) response.raise_for_status() serializer = serializers.NodeInfoSerializer(data=response.json()) serializer.is_valid(raise_exception=True) @@ -175,9 +171,7 @@ def fetch_nodeinfo(domain_name): nodeinfo_url = link["href"] break - response = s.get( - url=nodeinfo_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL - ) + response = s.get(url=nodeinfo_url) response.raise_for_status() return response.json() @@ -308,8 +302,6 @@ def fetch(fetch): response = session.get_session().get( auth=auth, url=fetch.url, - timeout=5, - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={"Content-Type": "application/activity+json"}, ) logger.debug("Remote answered with %s", response.status_code) diff --git a/api/funkwhale_api/federation/utils.py b/api/funkwhale_api/federation/utils.py index c2eacfe9d..74e9fef69 100644 --- a/api/funkwhale_api/federation/utils.py +++ b/api/funkwhale_api/federation/utils.py @@ -84,8 +84,6 @@ def retrieve_ap_object( response = session.get_session().get( fid, auth=auth, - timeout=5, - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={ "Accept": "application/activity+json", "Content-Type": "application/activity+json", diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 85594e02b..7aebc2971 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -1,5 +1,6 @@ from django import forms from django.core import paginator +from django.db.models import Prefetch from django.http import HttpResponse from django.urls import reverse from rest_framework import exceptions, mixins, permissions, response, viewsets @@ -163,7 +164,7 @@ class MusicLibraryViewSet( FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet ): authentication_classes = [authentication.SignatureAuthentication] - renderer_classes = renderers.get_ap_renderers() + # renderer_classes = renderers.get_ap_renderers() serializer_class = serializers.LibrarySerializer queryset = music_models.Library.objects.all().select_related("actor") lookup_field = "uuid" @@ -176,7 +177,25 @@ class MusicLibraryViewSet( "actor": lb.actor, "name": lb.name, "summary": lb.description, - "items": lb.uploads.for_federation().order_by("-creation_date"), + "items": lb.uploads.for_federation() + .order_by("-creation_date") + .prefetch_related( + Prefetch( + "track", + queryset=music_models.Track.objects.select_related( + "album__artist__attributed_to", + "artist__attributed_to", + "album__attributed_to", + "attributed_to", + "album__attachment_cover", + ).prefetch_related( + "tagged_items__tag", + "album__tagged_items__tag", + "album__artist__tagged_items__tag", + "artist__tagged_items__tag", + ), + ) + ), "item_serializer": serializers.UploadSerializer, } page = request.GET.get("page") @@ -219,7 +238,10 @@ class MusicUploadViewSet( authentication_classes = [authentication.SignatureAuthentication] renderer_classes = renderers.get_ap_renderers() queryset = music_models.Upload.objects.local().select_related( - "library__actor", "track__artist", "track__album__artist" + "library__actor", + "track__artist", + "track__album__artist", + "track__album__attachment_cover", ) serializer_class = serializers.UploadSerializer lookup_field = "uuid" diff --git a/api/funkwhale_api/federation/webfinger.py b/api/funkwhale_api/federation/webfinger.py index 874b3c158..765c5e535 100644 --- a/api/funkwhale_api/federation/webfinger.py +++ b/api/funkwhale_api/federation/webfinger.py @@ -41,9 +41,7 @@ def get_resource(resource_string): url = "https://{}/.well-known/webfinger?resource={}".format( hostname, resource_string ) - response = session.get_session().get( - url, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, timeout=5 - ) + response = session.get_session().get(url) response.raise_for_status() serializer = serializers.ActorWebfingerSerializer(data=response.json()) serializer.is_valid(raise_exception=True) diff --git a/api/funkwhale_api/manage/views.py b/api/funkwhale_api/manage/views.py index 200dccf1f..c946c37e2 100644 --- a/api/funkwhale_api/manage/views.py +++ b/api/funkwhale_api/manage/views.py @@ -69,9 +69,9 @@ class ManageArtistViewSet( "tracks", Prefetch( "albums", - queryset=music_models.Album.objects.annotate( - tracks_count=Count("tracks") - ), + queryset=music_models.Album.objects.select_related( + "attachment_cover" + ).annotate(tracks_count=Count("tracks")), ), music_views.TAG_PREFETCH, ) @@ -110,7 +110,7 @@ class ManageAlbumViewSet( queryset = ( music_models.Album.objects.all() .order_by("-id") - .select_related("attributed_to", "artist") + .select_related("attributed_to", "artist", "attachment_cover") .prefetch_related("tracks", music_views.TAG_PREFETCH) ) serializer_class = serializers.ManageAlbumSerializer @@ -153,7 +153,9 @@ class ManageTrackViewSet( queryset = ( music_models.Track.objects.all() .order_by("-id") - .select_related("attributed_to", "artist", "album__artist") + .select_related( + "attributed_to", "artist", "album__artist", "album__attachment_cover" + ) .annotate(uploads_count=Coalesce(Subquery(uploads_subquery), 0)) .prefetch_related(music_views.TAG_PREFETCH) ) diff --git a/api/funkwhale_api/moderation/management/commands/mrf_check.py b/api/funkwhale_api/moderation/management/commands/mrf_check.py index b518daa08..6462bd9a0 100644 --- a/api/funkwhale_api/moderation/management/commands/mrf_check.py +++ b/api/funkwhale_api/moderation/management/commands/mrf_check.py @@ -6,8 +6,6 @@ import logging from django.core.management.base import BaseCommand, CommandError from django.core import validators -from django.conf import settings - from funkwhale_api.common import session from funkwhale_api.federation import models from funkwhale_api.moderation import mrf @@ -84,10 +82,7 @@ class Command(BaseCommand): content = models.Activity.objects.get(uuid=input).payload elif is_url(input): response = session.get_session().get( - input, - timeout=5, - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, - headers={"Content-Type": "application/activity+json"}, + input, headers={"Content-Type": "application/activity+json"}, ) response.raise_for_status() content = response.json() diff --git a/api/funkwhale_api/music/factories.py b/api/funkwhale_api/music/factories.py index 52e5020bb..07f0d9aa3 100644 --- a/api/funkwhale_api/music/factories.py +++ b/api/funkwhale_api/music/factories.py @@ -4,6 +4,7 @@ import factory from funkwhale_api.factories import registry, NoUpdateOnCreate +from funkwhale_api.common import factories as common_factories from funkwhale_api.federation import factories as federation_factories from funkwhale_api.music import licenses from funkwhale_api.tags import factories as tags_factories @@ -81,7 +82,7 @@ class AlbumFactory( title = factory.Faker("sentence", nb_words=3) mbid = factory.Faker("uuid4") release_date = factory.Faker("date_object") - cover = factory.django.ImageField() + attachment_cover = factory.SubFactory(common_factories.AttachmentFactory) artist = factory.SubFactory(ArtistFactory) release_group_id = factory.Faker("uuid4") fid = factory.Faker("federation_url") diff --git a/api/funkwhale_api/music/migrations/0042_album_attachment_cover.py b/api/funkwhale_api/music/migrations/0042_album_attachment_cover.py new file mode 100644 index 000000000..b27dc03eb --- /dev/null +++ b/api/funkwhale_api/music/migrations/0042_album_attachment_cover.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.6 on 2019-11-12 09:56 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0004_auto_20191111_1338'), + ('music', '0041_auto_20191021_1705'), + ] + + operations = [ + migrations.AddField( + model_name='album', + name='attachment_cover', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='common.Attachment', blank=True), + ), + ] diff --git a/api/funkwhale_api/music/migrations/0043_album_cover_attachment.py b/api/funkwhale_api/music/migrations/0043_album_cover_attachment.py new file mode 100644 index 000000000..f5da4072a --- /dev/null +++ b/api/funkwhale_api/music/migrations/0043_album_cover_attachment.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + + +def create_attachments(apps, schema_editor): + Album = apps.get_model("music", "Album") + Attachment = apps.get_model("common", "Attachment") + + album_attachment_mapping = {} + def get_mimetype(path): + if path.lower().endswith('.png'): + return "image/png" + return "image/jpeg" + + for album in Album.objects.filter(attachment_cover=None).exclude(cover="").exclude(cover=None): + album_attachment_mapping[album] = Attachment( + file=album.cover, + size=album.cover.size, + mimetype=get_mimetype(album.cover.path), + ) + + Attachment.objects.bulk_create(album_attachment_mapping.values(), batch_size=2000) + # map each attachment to the corresponding album + # and bulk save + for album, attachment in album_attachment_mapping.items(): + album.attachment_cover = attachment + + Album.objects.bulk_update(album_attachment_mapping.keys(), fields=['attachment_cover'], batch_size=2000) + + +def rewind(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [("music", "0042_album_attachment_cover")] + + operations = [migrations.RunPython(create_attachments, rewind)] diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index f40a4b186..a1f4a7c64 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -20,7 +20,6 @@ from django.urls import reverse from django.utils import timezone from versatileimagefield.fields import VersatileImageField -from versatileimagefield.image_warmer import VersatileImageFieldWarmer from funkwhale_api import musicbrainz from funkwhale_api.common import fields @@ -286,9 +285,17 @@ class Album(APIModelMixin): artist = models.ForeignKey(Artist, related_name="albums", on_delete=models.CASCADE) release_date = models.DateField(null=True, blank=True, db_index=True) release_group_id = models.UUIDField(null=True, blank=True) + # XXX: 1.0 clean this uneeded field in favor of attachment_cover cover = VersatileImageField( upload_to="albums/covers/%Y/%m/%d", null=True, blank=True ) + attachment_cover = models.ForeignKey( + "common.Attachment", + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="covered_album", + ) TYPE_CHOICES = (("album", "Album"),) type = models.CharField(choices=TYPE_CHOICES, max_length=30, default="album") @@ -334,40 +341,46 @@ class Album(APIModelMixin): objects = AlbumQuerySet.as_manager() def get_image(self, data=None): + from funkwhale_api.common import tasks as common_tasks + + attachment = None if data: extensions = {"image/jpeg": "jpg", "image/png": "png", "image/gif": "gif"} extension = extensions.get(data["mimetype"], "jpg") + attachment = common_models.Attachment(mimetype=data["mimetype"]) f = None + filename = "{}.{}".format(self.uuid, extension) if data.get("content"): # we have to cover itself f = ContentFile(data["content"]) + attachment.file.save(filename, f, save=False) elif data.get("url"): + attachment.url = data.get("url") # we can fetch from a url try: - response = session.get_session().get( - data.get("url"), - timeout=3, - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, + common_tasks.fetch_remote_attachment( + attachment, filename=filename, save=False ) - response.raise_for_status() except Exception as e: logger.warn( "Cannot download cover at url %s: %s", data.get("url"), e ) return - else: - f = ContentFile(response.content) - if f: - self.cover.save("{}.{}".format(self.uuid, extension), f, save=False) - self.save(update_fields=["cover"]) - return self.cover.file - if self.mbid: + + elif self.mbid: image_data = musicbrainz.api.images.get_front(str(self.mbid)) f = ContentFile(image_data) - self.cover.save("{0}.jpg".format(self.mbid), f, save=False) - self.save(update_fields=["cover"]) - if self.cover: - return self.cover.file + attachment = common_models.Attachment(mimetype="image/jpeg") + attachment.file.save("{0}.jpg".format(self.mbid), f, save=False) + if attachment and attachment.file: + attachment.save() + self.attachment_cover = attachment + self.save(update_fields=["attachment_cover"]) + return self.attachment_cover.file + + @property + def cover(self): + return self.attachment_cover def __str__(self): return self.title @@ -378,16 +391,6 @@ class Album(APIModelMixin): def get_moderation_url(self): return "/manage/library/albums/{}".format(self.pk) - @property - def cover_path(self): - if not self.cover: - return None - try: - return self.cover.path - except NotImplementedError: - # external storage - return self.cover.name - @classmethod def get_or_create_from_title(cls, title, **kwargs): kwargs.update({"title": title}) @@ -415,7 +418,9 @@ def import_album(v): class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet): def for_nested_serialization(self): - return self.prefetch_related("artist", "album__artist") + return self.prefetch_related( + "artist", "album__artist", "album__attachment_cover" + ) def annotate_playable_by_actor(self, actor): @@ -729,7 +734,6 @@ class Upload(models.Model): return parsed.hostname def download_audio_from_remote(self, actor): - from funkwhale_api.common import session from funkwhale_api.federation import signing if actor: @@ -743,7 +747,6 @@ class Upload(models.Model): stream=True, timeout=20, headers={"Content-Type": "application/octet-stream"}, - verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, ) with remote_response as r: remote_response.raise_for_status() @@ -1307,13 +1310,3 @@ def update_request_status(sender, instance, created, **kwargs): # let's mark the request as imported since the import is over instance.import_request.status = "imported" return instance.import_request.save(update_fields=["status"]) - - -@receiver(models.signals.post_save, sender=Album) -def warm_album_covers(sender, instance, **kwargs): - if not instance.cover or not settings.CREATE_IMAGE_THUMBNAILS: - return - album_covers_warmer = VersatileImageFieldWarmer( - instance_or_queryset=instance, rendition_key_set="square", image_attr="cover" - ) - num_created, failed_to_create = album_covers_warmer.warm() diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index 9ea2dd2cb..66790cf6d 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -4,7 +4,6 @@ from django.db import transaction from django import urls from django.conf import settings from rest_framework import serializers -from versatileimagefield.serializers import VersatileImageFieldSerializer from funkwhale_api.activity import serializers as activity_serializers from funkwhale_api.common import serializers as common_serializers @@ -17,7 +16,25 @@ from funkwhale_api.tags.models import Tag from . import filters, models, tasks -cover_field = VersatileImageFieldSerializer(allow_null=True, sizes="square") +class NullToEmptDict(object): + def get_attribute(self, o): + attr = super().get_attribute(o) + if attr is None: + return {} + return attr + + def to_representation(self, v): + if not v: + return v + return super().to_representation(v) + + +class CoverField(NullToEmptDict, common_serializers.AttachmentSerializer): + # XXX: BACKWARD COMPATIBILITY + pass + + +cover_field = CoverField() def serialize_attributed_to(self, obj): @@ -450,12 +467,12 @@ class OembedSerializer(serializers.Serializer): embed_type = "track" embed_id = track.pk data["title"] = "{} by {}".format(track.title, track.artist.name) - if track.album.cover: - data["thumbnail_url"] = federation_utils.full_url( - track.album.cover.crop["400x400"].url - ) - data["thumbnail_width"] = 400 - data["thumbnail_height"] = 400 + if track.album.attachment_cover: + data[ + "thumbnail_url" + ] = track.album.attachment_cover.download_url_medium_square_crop + data["thumbnail_width"] = 200 + data["thumbnail_height"] = 200 data["description"] = track.full_name data["author_name"] = track.artist.name data["height"] = 150 @@ -476,12 +493,12 @@ class OembedSerializer(serializers.Serializer): ) embed_type = "album" embed_id = album.pk - if album.cover: - data["thumbnail_url"] = federation_utils.full_url( - album.cover.crop["400x400"].url - ) - data["thumbnail_width"] = 400 - data["thumbnail_height"] = 400 + if album.attachment_cover: + data[ + "thumbnail_url" + ] = album.attachment_cover.download_url_medium_square_crop + data["thumbnail_width"] = 200 + data["thumbnail_height"] = 200 data["title"] = "{} by {}".format(album.title, album.artist.name) data["description"] = "{} by {}".format(album.title, album.artist.name) data["author_name"] = album.artist.name @@ -501,19 +518,14 @@ class OembedSerializer(serializers.Serializer): ) embed_type = "artist" embed_id = artist.pk - album = ( - artist.albums.filter(cover__isnull=False) - .exclude(cover="") - .order_by("-id") - .first() - ) + album = artist.albums.exclude(attachment_cover=None).order_by("-id").first() - if album and album.cover: - data["thumbnail_url"] = federation_utils.full_url( - album.cover.crop["400x400"].url - ) - data["thumbnail_width"] = 400 - data["thumbnail_height"] = 400 + if album and album.attachment_cover: + data[ + "thumbnail_url" + ] = album.attachment_cover.download_url_medium_square_crop + data["thumbnail_width"] = 200 + data["thumbnail_height"] = 200 data["title"] = artist.name data["description"] = artist.name data["author_name"] = artist.name @@ -533,19 +545,22 @@ class OembedSerializer(serializers.Serializer): ) embed_type = "playlist" embed_id = obj.pk - playlist_tracks = obj.playlist_tracks.exclude(track__album__cover="") - playlist_tracks = playlist_tracks.exclude(track__album__cover=None) - playlist_tracks = playlist_tracks.select_related("track__album").order_by( - "index" + playlist_tracks = obj.playlist_tracks.exclude( + track__album__attachment_cover=None ) + playlist_tracks = playlist_tracks.select_related( + "track__album__attachment_cover" + ).order_by("index") first_playlist_track = playlist_tracks.first() if first_playlist_track: - data["thumbnail_url"] = federation_utils.full_url( - first_playlist_track.track.album.cover.crop["400x400"].url + data[ + "thumbnail_url" + ] = ( + first_playlist_track.track.album.attachment_cover.download_url_medium_square_crop ) - data["thumbnail_width"] = 400 - data["thumbnail_height"] = 400 + data["thumbnail_width"] = 200 + data["thumbnail_height"] = 200 data["title"] = obj.name data["description"] = obj.name data["author_name"] = obj.name diff --git a/api/funkwhale_api/music/spa_views.py b/api/funkwhale_api/music/spa_views.py index 5215dcdd8..cbf143162 100644 --- a/api/funkwhale_api/music/spa_views.py +++ b/api/funkwhale_api/music/spa_views.py @@ -57,14 +57,12 @@ def library_track(request, pk): ), }, ] - if obj.album.cover: + if obj.album.attachment_cover: metas.append( { "tag": "meta", "property": "og:image", - "content": utils.join_url( - settings.FUNKWHALE_URL, obj.album.cover.crop["400x400"].url - ), + "content": obj.album.attachment_cover.download_url_medium_square_crop, } ) @@ -126,14 +124,12 @@ def library_album(request, pk): } ) - if obj.cover: + if obj.attachment_cover: metas.append( { "tag": "meta", "property": "og:image", - "content": utils.join_url( - settings.FUNKWHALE_URL, obj.cover.crop["400x400"].url - ), + "content": obj.attachment_cover.download_url_medium_square_crop, } ) @@ -166,7 +162,7 @@ def library_artist(request, pk): ) # we use latest album's cover as artist image latest_album = ( - obj.albums.exclude(cover="").exclude(cover=None).order_by("release_date").last() + obj.albums.exclude(attachment_cover=None).order_by("release_date").last() ) metas = [ {"tag": "meta", "property": "og:url", "content": artist_url}, @@ -174,14 +170,12 @@ def library_artist(request, pk): {"tag": "meta", "property": "og:type", "content": "profile"}, ] - if latest_album and latest_album.cover: + if latest_album and latest_album.attachment_cover: metas.append( { "tag": "meta", "property": "og:image", - "content": utils.join_url( - settings.FUNKWHALE_URL, latest_album.cover.crop["400x400"].url - ), + "content": latest_album.attachment_cover.download_url_medium_square_crop, } ) @@ -217,8 +211,7 @@ def library_playlist(request, pk): utils.spa_reverse("library_playlist", kwargs={"pk": obj.pk}), ) # we use the first playlist track's album's cover as image - playlist_tracks = obj.playlist_tracks.exclude(track__album__cover="") - playlist_tracks = playlist_tracks.exclude(track__album__cover=None) + playlist_tracks = obj.playlist_tracks.exclude(track__album__attachment_cover=None) playlist_tracks = playlist_tracks.select_related("track__album").order_by("index") first_playlist_track = playlist_tracks.first() metas = [ @@ -232,10 +225,7 @@ def library_playlist(request, pk): { "tag": "meta", "property": "og:image", - "content": utils.join_url( - settings.FUNKWHALE_URL, - first_playlist_track.track.album.cover.crop["400x400"].url, - ), + "content": first_playlist_track.track.album.attachment_cover.download_url_medium_square_crop, } ) diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py index 20265b320..1bf0f823e 100644 --- a/api/funkwhale_api/music/tasks.py +++ b/api/funkwhale_api/music/tasks.py @@ -21,7 +21,6 @@ from . import licenses from . import models from . import metadata from . import signals -from . import serializers logger = logging.getLogger(__name__) @@ -29,7 +28,7 @@ logger = logging.getLogger(__name__) def update_album_cover( album, source=None, cover_data=None, musicbrainz=True, replace=False ): - if album.cover and not replace: + if album.attachment_cover and not replace: return if cover_data: return album.get_image(data=cover_data) @@ -257,7 +256,7 @@ def process_upload(upload, update_denormalization=True): ) # update album cover, if needed - if not track.album.cover: + if not track.album.attachment_cover: update_album_cover( track.album, source=final_metadata.get("upload_source"), @@ -404,7 +403,7 @@ def sort_candidates(candidates, important_fields): @transaction.atomic def get_track_from_import_metadata(data, update_cover=False, attributed_to=None): track = _get_track(data, attributed_to=attributed_to) - if update_cover and track and not track.album.cover: + if update_cover and track and not track.album.attachment_cover: update_album_cover( track.album, source=data.get("upload_source"), @@ -584,6 +583,8 @@ def broadcast_import_status_update_to_owner(old_status, new_status, upload, **kw if not user: return + from . import serializers + group = "user.{}.imports".format(user.pk) channels.group_send( group, diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index c451d6b76..2260d9268 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -128,7 +128,9 @@ class ArtistViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelV def get_queryset(self): queryset = super().get_queryset() - albums = models.Album.objects.with_tracks_count() + albums = models.Album.objects.with_tracks_count().select_related( + "attachment_cover" + ) albums = albums.annotate_playable_by_actor( utils.get_actor_from_request(self.request) ) @@ -149,7 +151,7 @@ class AlbumViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelVi queryset = ( models.Album.objects.all() .order_by("-creation_date") - .prefetch_related("artist", "attributed_to") + .prefetch_related("artist", "attributed_to", "attachment_cover") ) serializer_class = serializers.AlbumSerializer permission_classes = [oauth_permissions.ScopePermission] diff --git a/api/funkwhale_api/playlists/models.py b/api/funkwhale_api/playlists/models.py index 37d498c4f..affa86ea5 100644 --- a/api/funkwhale_api/playlists/models.py +++ b/api/funkwhale_api/playlists/models.py @@ -17,7 +17,8 @@ class PlaylistQuerySet(models.QuerySet): def with_covers(self): album_prefetch = models.Prefetch( - "album", queryset=music_models.Album.objects.only("cover", "artist_id") + "album", + queryset=music_models.Album.objects.select_related("attachment_cover"), ) track_prefetch = models.Prefetch( "track", @@ -29,8 +30,7 @@ class PlaylistQuerySet(models.QuerySet): plt_prefetch = models.Prefetch( "playlist_tracks", queryset=PlaylistTrack.objects.all() - .exclude(track__album__cover=None) - .exclude(track__album__cover="") + .exclude(track__album__attachment_cover=None) .order_by("index") .only("id", "playlist_id", "track_id") .prefetch_related(track_prefetch), @@ -179,7 +179,9 @@ class Playlist(models.Model): class PlaylistTrackQuerySet(models.QuerySet): def for_nested_serialization(self, actor=None): tracks = music_models.Track.objects.with_playable_uploads(actor) - tracks = tracks.select_related("artist", "album__artist") + tracks = tracks.select_related( + "artist", "album__artist", "album__attachment_cover", "attributed_to" + ) return self.prefetch_related( models.Prefetch("track", queryset=tracks, to_attr="_prefetched_track") ) diff --git a/api/funkwhale_api/playlists/serializers.py b/api/funkwhale_api/playlists/serializers.py index dc61950dd..727edd525 100644 --- a/api/funkwhale_api/playlists/serializers.py +++ b/api/funkwhale_api/playlists/serializers.py @@ -145,7 +145,7 @@ class PlaylistSerializer(serializers.ModelSerializer): for plt in plts: if plt.track.album.artist_id in excluded_artists: continue - url = plt.track.album.cover.crop["200x200"].url + url = plt.track.album.attachment_cover.download_url_medium_square_crop if url in covers: continue covers.append(url) diff --git a/api/funkwhale_api/subsonic/serializers.py b/api/funkwhale_api/subsonic/serializers.py index cae99e242..7b6e37686 100644 --- a/api/funkwhale_api/subsonic/serializers.py +++ b/api/funkwhale_api/subsonic/serializers.py @@ -89,7 +89,7 @@ class GetArtistSerializer(serializers.Serializer): "created": to_subsonic_date(album.creation_date), "songCount": len(album.tracks.all()), } - if album.cover: + if album.attachment_cover_id: album_data["coverArt"] = "al-{}".format(album.id) if album.release_date: album_data["year"] = album.release_date.year @@ -122,7 +122,7 @@ def get_track_data(album, track, upload): "artistId": album.artist.pk, "type": "music", } - if track.album.cover: + if track.album.attachment_cover_id: data["coverArt"] = "al-{}".format(track.album.id) if upload.bitrate: data["bitrate"] = int(upload.bitrate / 1000) @@ -141,7 +141,7 @@ def get_album2_data(album): "artist": album.artist.name, "created": to_subsonic_date(album.creation_date), } - if album.cover: + if album.attachment_cover_id: payload["coverArt"] = "al-{}".format(album.id) try: diff --git a/api/funkwhale_api/subsonic/views.py b/api/funkwhale_api/subsonic/views.py index db2620100..53861572a 100644 --- a/api/funkwhale_api/subsonic/views.py +++ b/api/funkwhale_api/subsonic/views.py @@ -16,7 +16,12 @@ from rest_framework.serializers import ValidationError import funkwhale_api from funkwhale_api.activity import record -from funkwhale_api.common import fields, preferences, utils as common_utils +from funkwhale_api.common import ( + fields, + preferences, + utils as common_utils, + tasks as common_tasks, +) from funkwhale_api.favorites.models import TrackFavorite from funkwhale_api.moderation import filters as moderation_filters from funkwhale_api.music import models as music_models @@ -732,20 +737,23 @@ class SubsonicViewSet(viewsets.GenericViewSet): try: album_id = int(id.replace("al-", "")) album = ( - music_models.Album.objects.exclude(cover__isnull=True) - .exclude(cover="") + music_models.Album.objects.exclude(attachment_cover=None) + .select_related("attachment_cover") .get(pk=album_id) ) except (TypeError, ValueError, music_models.Album.DoesNotExist): return response.Response( {"error": {"code": 70, "message": "cover art not found."}} ) - cover = album.cover + attachment = album.attachment_cover else: return response.Response( {"error": {"code": 70, "message": "cover art not found."}} ) + if not attachment.file: + common_tasks.fetch_remote_attachment(attachment) + cover = attachment.file mapping = {"nginx": "X-Accel-Redirect", "apache2": "X-Sendfile"} path = music_views.get_file_path(cover) file_header = mapping[settings.REVERSE_PROXY_TYPE] diff --git a/api/tests/common/test_models.py b/api/tests/common/test_models.py index a2ea89ef2..b5a56614d 100644 --- a/api/tests/common/test_models.py +++ b/api/tests/common/test_models.py @@ -46,3 +46,28 @@ def test_get_moderation_url(factory_name, factories, expected): obj = factories[factory_name]() assert obj.get_moderation_url() == expected.format(obj=obj) + + +def test_attachment(factories, now): + attachment = factories["common.Attachment"]() + + assert attachment.uuid is not None + assert attachment.mimetype == "image/jpeg" + assert attachment.file is not None + assert attachment.url is not None + assert attachment.actor is not None + assert attachment.creation_date > now + assert attachment.last_fetch_date is None + assert attachment.size > 0 + + +@pytest.mark.parametrize("args, expected", [([], [0]), ([True], [0]), ([False], [1])]) +def test_attachment_queryset_attached(args, expected, factories, queryset_equal_list): + attachments = [ + factories["music.Album"]().attachment_cover, + factories["common.Attachment"](), + ] + + queryset = attachments[0].__class__.objects.attached(*args).order_by("id") + expected_objs = [attachments[i] for i in expected] + assert queryset == expected_objs diff --git a/api/tests/common/test_serializers.py b/api/tests/common/test_serializers.py index 6d20443af..067a9e4a2 100644 --- a/api/tests/common/test_serializers.py +++ b/api/tests/common/test_serializers.py @@ -2,11 +2,13 @@ import os import PIL from django.core.files.uploadedfile import SimpleUploadedFile +from django.urls import reverse import django_filters from funkwhale_api.common import serializers from funkwhale_api.users import models +from funkwhale_api.federation import utils as federation_utils class TestActionFilterSet(django_filters.FilterSet): @@ -182,3 +184,71 @@ def test_strip_exif_field(): cleaned = PIL.Image.open(field.to_internal_value(uploaded)) assert cleaned._getexif() is None + + +def test_attachment_serializer_existing_file(factories, to_api_date): + attachment = factories["common.Attachment"]() + expected = { + "uuid": str(attachment.uuid), + "size": attachment.size, + "mimetype": attachment.mimetype, + "creation_date": to_api_date(attachment.creation_date), + "urls": { + "source": attachment.url, + "original": federation_utils.full_url(attachment.file.url), + "medium_square_crop": federation_utils.full_url( + attachment.file.crop["200x200"].url + ), + }, + # XXX: BACKWARD COMPATIBILITY + "original": federation_utils.full_url(attachment.file.url), + "medium_square_crop": federation_utils.full_url( + attachment.file.crop["200x200"].url + ), + "small_square_crop": federation_utils.full_url( + attachment.file.crop["200x200"].url + ), + "square_crop": federation_utils.full_url(attachment.file.crop["200x200"].url), + } + + serializer = serializers.AttachmentSerializer(attachment) + + assert serializer.data == expected + + +def test_attachment_serializer_remote_file(factories, to_api_date): + attachment = factories["common.Attachment"](file=None) + proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": attachment.uuid}) + expected = { + "uuid": str(attachment.uuid), + "size": attachment.size, + "mimetype": attachment.mimetype, + "creation_date": to_api_date(attachment.creation_date), + # everything is the same, except for the urls field because: + # - the file isn't available on the local pod + # - we need to return different URLs so that the client can trigger + # a fetch and get redirected to the desired version + # + "urls": { + "source": attachment.url, + "original": federation_utils.full_url(proxy_url + "?next=original"), + "medium_square_crop": federation_utils.full_url( + proxy_url + "?next=medium_square_crop" + ), + }, + # XXX: BACKWARD COMPATIBILITY + "original": federation_utils.full_url(proxy_url + "?next=original"), + "medium_square_crop": federation_utils.full_url( + proxy_url + "?next=medium_square_crop" + ), + "square_crop": federation_utils.full_url( + proxy_url + "?next=medium_square_crop" + ), + "small_square_crop": federation_utils.full_url( + proxy_url + "?next=medium_square_crop" + ), + } + + serializer = serializers.AttachmentSerializer(attachment) + + assert serializer.data == expected diff --git a/api/tests/common/test_tasks.py b/api/tests/common/test_tasks.py index f097c4423..fc62d901b 100644 --- a/api/tests/common/test_tasks.py +++ b/api/tests/common/test_tasks.py @@ -1,4 +1,5 @@ import pytest +import datetime from funkwhale_api.common import serializers from funkwhale_api.common import signals @@ -63,3 +64,25 @@ def test_cannot_apply_already_applied_migration(factories): mutation = factories["common.Mutation"](payload={}, is_applied=True) with pytest.raises(mutation.__class__.DoesNotExist): tasks.apply_mutation(mutation_id=mutation.pk) + + +def test_prune_unattached_attachments(factories, settings, now): + settings.ATTACHMENTS_UNATTACHED_PRUNE_DELAY = 5 + attachments = [ + # attached, kept + factories["music.Album"]().attachment_cover, + # recent, kept + factories["common.Attachment"](), + # too old, pruned + factories["common.Attachment"]( + creation_date=now + - datetime.timedelta(seconds=settings.ATTACHMENTS_UNATTACHED_PRUNE_DELAY) + ), + ] + + tasks.prune_unattached_attachments() + + attachments[0].refresh_from_db() + attachments[1].refresh_from_db() + with pytest.raises(attachments[2].DoesNotExist): + attachments[2].refresh_from_db() diff --git a/api/tests/common/test_views.py b/api/tests/common/test_views.py index c1cbfd761..0ca7cbfd9 100644 --- a/api/tests/common/test_views.py +++ b/api/tests/common/test_views.py @@ -1,4 +1,6 @@ +import io import pytest + from django.urls import reverse from funkwhale_api.common import serializers @@ -181,3 +183,69 @@ def test_rate_limit(logged_in_api_client, now_time, settings, mocker): assert response.status_code == 200 assert response.data == expected get_status.assert_called_once_with(expected_ident, now_time) + + +@pytest.mark.parametrize( + "next, expected", + [ + ("original", "original"), + ("medium_square_crop", "medium_square_crop"), + ("unknown", "original"), + ], +) +def test_attachment_proxy_redirects_original( + next, expected, factories, logged_in_api_client, mocker, avatar, r_mock, now +): + attachment = factories["common.Attachment"](file=None) + + avatar_content = avatar.read() + fetch_remote_attachment = mocker.spy(tasks, "fetch_remote_attachment") + m = r_mock.get(attachment.url, body=io.BytesIO(avatar_content)) + proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": attachment.uuid}) + + response = logged_in_api_client.get(proxy_url, {"next": next}) + attachment.refresh_from_db() + + urls = serializers.AttachmentSerializer(attachment).data["urls"] + + assert attachment.file.read() == avatar_content + assert attachment.last_fetch_date == now + fetch_remote_attachment.assert_called_once_with(attachment) + assert len(m.request_history) == 1 + assert response.status_code == 302 + assert response["Location"] == urls[expected] + + +def test_attachment_create(logged_in_api_client, avatar): + actor = logged_in_api_client.user.create_actor() + url = reverse("api:v1:attachments-list") + content = avatar.read() + avatar.seek(0) + payload = {"file": avatar} + response = logged_in_api_client.post(url, payload) + + assert response.status_code == 201 + attachment = actor.attachments.latest("id") + assert attachment.file.read() == content + assert attachment.file.size == len(content) + + +def test_attachment_destroy(factories, logged_in_api_client): + actor = logged_in_api_client.user.create_actor() + attachment = factories["common.Attachment"](actor=actor) + url = reverse("api:v1:attachments-detail", kwargs={"uuid": attachment.uuid}) + response = logged_in_api_client.delete(url) + + assert response.status_code == 204 + with pytest.raises(attachment.DoesNotExist): + attachment.refresh_from_db() + + +def test_attachment_destroy_not_owner(factories, logged_in_api_client): + logged_in_api_client.user.create_actor() + attachment = factories["common.Attachment"]() + url = reverse("api:v1:attachments-detail", kwargs={"uuid": attachment.uuid}) + response = logged_in_api_client.delete(url) + + assert response.status_code == 403 + attachment.refresh_from_db() diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index a0b773fb4..34460a5a6 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -589,7 +589,7 @@ def test_activity_pub_album_serializer_to_ap(factories): "cover": { "type": "Link", "mediaType": "image/jpeg", - "href": utils.full_url(album.cover.url), + "href": utils.full_url(album.attachment_cover.file.url), }, "musicbrainzId": album.mbid, "published": album.creation_date.isoformat(), @@ -729,8 +729,8 @@ def test_activity_pub_track_serializer_from_ap(factories, r_mock, mocker): assert str(track.mbid) == data["musicbrainzId"] assert album.from_activity == activity - assert album.cover.read() == b"coucou" - assert album.cover_path.endswith(".png") + assert album.attachment_cover.file.read() == b"coucou" + assert album.attachment_cover.file.path.endswith(".png") assert album.title == data["album"]["name"] assert album.fid == data["album"]["id"] assert str(album.mbid) == data["album"]["musicbrainzId"] diff --git a/api/tests/manage/test_serializers.py b/api/tests/manage/test_serializers.py index d8a2ee8f9..37490383a 100644 --- a/api/tests/manage/test_serializers.py +++ b/api/tests/manage/test_serializers.py @@ -1,7 +1,8 @@ import pytest -from funkwhale_api.manage import serializers +from funkwhale_api.common import serializers as common_serializers from funkwhale_api.federation import tasks as federation_tasks +from funkwhale_api.manage import serializers def test_manage_upload_action_delete(factories): @@ -339,12 +340,7 @@ def test_manage_nested_album_serializer(factories, now, to_api_date): "mbid": album.mbid, "creation_date": to_api_date(album.creation_date), "release_date": album.release_date.isoformat(), - "cover": { - "original": album.cover.url, - "square_crop": album.cover.crop["400x400"].url, - "medium_square_crop": album.cover.crop["200x200"].url, - "small_square_crop": album.cover.crop["50x50"].url, - }, + "cover": common_serializers.AttachmentSerializer(album.attachment_cover).data, "tracks_count": 44, } s = serializers.ManageNestedAlbumSerializer(album) @@ -380,12 +376,7 @@ def test_manage_album_serializer(factories, now, to_api_date): "mbid": album.mbid, "creation_date": to_api_date(album.creation_date), "release_date": album.release_date.isoformat(), - "cover": { - "original": album.cover.url, - "square_crop": album.cover.crop["400x400"].url, - "medium_square_crop": album.cover.crop["200x200"].url, - "small_square_crop": album.cover.crop["50x50"].url, - }, + "cover": common_serializers.AttachmentSerializer(album.attachment_cover).data, "artist": serializers.ManageNestedArtistSerializer(album.artist).data, "tracks": [serializers.ManageNestedTrackSerializer(track).data], "attributed_to": serializers.ManageBaseActorSerializer( diff --git a/api/tests/music/test_models.py b/api/tests/music/test_models.py index a5090e89a..c723b0104 100644 --- a/api/tests/music/test_models.py +++ b/api/tests/music/test_models.py @@ -192,7 +192,7 @@ def test_album_get_image_content(factories): album.get_image(data={"content": b"test", "mimetype": "image/jpeg"}) album.refresh_from_db() - assert album.cover.read() == b"test" + assert album.attachment_cover.file.read() == b"test" def test_library(factories): diff --git a/api/tests/music/test_music.py b/api/tests/music/test_music.py index 727214af5..0709ab340 100644 --- a/api/tests/music/test_music.py +++ b/api/tests/music/test_music.py @@ -132,11 +132,11 @@ def test_can_download_image_file_for_album(binary_cover, mocker, factories): album.get_image() album.save() - assert album.cover.file.read() == binary_cover + assert album.attachment_cover.file.read() == binary_cover def test_album_get_image_doesnt_crash_with_empty_data(mocker, factories): - album = factories["music.Album"](mbid=None, cover=None) + album = factories["music.Album"](mbid=None, attachment_cover=None) assert ( album.get_image(data={"content": "", "url": "", "mimetype": "image/png"}) is None diff --git a/api/tests/music/test_serializers.py b/api/tests/music/test_serializers.py index 4eaf54ad5..a3e94e768 100644 --- a/api/tests/music/test_serializers.py +++ b/api/tests/music/test_serializers.py @@ -1,5 +1,6 @@ import pytest +from funkwhale_api.common import serializers as common_serializers from funkwhale_api.federation import serializers as federation_serializers from funkwhale_api.music import licenses from funkwhale_api.music import models @@ -42,12 +43,7 @@ def test_artist_album_serializer(factories, to_api_date): "creation_date": to_api_date(album.creation_date), "tracks_count": 1, "is_playable": None, - "cover": { - "original": album.cover.url, - "square_crop": album.cover.crop["400x400"].url, - "medium_square_crop": album.cover.crop["200x200"].url, - "small_square_crop": album.cover.crop["50x50"].url, - }, + "cover": common_serializers.AttachmentSerializer(album.attachment_cover).data, "release_date": to_api_date(album.release_date), "is_local": album.is_local, } @@ -172,12 +168,7 @@ def test_album_serializer(factories, to_api_date): "artist": serializers.serialize_artist_simple(album.artist), "creation_date": to_api_date(album.creation_date), "is_playable": False, - "cover": { - "original": album.cover.url, - "square_crop": album.cover.crop["400x400"].url, - "medium_square_crop": album.cover.crop["200x200"].url, - "small_square_crop": album.cover.crop["50x50"].url, - }, + "cover": common_serializers.AttachmentSerializer(album.attachment_cover).data, "release_date": to_api_date(album.release_date), "tracks": [serializers.serialize_album_track(t) for t in [track2, track1]], "is_local": album.is_local, @@ -189,6 +180,15 @@ def test_album_serializer(factories, to_api_date): assert serializer.data == expected +def test_album_serializer_empty_cover(factories, to_api_date): + # XXX: BACKWARD COMPATIBILITY + album = factories["music.Album"](attachment_cover=None) + + serializer = serializers.AlbumSerializer(album) + + assert serializer.data["cover"] == {} + + def test_track_serializer(factories, to_api_date): actor = factories["federation.Actor"]() upload = factories["music.Upload"]( diff --git a/api/tests/music/test_spa_views.py b/api/tests/music/test_spa_views.py index bf85ab888..42f723460 100644 --- a/api/tests/music/test_spa_views.py +++ b/api/tests/music/test_spa_views.py @@ -49,9 +49,7 @@ def test_library_track(spa_html, no_api_auth, client, factories, settings): { "tag": "meta", "property": "og:image", - "content": utils.join_url( - settings.FUNKWHALE_URL, track.album.cover.crop["400x400"].url - ), + "content": track.album.attachment_cover.download_url_medium_square_crop, }, { "tag": "meta", @@ -116,9 +114,7 @@ def test_library_album(spa_html, no_api_auth, client, factories, settings): { "tag": "meta", "property": "og:image", - "content": utils.join_url( - settings.FUNKWHALE_URL, album.cover.crop["400x400"].url - ), + "content": album.attachment_cover.download_url_medium_square_crop, }, { "tag": "link", @@ -166,9 +162,7 @@ def test_library_artist(spa_html, no_api_auth, client, factories, settings): { "tag": "meta", "property": "og:image", - "content": utils.join_url( - settings.FUNKWHALE_URL, album.cover.crop["400x400"].url - ), + "content": album.attachment_cover.download_url_medium_square_crop, }, { "tag": "link", @@ -217,9 +211,7 @@ def test_library_playlist(spa_html, no_api_auth, client, factories, settings): { "tag": "meta", "property": "og:image", - "content": utils.join_url( - settings.FUNKWHALE_URL, track.album.cover.crop["400x400"].url - ), + "content": track.album.attachment_cover.download_url_medium_square_crop, }, { "tag": "link", diff --git a/api/tests/music/test_tasks.py b/api/tests/music/test_tasks.py index fbbb0197a..65ed3f9b9 100644 --- a/api/tests/music/test_tasks.py +++ b/api/tests/music/test_tasks.py @@ -285,8 +285,11 @@ def test_can_create_track_from_file_metadata_federation(factories, mocker, r_moc assert track.fid == metadata["fid"] assert track.creation_date == metadata["fdate"] assert track.position == 4 - assert track.album.cover.read() == b"coucou" - assert track.album.cover_path.endswith(".png") + assert track.album.attachment_cover.file.read() == b"coucou" + assert track.album.attachment_cover.file.path.endswith(".png") + assert track.album.attachment_cover.url == metadata["cover_data"]["url"] + assert track.album.attachment_cover.mimetype == metadata["cover_data"]["mimetype"] + assert track.album.fid == metadata["album"]["fid"] assert track.album.title == metadata["album"]["title"] assert track.album.creation_date == metadata["album"]["fdate"] @@ -312,7 +315,7 @@ def test_upload_import(now, factories, temp_signal, mocker): update_album_cover = mocker.patch("funkwhale_api.music.tasks.update_album_cover") get_picture = mocker.patch("funkwhale_api.music.metadata.Metadata.get_picture") get_track_from_import_metadata = mocker.spy(tasks, "get_track_from_import_metadata") - track = factories["music.Track"](album__cover="") + track = factories["music.Track"](album__attachment_cover=None) upload = factories["music.Upload"]( track=None, import_metadata={"funkwhale": {"track": {"uuid": str(track.uuid)}}} ) @@ -531,7 +534,7 @@ def test_upload_import_error_metadata(factories, now, temp_signal, mocker): def test_upload_import_updates_cover_if_no_cover(factories, mocker, now): mocked_update = mocker.patch("funkwhale_api.music.tasks.update_album_cover") - album = factories["music.Album"](cover="") + album = factories["music.Album"](attachment_cover=None) track = factories["music.Track"](album=album) upload = factories["music.Upload"]( track=None, import_metadata={"funkwhale": {"track": {"uuid": track.uuid}}} @@ -541,7 +544,7 @@ def test_upload_import_updates_cover_if_no_cover(factories, mocker, now): def test_update_album_cover_mbid(factories, mocker): - album = factories["music.Album"](cover="") + album = factories["music.Album"](attachment_cover=None) mocked_get = mocker.patch("funkwhale_api.music.models.Album.get_image") tasks.update_album_cover(album=album) @@ -550,7 +553,7 @@ def test_update_album_cover_mbid(factories, mocker): def test_update_album_cover_file_data(factories, mocker): - album = factories["music.Album"](cover="", mbid=None) + album = factories["music.Album"](attachment_cover=None, mbid=None) mocked_get = mocker.patch("funkwhale_api.music.models.Album.get_image") tasks.update_album_cover(album=album, cover_data={"hello": "world"}) @@ -563,7 +566,7 @@ def test_update_album_cover_file_cover_separate_file(ext, mimetype, factories, m image_path = os.path.join(DATA_DIR, "cover.{}".format(ext)) with open(image_path, "rb") as f: image_content = f.read() - album = factories["music.Album"](cover="", mbid=None) + album = factories["music.Album"](attachment_cover=None, mbid=None) mocked_get = mocker.patch("funkwhale_api.music.models.Album.get_image") mocker.patch("funkwhale_api.music.metadata.Metadata.get_picture", return_value=None) diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py index 22f7da635..ad3bf0413 100644 --- a/api/tests/music/test_views.py +++ b/api/tests/music/test_views.py @@ -780,10 +780,10 @@ def test_oembed_track(factories, no_api_auth, api_client, settings): "title": "{} by {}".format(track.title, track.artist.name), "description": track.full_name, "thumbnail_url": federation_utils.full_url( - track.album.cover.crop["400x400"].url + track.album.attachment_cover.file.crop["200x200"].url ), - "thumbnail_height": 400, - "thumbnail_width": 400, + "thumbnail_height": 200, + "thumbnail_width": 200, "html": ''.format( iframe_src ), @@ -815,9 +815,11 @@ def test_oembed_album(factories, no_api_auth, api_client, settings): "width": 600, "title": "{} by {}".format(album.title, album.artist.name), "description": "{} by {}".format(album.title, album.artist.name), - "thumbnail_url": federation_utils.full_url(album.cover.crop["400x400"].url), - "thumbnail_height": 400, - "thumbnail_width": 400, + "thumbnail_url": federation_utils.full_url( + album.attachment_cover.file.crop["200x200"].url + ), + "thumbnail_height": 200, + "thumbnail_width": 200, "html": ''.format( iframe_src ), @@ -850,9 +852,11 @@ def test_oembed_artist(factories, no_api_auth, api_client, settings): "width": 600, "title": artist.name, "description": artist.name, - "thumbnail_url": federation_utils.full_url(album.cover.crop["400x400"].url), - "thumbnail_height": 400, - "thumbnail_width": 400, + "thumbnail_url": federation_utils.full_url( + album.attachment_cover.file.crop["200x200"].url + ), + "thumbnail_height": 200, + "thumbnail_width": 200, "html": ''.format( iframe_src ), @@ -886,10 +890,10 @@ def test_oembed_playlist(factories, no_api_auth, api_client, settings): "title": playlist.name, "description": playlist.name, "thumbnail_url": federation_utils.full_url( - track.album.cover.crop["400x400"].url + track.album.attachment_cover.file.crop["200x200"].url ), - "thumbnail_height": 400, - "thumbnail_width": 400, + "thumbnail_height": 200, + "thumbnail_width": 200, "html": ''.format( iframe_src ), diff --git a/api/tests/playlists/test_serializers.py b/api/tests/playlists/test_serializers.py index f84df4bb2..79105d3cc 100644 --- a/api/tests/playlists/test_serializers.py +++ b/api/tests/playlists/test_serializers.py @@ -95,7 +95,7 @@ def test_playlist_serializer_include_covers(factories, api_request): playlist = factories["playlists.Playlist"]() t1 = factories["music.Track"]() t2 = factories["music.Track"]() - t3 = factories["music.Track"](album__cover=None) + t3 = factories["music.Track"](album__attachment_cover=None) t4 = factories["music.Track"]() t5 = factories["music.Track"]() t6 = factories["music.Track"]() @@ -106,11 +106,11 @@ def test_playlist_serializer_include_covers(factories, api_request): qs = playlist.__class__.objects.with_covers().with_tracks_count() expected = [ - request.build_absolute_uri(t1.album.cover.crop["200x200"].url), - request.build_absolute_uri(t2.album.cover.crop["200x200"].url), - request.build_absolute_uri(t4.album.cover.crop["200x200"].url), - request.build_absolute_uri(t5.album.cover.crop["200x200"].url), - request.build_absolute_uri(t6.album.cover.crop["200x200"].url), + t1.album.attachment_cover.download_url_medium_square_crop, + t2.album.attachment_cover.download_url_medium_square_crop, + t4.album.attachment_cover.download_url_medium_square_crop, + t5.album.attachment_cover.download_url_medium_square_crop, + t6.album.attachment_cover.download_url_medium_square_crop, ] serializer = serializers.PlaylistSerializer(qs.get(), context={"request": request}) diff --git a/api/tests/subsonic/test_views.py b/api/tests/subsonic/test_views.py index 3dd9f6cc6..0e7acfd66 100644 --- a/api/tests/subsonic/test_views.py +++ b/api/tests/subsonic/test_views.py @@ -724,7 +724,7 @@ def test_get_cover_art_album(factories, logged_in_api_client): assert response.status_code == 200 assert response["Content-Type"] == "" assert response["X-Accel-Redirect"] == music_views.get_file_path( - album.cover + album.attachment_cover.file ).decode("utf-8") diff --git a/docs/swagger.yml b/docs/swagger.yml index a6952098b..eff119aa5 100644 --- a/docs/swagger.yml +++ b/docs/swagger.yml @@ -185,6 +185,8 @@ tags: url: https://docs.funkwhale.audio/users/managing.html - name: Content curation description: Favorites, playlists, radios + - name: Other + description: Other endpoints that don't fit in the categories above paths: /api/v1/oauth/apps/: @@ -1022,6 +1024,54 @@ paths: 204: $ref: "#/responses/204" + /api/v1/attachments/: + post: + tags: + - "Other" + description: + Upload a new file as an attachment that can be later associated with other objects. + responses: + 201: + $ref: "#/responses/201" + 400: + $ref: "#/responses/400" + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + file: + type: string + format: binary + + /api/v1/attachments/{uuid}/: + parameters: + - name: uuid + in: path + required: true + schema: + type: "string" + format: "uuid" + get: + summary: Retrieve an attachment + tags: + - "Other" + responses: + 200: + content: + application/json: + schema: + $ref: "#/definitions/Attachment" + delete: + summary: Delete an attachment + tags: + - "Other" + responses: + 204: + $ref: "#/responses/204" + parameters: ObjectId: name: id @@ -1114,6 +1164,12 @@ properties: - "audio/mpeg" - "audio/x-flac" - "audio/flac" + image_mimetype: + type: string + example: "image/png" + enum: + - "image/png" + - "image/jpeg" import_status: type: string example: "finished" @@ -1180,28 +1236,33 @@ definitions: format: "uri" description: "Link to the previous page of results" - - Image: + Attachment: type: "object" properties: - original: + uuid: + type: string + format: uuid + size: + type: "integer" + format: "int64" + example: 2787000 + description: "Size of the file, in bytes" + mimetype: + $ref: "#/properties/image_mimetype" + creation_date: type: "string" - description: "URL to the original image" - example: "https://mydomain/media/albums/covers/ec2c53aeaac6.jpg" - small_square_crop: - type: "string" - description: "URL to a small, squared thumbnail of the image" - example: "https://mydomain/media/__sized__/albums/covers/ec2c53aeaac6-crop-c0-5__0-5-50x50-70.jpg" - - medium_square_crop: - type: "string" - description: "URL to a medium, squared thumbnail of the image" - example: "https://mydomain/media/__sized__/albums/covers/ec2c53aeaac6-crop-c0-5__0-5-200x200-70.jpg" - - square_crop: - type: "string" - description: "URL to a large, squared thumbnail of the image" - example: "https://mydomain/media/__sized__/albums/covers/ec2c53aeaac6-crop-c0-5__0-5-400x400-70.jpg" + format: "date-time" + urls: + type: "object" + properties: + original: + type: "string" + description: "URL to the original image" + example: "https://mydomain/media/attachments/ec2c53aeaac6.jpg" + medium_square_crop: + type: "string" + description: "URL to a medium, squared thumbnail of the image" + example: "https://mydomain/media/__sized__/attachments/ec2c53aeaac6-crop-c0-5__0-5-200x200-70.jpg" Actor: type: object @@ -1317,7 +1378,7 @@ definitions: is_playable: type: "boolean" cover: - $ref: "#/definitions/Image" + $ref: "#/definitions/Attachment" is_local: type: "boolean" description: "Indicates if the object was initally created locally or on another server" @@ -1508,7 +1569,7 @@ definitions: uuid: type: string format: uuid - size: + size:size: type: "integer" format: "int64" example: 278987000