diff --git a/api/funkwhale_api/common/schema.yml b/api/funkwhale_api/common/schema.yml index 8289537e5..f25354a85 100644 --- a/api/funkwhale_api/common/schema.yml +++ b/api/funkwhale_api/common/schema.yml @@ -2462,6 +2462,9 @@ paths: responses: '201': content: + application/activity+json: + schema: + $ref: '#/components/schemas/Fetch' application/json: schema: $ref: '#/components/schemas/Fetch' @@ -2484,6 +2487,9 @@ paths: responses: '200': content: + application/activity+json: + schema: + $ref: '#/components/schemas/Fetch' application/json: schema: $ref: '#/components/schemas/Fetch' @@ -9320,6 +9326,36 @@ paths: schema: $ref: '#/components/schemas/UploadForOwner' description: '' + /api/v1/uploads/trigger-download/: + post: + operationId: create_upload_trigger_download + tags: + - uploads + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UploadForOwnerRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/UploadForOwnerRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadForOwnerRequest' + application/activity+json: + schema: + $ref: '#/components/schemas/UploadForOwnerRequest' + required: true + security: + - oauth2: [] + - ApplicationToken: [] + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UploadForOwner' + description: '' /api/v1/users/{username}/: put: operationId: update_user @@ -11972,6 +12008,9 @@ paths: responses: '201': content: + application/activity+json: + schema: + $ref: '#/components/schemas/Fetch' application/json: schema: $ref: '#/components/schemas/Fetch' @@ -11994,6 +12033,9 @@ paths: responses: '200': content: + application/activity+json: + schema: + $ref: '#/components/schemas/Fetch' application/json: schema: $ref: '#/components/schemas/Fetch' @@ -18995,6 +19037,36 @@ paths: schema: $ref: '#/components/schemas/UploadForOwner' description: '' + /api/v2/uploads/trigger-download/: + post: + operationId: create_upload_trigger_download_2 + tags: + - uploads + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UploadForOwnerRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/UploadForOwnerRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadForOwnerRequest' + application/activity+json: + schema: + $ref: '#/components/schemas/UploadForOwnerRequest' + required: true + security: + - oauth2: [] + - ApplicationToken: [] + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UploadForOwner' + description: '' /api/v2/users/{username}/: put: operationId: update_user_3 @@ -24755,6 +24827,10 @@ components: default: pending privacy_level: $ref: '#/components/schemas/LibraryPrivacyLevelEnum' + third_party_provider: + type: string + nullable: true + maxLength: 100 import_metadata: $ref: '#/components/schemas/ImportMetadataRequest' import_reference: @@ -25896,6 +25972,10 @@ components: default: pending privacy_level: $ref: '#/components/schemas/LibraryPrivacyLevelEnum' + third_party_provider: + type: string + nullable: true + maxLength: 100 import_details: readOnly: true import_metadata: @@ -25939,6 +26019,10 @@ components: default: pending privacy_level: $ref: '#/components/schemas/LibraryPrivacyLevelEnum' + third_party_provider: + type: string + nullable: true + maxLength: 100 import_metadata: $ref: '#/components/schemas/ImportMetadataRequest' import_reference: diff --git a/api/funkwhale_api/contrib/archivedl/funkwhale_ready.py b/api/funkwhale_api/contrib/archivedl/funkwhale_ready.py index 38425f3ed..5f500f408 100644 --- a/api/funkwhale_api/contrib/archivedl/funkwhale_ready.py +++ b/api/funkwhale_api/contrib/archivedl/funkwhale_ready.py @@ -9,5 +9,5 @@ logger = logging.getLogger(__name__) @plugins.register_hook(plugins.TRIGGER_THIRD_PARTY_UPLOAD, PLUGIN) -def lauch_download(track, conf={}): +def lauch_download(track, actor, conf={}): tasks.archive_download.delay(track_id=track.pk, conf=conf) diff --git a/api/funkwhale_api/contrib/archivedl/tasks.py b/api/funkwhale_api/contrib/archivedl/tasks.py index 51e0d9da4..f4bbb4cd2 100644 --- a/api/funkwhale_api/contrib/archivedl/tasks.py +++ b/api/funkwhale_api/contrib/archivedl/tasks.py @@ -53,7 +53,7 @@ def check_last_third_party_queries(track, count): check_last_third_party_queries(track, count) -def create_upload(url, track, files_data): +def get_or_create_upload(url, track, files_data): mimetype = f"audio/{files_data.get('format', 'unknown')}" duration = files_data.get("mtime", 0) filesize = files_data.get("size", 0) @@ -64,19 +64,23 @@ def create_upload(url, track, files_data): actor=actors.get_service_actor(), ) - return models.Upload.objects.create( + defaults = { + "creation_date": timezone.now(), + "duration": duration, + "size": filesize, + "bitrate": bitrate, + "import_status": "pending", + } + upload, created = models.Upload.objects.get_or_create( + defaults=defaults, mimetype=mimetype, source=url, third_party_provider="archive-dl", - creation_date=timezone.now(), track=track, - duration=duration, - size=filesize, - bitrate=bitrate, library=service_library, from_activity=None, - import_status="pending", ) + return upload @celery.app.task(name="archivedl.archive_download") @@ -89,9 +93,11 @@ def archive_download(track, conf): logger.error(e) return artist_name = utils.get_artist_credit_string(track) + # a lot of times this don't find anything, archive relies more on albums than tracks query = f"mediatype:audio AND title:{track.title} AND creator:{artist_name}" with requests.Session() as session: url = get_search_url(query, page_size=1, page=1) + get_or_create_upload(url, track, files_data={}) page_data = fetch_json(url, session) for obj in page_data["response"]["docs"]: logger.info(f"launching download item for {str(obj)}") @@ -123,7 +129,7 @@ def download_item( ) ) url = f"https://archive.org/download/{item_data['identifier']}/{to_download[0]['name']}" - upload = create_upload(url, track, to_download[0]) + upload = get_or_create_upload(url, track, to_download[0]) try: with tempfile.TemporaryDirectory() as temp_dir: path = os.path.join(temp_dir, to_download[0]["name"]) diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py index c3e103af7..1fd9afdc1 100644 --- a/api/funkwhale_api/federation/api_serializers.py +++ b/api/funkwhale_api/federation/api_serializers.py @@ -13,7 +13,9 @@ from funkwhale_api.audio import models as audio_models from funkwhale_api.audio import serializers as audio_serializers from funkwhale_api.common import serializers as common_serializers from funkwhale_api.music import models as music_models +from funkwhale_api.music import serializers as music_serializers from funkwhale_api.playlists import models as playlists_models +from funkwhale_api.playlists import serializers as playlist_serializers from funkwhale_api.users import serializers as users_serializers from . import filters, models @@ -197,10 +199,19 @@ OBJECT_SERIALIZER_MAPPING = { music_models.Artist: federation_serializers.ArtistSerializer, music_models.Album: federation_serializers.AlbumSerializer, music_models.Track: federation_serializers.TrackSerializer, + music_models.Library: federation_serializers.LibrarySerializer, models.Actor: federation_serializers.APIActorSerializer, audio_models.Channel: audio_serializers.ChannelSerializer, playlists_models.Playlist: federation_serializers.PlaylistSerializer, } +OBJECT_MUSIC_SERIALIZER_MAPPING = { + music_models.Artist: music_serializers.ArtistSerializer, + music_models.Album: music_serializers.AlbumSerializer, + music_models.Track: music_serializers.TrackSerializer, + models.Actor: federation_serializers.APIActorSerializer, + audio_models.Channel: audio_serializers.ChannelSerializer, + playlists_models.Playlist: playlist_serializers.PlaylistSerializer, +} def convert_url_to_webfinger(url): @@ -283,6 +294,9 @@ class FetchSerializer(serializers.ModelSerializer): return value return f"webfinger://{value}" + # to do : this is incomplete, schema conflict because + # federation serializers have the same name than musi serializer -> upgrade fed serializers to new names + # and add the new object here @extend_schema_field( { "oneOf": [ @@ -300,7 +314,12 @@ class FetchSerializer(serializers.ModelSerializer): if obj is None: return None - serializer_class = OBJECT_SERIALIZER_MAPPING.get(type(obj)) + media_type = self.context.get("media_type") + + if media_type == "application/activity+json": + serializer_class = OBJECT_SERIALIZER_MAPPING.get(type(obj)) + else: + serializer_class = OBJECT_MUSIC_SERIALIZER_MAPPING.get(type(obj)) if serializer_class: return serializer_class(obj).data return None diff --git a/api/funkwhale_api/federation/api_views.py b/api/funkwhale_api/federation/api_views.py index a076f5a1d..3386dd173 100644 --- a/api/funkwhale_api/federation/api_views.py +++ b/api/funkwhale_api/federation/api_views.py @@ -1,3 +1,5 @@ +from urllib.parse import urlparse + import requests.exceptions from django.conf import settings from django.db import transaction @@ -5,10 +7,13 @@ from django.db.models import Count, Q from drf_spectacular.utils import extend_schema, extend_schema_view from rest_framework import decorators, mixins, permissions, response, viewsets from rest_framework.exceptions import NotFound as RestNotFound +from rest_framework.negotiation import DefaultContentNegotiation +from rest_framework.renderers import JSONRenderer from funkwhale_api.common import preferences from funkwhale_api.common import utils as common_utils from funkwhale_api.common.permissions import ConditionalAuthentication +from funkwhale_api.common.renderers import ActivityStreamRenderer from funkwhale_api.music import models as music_models from funkwhale_api.music import serializers as music_serializers from funkwhale_api.music import views as music_views @@ -243,20 +248,41 @@ class FetchViewSet( serializer_class = api_serializers.FetchSerializer permission_classes = [permissions.IsAuthenticated] throttling_scopes = {"create": {"authenticated": "fetch"}} + renderer_classes = [ActivityStreamRenderer, JSONRenderer] def get_queryset(self): return super().get_queryset().filter(actor=self.request.user.actor) + def get_serializer_context(self): + context = super().get_serializer_context() + + negotiator = DefaultContentNegotiation() + try: + renderer, media_type = negotiator.select_renderer( + self.request, self.get_renderers() + ) + context["media_type"] = media_type + except Exception: + context["media_type"] = None + return context + def perform_create(self, serializer): fetch = serializer.save(actor=self.request.user.actor) if fetch.status == "finished": # a duplicate was returned, no need to fetch again return - if settings.FEDERATION_SYNCHRONOUS_FETCH: - tasks.fetch(fetch_id=fetch.pk) + + parsed_url = urlparse(fetch.url) + domain = parsed_url.netloc + if domain in fetch.supported_services: + tasks.third_party_fetch(fetch_id=fetch.pk) fetch.refresh_from_db() else: - common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk) + if settings.FEDERATION_SYNCHRONOUS_FETCH: + tasks.fetch(fetch_id=fetch.pk) + fetch.refresh_from_db() + else: + common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk) class DomainViewSet( diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index b01355493..fbec8d601 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -18,6 +18,7 @@ from funkwhale_api.common import session from funkwhale_api.common import utils as common_utils from funkwhale_api.common import validators as common_validators from funkwhale_api.music import utils as music_utils +from funkwhale_api.musicbrainz import serializers as musicbrainz_serializers from . import utils as federation_utils @@ -411,8 +412,15 @@ class Fetch(models.Model): contexts.AS.Organization: [serializers.ActorSerializer], contexts.AS.Service: [serializers.ActorSerializer], contexts.AS.Application: [serializers.ActorSerializer], + # for mb the key must be the api namespace + "recordings": [musicbrainz_serializers.RecordingSerializer], + "releases": [musicbrainz_serializers.ReleaseSerializer], } + @property + def supported_services(self): + return ["musicbrainz.org"] + class InboxItem(models.Model): """ diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index dcdaca87d..490a0b3b6 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -1066,6 +1066,7 @@ class LibrarySerializer(PaginatedCollectionSerializer): privacy = {"": "me", "./": "me", None: "me", contexts.AS.Public: "everyone"} library, created = music_models.Library.objects.update_or_create( fid=validated_data["id"], + uuid=validated_data["id"].rstrip("/").split("/")[-1], actor=actor, defaults={ "uploads_count": validated_data["totalItems"], @@ -1449,7 +1450,7 @@ class AlbumSerializer(MusicEntitySerializer): acs.append( utils.retrieve_ap_object( ac["id"], - actor=self.context.get("fetch_actor"), + actor=self.context.get("_actor"), queryset=music_models.ArtistCredit, serializer_class=ArtistCreditSerializer, ) diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index fbdf83a0c..dbf3168f9 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -2,9 +2,12 @@ import datetime import json import logging import os +import uuid +from urllib.parse import urlparse import requests from django.conf import settings +from django.contrib.contenttypes.models import ContentType from django.core.cache import cache from django.db import transaction from django.db.models import F, Q @@ -13,6 +16,8 @@ from django.utils import timezone from dynamic_preferences.registries import global_preferences_registry from requests.exceptions import RequestException +from config import plugins +from funkwhale_api import musicbrainz from funkwhale_api.audio import models as audio_models from funkwhale_api.common import models as common_models from funkwhale_api.common import preferences, session @@ -449,7 +454,6 @@ def fetch(fetch_obj): max_pages=settings.FEDERATION_COLLECTION_MAX_PAGES - 1, is_page=True, ) - fetch_obj.object = obj fetch_obj.status = "finished" fetch_obj.fetch_date = timezone.now() @@ -458,6 +462,167 @@ def fetch(fetch_obj): ) +def trigger_third_party_upload_hook(fetch): + if fetch.status == "finished" and fetch.object: + if fetch.object_content_type == ContentType.objects.get_for_model( + music_models.Track + ): + if not music_models.Track.objects.filter(pk=fetch.object.pk).playable_by( + fetch.actor + ): + plugins.trigger_hook( + plugins.TRIGGER_THIRD_PARTY_UPLOAD, + track=fetch.object, + ) + if fetch.object_content_type == ContentType.objects.get_for_model( + music_models.Album + ): + for track in fetch.object.tracks.all(): + if not music_models.Track.objects.filter( + pk=fetch.object.pk + ).playable_by(fetch.actor): + plugins.trigger_hook( + plugins.TRIGGER_THIRD_PARTY_UPLOAD, + track=track, + ) + + +def musicbrainz_type_handler(fetch): + url = fetch.url + path_parts = urlparse(url).path.strip("/").split("/") + type_ = path_parts[0] + "s" + mbid = path_parts[1] + try: + uuid.UUID(mbid) + except ValueError: + raise ValueError(f"could no get mbid from url {url}") + return type_, mbid + + +def musicbrainz_metadata_handler(type_, id): + def replace_hyphens_in_keys(obj): + if isinstance(obj, dict): + return { + k.replace("-", "_"): replace_hyphens_in_keys(v) for k, v in obj.items() + } + elif isinstance(obj, list): + return [replace_hyphens_in_keys(item) for item in obj] + else: + return obj + + if type_ == "recordings": + includes = ["tags", "artists", "releases"] + elif type_ == "releases": + includes = ["tags", "artists", "recordings"] + + result = replace_hyphens_in_keys( + getattr(musicbrainz.api, type_).get(id=id, includes=includes) + ) + + existing = ( + music_models.Track.objects.filter(mbid=id).first() + if music_models.Track.objects.filter(mbid=id).exists() + else None + ) + return result, existing + + +type_and_id_from_third_party = {"musicbrainz.org": musicbrainz_type_handler} +metadata_from_third_party_ = {"musicbrainz.org": musicbrainz_metadata_handler} + + +@celery.app.task(name="third_party_fetch") +@transaction.atomic +@celery.require_instance( + models.Fetch.objects.filter(status="pending").select_related("actor"), + "fetch_obj", + "fetch_id", +) +def third_party_fetch(fetch_obj): + def error(code, **kwargs): + fetch_obj.status = "errored" + fetch_obj.fetch_date = timezone.now() + fetch_obj.detail = {"error_code": code} + fetch_obj.detail.update(kwargs) + fetch_obj.save(update_fields=["fetch_date", "status", "detail"]) + + def check_url(url): + if not url.startswith("webfinger://"): + payload, updated = mrf.inbox.apply({"id": url}) + if not payload: + return error("blocked", message="Blocked by MRF") + + parsed_url = urlparse(url) + service = parsed_url.netloc + if service not in fetch_obj.supported_services: + return error("invalid_url", message=f"Unsupported domain {service}") + return service + + url = fetch_obj.url + actor = fetch_obj.actor + service = check_url(url) + + try: + type_, id = type_and_id_from_third_party[service](fetch_obj) + logger.debug("Parsed URL %s into type %s and id %s", url, type_, id) + except ValueError as e: + return error("url_parse_error", message=str(e)) + + try: + result, existing = metadata_from_third_party_[service](type_, id) + logger.debug( + f"Remote answered with {result} and we found {existing} in database" + ) + except requests.exceptions.HTTPError as e: + return error( + "http", + status_code=e.response.status_code if e.response else None, + message=e.response.text, + ) + except requests.exceptions.Timeout: + return error("timeout") + except requests.exceptions.ConnectionError as e: + return error("connection", message=str(e)) + except requests.RequestException as e: + return error("request", message=str(e)) + except Exception as e: + return error("unhandled", message=str(e)) + + try: + serializer_classes = fetch_obj.serializers.get(type_) + except (KeyError, AttributeError): + fetch_obj.status = "skipped" + fetch_obj.fetch_date = timezone.now() + fetch_obj.detail = {"reason": "unhandled_type", "type": type_} + return fetch_obj.save(update_fields=["fetch_date", "status", "detail"]) + + serializer = None + for serializer_class in serializer_classes: + serializer = serializer_class( + existing, data=result, context={"fetch_actor": actor} + ) + if not serializer.is_valid(): + continue + else: + break + if serializer.errors: + return error("validation", validation_errors=serializer.errors) + try: + obj = serializer.save() + except Exception as e: + error("save", message=str(e)) + raise + + fetch_obj.object = obj + fetch_obj.status = "finished" + fetch_obj.fetch_date = timezone.now() + trigger_third_party_upload_hook(fetch_obj) + fetch_obj.save( + update_fields=["fetch_date", "status", "object_id", "object_content_type"] + ) + return fetch_obj + + class PreserveSomeDataCollector(Collector): """ We need to delete everything related to an actor. Well… Almost everything. diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index 73907e090..76a7abe32 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -401,6 +401,7 @@ class UploadSerializer(serializers.ModelSerializer): "import_date", "import_status", "privacy_level", + "third_party_provider", ] read_only_fields = [ diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index 894878cba..5cf51f690 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -18,6 +18,7 @@ from rest_framework import views, viewsets from rest_framework.decorators import action from rest_framework.response import Response +from config import plugins from funkwhale_api.common import decorators as common_decorators from funkwhale_api.common import permissions as common_permissions from funkwhale_api.common import preferences @@ -821,6 +822,27 @@ class UploadViewSet( status=200, ) + @action(methods=["post"], detail=False, url_path="trigger-download") + def trigger_download(self, request, *args, **kwargs): + qs = self.get_queryset() + track = models.Track.objects.get(pk=request.data["track"]) + actor = utils.get_actor_from_request(self.request) + tp_upload = models.Upload.objects.filter( + track=track, third_party_provider__isnull=False + ) + if tp_upload.exists(): + return Response( + serializers.UploadSerializer(tp_upload.first()).data, + status=200, + ) + if not qs.filter(track=track).exists(): + plugins.trigger_hook( + plugins.TRIGGER_THIRD_PARTY_UPLOAD, + track=track, + actor=actor, + ) + return Response(status=404) + @action(methods=["post"], detail=False) def action(self, request, *args, **kwargs): queryset = self.get_queryset() diff --git a/api/funkwhale_api/musicbrainz/serializers.py b/api/funkwhale_api/musicbrainz/serializers.py new file mode 100644 index 000000000..787123bf4 --- /dev/null +++ b/api/funkwhale_api/musicbrainz/serializers.py @@ -0,0 +1,251 @@ +import logging + +from rest_framework import serializers + +from funkwhale_api import musicbrainz +from funkwhale_api.tags import models as tags_models + +from . import client + +logger = logging.getLogger(__name__) + + +class ArtistSerializer(serializers.Serializer): + """ + Serializer for Musicbrainz artist data. + """ + + id = serializers.CharField() + name = serializers.CharField() + + def create(self, validated_data): + from funkwhale_api.music.models import Artist + + data = { + "name": validated_data["name"], + "mbid": validated_data["id"], + } + artist, created = Artist.objects.get_or_create(**data) + return artist + + +class ArtistCreditSerializer(serializers.Serializer): + """ + Serializer for Musicbrainz artist data. + """ + + name = serializers.CharField() + joinphrase = serializers.CharField(allow_blank=True) + artist = ArtistSerializer() + + def create(self, validated_data): + from funkwhale_api.music.models import ArtistCredit + + data = { + "credit": validated_data["name"], + "joinphrase": validated_data.get("joinphrase", ""), + "artist": ArtistSerializer().create(validated_data["artist"]), + } + artist_credit, created = ArtistCredit.objects.get_or_create(**data) + return artist_credit + + +class ReleaseForRecordingSerializer(serializers.Serializer): + """ + Serializer for Musicbrainz release data when returned in a recording object. + """ + + id = serializers.CharField() + title = serializers.CharField() + artist_credit = ArtistCreditSerializer(many=True) + tags = serializers.ListField(child=serializers.CharField(), allow_empty=True) + date = serializers.DateField(input_formats=["%Y", "%Y/%m/%d", "%Y-%m-%d"]) + + def create(self, validated_data): + from funkwhale_api.music.models import Album + + data = { + "title": validated_data["title"], + "mbid": validated_data["id"], + "release_date": validated_data.get("date", None), + } + album, created = Album.objects.get_or_create(**data) + artist_credit = ArtistCreditSerializer(many=True).create( + validated_data["artist_credit"] + ) + album.artist_credit.set(artist_credit) + album.save() + tags_models.add_tags(album, *validated_data.get("tags", [])) + return album + + def update(self, instance, validated_data): + instance.title = validated_data["title"] + instance.release_date = validated_data.get("date") + instance.save() + tags_models.add_tags(instance, *validated_data.get("tags", [])) + + return instance + + +class RecordingSerializer(serializers.Serializer): + """ + Serializer for Musicbrainz track data. + """ + + id = serializers.CharField() + title = serializers.CharField() + artist_credit = ArtistCreditSerializer(many=True) + releases = ReleaseForRecordingSerializer(many=True, required=False) + tags = serializers.ListField(child=serializers.CharField(), allow_empty=True) + position = serializers.IntegerField(required=False, allow_null=True) + + def create(self, validated_data): + from funkwhale_api.music.models import Track + + data = {"mbid": validated_data["id"]} + defaults = { + "title": validated_data["title"], + "mbid": validated_data["id"], + # In mb a recording can have various releases, we take the fist one + "album": ( + ReleaseForRecordingSerializer(many=True).create( + validated_data["releases"] + )[0] + if validated_data.get("releases") + else None + ), + # this will be none if the recording is not fetched from the release endpoint + "position": validated_data.get("position", None), + } + + if defaults["album"] is not None and defaults["position"] is None: + result = client.api.releases.get( + id=validated_data["releases"][0]["id"], + includes=["tags", "artists", "recordings"], + ) + tracks = result["media"][0]["tracks"] + defaults["position"] = next( + (o for o in tracks if o["recording"]["id"] == data["mbid"]), {} + ).get("position", None) + + track, created = Track.objects.get_or_create(**data, defaults=defaults) + artist_credit = ArtistCreditSerializer(many=True).create( + validated_data["artist_credit"] + ) + track.artist_credit.set(artist_credit) + track.save() + + tags_models.add_tags(track, *validated_data.get("tags", [])) + + return track + + def update(self, instance, validated_data): + instance.title = validated_data["title"] + instance.save() + tags_models.add_tags(instance, *validated_data.get("tags", [])) + + return instance + + +class RecordingForReleaseSerializer(serializers.Serializer): + id = serializers.CharField() + title = serializers.CharField() + # not in Musicbrainz recording object, but used to store the position of the track in the album + position = serializers.IntegerField(required=False, allow_null=True) + + def create(self, validated_data): + def replace_hyphens_in_keys(obj): + if isinstance(obj, dict): + return { + k.replace("-", "_"): replace_hyphens_in_keys(v) + for k, v in obj.items() + } + elif isinstance(obj, list): + return [replace_hyphens_in_keys(item) for item in obj] + else: + return obj + + recordings_data = musicbrainz.api.recordings.get( + id=validated_data["id"], includes=["tags", "artists"] + ) + recordings_data = replace_hyphens_in_keys(recordings_data) + recordings_data["position"] = validated_data.get("position", None) + serializer = RecordingSerializer(data=recordings_data) + serializer.is_valid(raise_exception=True) + track = serializer.save() + track.album = validated_data["album"] + track.save() + return track + + def update(self, instance, validated_data): + instance.title = validated_data["title"] + tags_models.add_tags(instance, *validated_data.get("tags", [])) + instance.album = validated_data["album"] + instance.save() + return instance + + +class TrackSerializer(serializers.Serializer): + recording = RecordingForReleaseSerializer() + position = serializers.IntegerField() + + +class MediaSerializer(serializers.Serializer): + tracks = TrackSerializer(many=True) + + +class ReleaseSerializer(serializers.Serializer): + """ + Serializer for Musicbrainz release data. + """ + + id = serializers.CharField() + title = serializers.CharField() + artist_credit = ArtistCreditSerializer(many=True) + tags = serializers.ListField(child=serializers.CharField(), allow_empty=True) + date = serializers.DateField(input_formats=["%Y", "%Y/%m/%d", "%Y-%m-%d"]) + media = serializers.ListField(child=MediaSerializer()) + + def create(self, validated_data): + from funkwhale_api.music.models import Album + + data = { + "title": validated_data["title"], + "mbid": validated_data["id"], + "release_date": validated_data.get("date"), + } + album, created = Album.objects.get_or_create(**data) + artist_credit = ArtistCreditSerializer(many=True).create( + validated_data["artist_credit"] + ) + album.artist_credit.set(artist_credit) + album.save() + tags_models.add_tags(album, *validated_data.get("tags", [])) + + # an album can have various media/physical representation, we take the first one + tracks = [t for t in validated_data["media"][0]["tracks"]] + recordings = [] + for t in tracks: + t["recording"]["position"] = t["position"] + recordings.append(t["recording"]) + + for r in recordings: + r["album"] = album + RecordingForReleaseSerializer().create(r) + + return album + + # this will never be used while FetchViewSet and third_party_fetch filter out finished fetch + # Would be nice to have a way to manually update releases from Musicbrainz + def update(self, instance, validated_data): + logger.info(f"Updating release {instance} with data: {validated_data}") + instance.title = validated_data["title"] + instance.release_date = validated_data.get("date") + instance.save() + tags_models.add_tags(instance, *validated_data.get("tags", [])) + + recordings = [t["recording"] for t in validated_data["media"][0]["tracks"]] + for r in recordings: + r["album"] = instance + RecordingForReleaseSerializer().update(r) + return instance diff --git a/api/tests/federation/test_api_serializers.py b/api/tests/federation/test_api_serializers.py index 25339b345..730cdf6e2 100644 --- a/api/tests/federation/test_api_serializers.py +++ b/api/tests/federation/test_api_serializers.py @@ -173,7 +173,12 @@ def test_fetch_serializer_with_object( "actor": serializers.APIActorSerializer(fetch.actor).data, } - assert api_serializers.FetchSerializer(fetch).data == expected + assert ( + api_serializers.FetchSerializer( + fetch, context={"media_type": "application/activity+json"} + ).data + == expected + ) def test_fetch_serializer_unhandled_obj(factories, to_api_date): diff --git a/changes/changelog.d/2452.feature b/changes/changelog.d/2452.feature new file mode 100644 index 000000000..a0700aabb --- /dev/null +++ b/changes/changelog.d/2452.feature @@ -0,0 +1 @@ +Import tracks and albums from Musicbrainz into Funkwhale though the search bar (#2926) diff --git a/docs/specs/fetch-third-party/index.md b/docs/specs/fetch-third-party/index.md new file mode 100644 index 000000000..467704e1e --- /dev/null +++ b/docs/specs/fetch-third-party/index.md @@ -0,0 +1,22 @@ +# Collections + +## The issue + +Has a user I want to be able to get metadata from third party services (to add tracks to my favorites or to a playlist) +Has a user I want to clone a third party playslit into Funkwhale + +## Solution + +paste the audio object (track, album, playlist) url into the search bar. Funkwhale will get the mmetadata and create the objects in db. First implementation with Musicbrainz : + +- track : https://musicbrainz.org/release/{mbid} +- release : https://musicbrainz.org/recording/{mbid} + +## Implementation + +third_party_fetch in federation.tasks +musicbrainz.serializers + +## Call for developers + +This is time consuming. The main logic is implemented. Now everybody can add support for any third party service. diff --git a/front/src/components/audio/PlayButton.vue b/front/src/components/audio/PlayButton.vue index b1debb5d3..3d5960f7d 100644 --- a/front/src/components/audio/PlayButton.vue +++ b/front/src/components/audio/PlayButton.vue @@ -3,7 +3,7 @@ import type { Track, Artist, Album, Playlist, Library, Channel, Actor } from '~/ import type { components } from '~/generated/types' import type { PlayOptionsProps } from '~/composables/audio/usePlayOptions' -import { computed, ref } from 'vue' +import { computed, ref, watch } from 'vue' import { useI18n } from 'vue-i18n' import usePlayOptions from '~/composables/audio/usePlayOptions' import useReport from '~/composables/moderation/useReport' @@ -14,6 +14,7 @@ import Button from '~/components/ui/Button.vue' import OptionsButton from '~/components/ui/button/Options.vue' import Popover from '~/components/ui/Popover.vue' import PopoverItem from '~/components/ui/popover/PopoverItem.vue' +import axios from 'axios' interface Props extends PlayOptionsProps { split?: boolean @@ -103,10 +104,36 @@ const labels = computed(() => ({ PlaylistUploadGranted: t('components.audio.PlayButton.button.PlaylistUploadGranted'), PlaylistUploadPending:t('components.audio.PlayButton.button.PlaylistUploadPending'), PlaylistUploadNotRequest: t('components.audio.PlayButton.button.PlaylistUploadNotRequest'), - PlaylistUploadTooltip: t('components.audio.PlayButton.button.PlaylistUploadTooltip') + PlaylistUploadTooltip: t('components.audio.PlayButton.button.PlaylistUploadTooltip'), + thirdPartyDownload: t('components.audio.PlayButton.button.thirdPartyDownload'), + thirdPartyDownloadTooltip: t('components.audio.PlayButton.button.thirdPartyDownload'), + thirdPartyDownloadTriggered: t('components.audio.PlayButton.button.thirdPartyDownloadTriggered') + })) const isOpen = ref(false) +const uploads = ref([]) + +const thirdPartyUpload = computed(() => { + if (uploads.value?.find(upload => 'third_party_provider' in upload)) { return false } else { return true } +}) + +const triggerThirdPartyHook = async () => { + const response = await axios.post( + 'uploads/trigger-download', + { track: props.track?.id } + ); + + if (response.status === 404) { + uploads.value.push({ third_party_upload: "triggered" }); + return null; + } else if (response.status === 200) { + console.log("uploads 400 (response.dat ", response.data) + uploads.value.push(response.data) + return response.data; + } +}; +triggerThirdPartyHook() const playlistLibraryFollowInfo = computed(() => { const playlist = props.playlist; @@ -246,7 +273,22 @@ const playlistLibraryFollowInfo = computed(() => { {{ t('components.audio.PlayButton.button.trackDetails') }} - + + {{ labels.thirdPartyDownload }} + + + {{ labels.thirdPartyDownloadTriggered }} +
props.url, () => { /> {{ t('components.federation.LibraryWidget.empty.noMatch') }} diff --git a/front/src/composables/audio/usePlayOptions.ts b/front/src/composables/audio/usePlayOptions.ts index 108571f33..c27bcc869 100644 --- a/front/src/composables/audio/usePlayOptions.ts +++ b/front/src/composables/audio/usePlayOptions.ts @@ -210,19 +210,24 @@ export default (props: PlayOptionsProps) => { if (!id) { throw new Error("Library id not found in response."); } - const fetchResponse = await axios.post('federation/fetches', - { object: id } + const fetchResponse = await axios.post( + 'federation/fetches', + { object_uri: id }, + { + headers: { + Accept: 'application/activity+json' + } + } ); const response = await axios.post( 'federation/follows/library', - { target: fetchResponse.data.object.uuid } + { target: fetchResponse.data.object.id.split('/').pop() } ); return response; }; - return { playable, filterableArtist, diff --git a/front/src/generated/types.ts b/front/src/generated/types.ts index 03e6c53af..a465a7133 100644 --- a/front/src/generated/types.ts +++ b/front/src/generated/types.ts @@ -2895,6 +2895,22 @@ export interface paths { patch: operations["partial_update_upload_bulk_update"]; trace?: never; }; + "/api/v1/uploads/trigger-download/": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["create_upload_trigger_download"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/users/{username}/": { parameters: { query?: never; @@ -6019,6 +6035,22 @@ export interface paths { patch: operations["partial_update_upload_bulk_update_2"]; trace?: never; }; + "/api/v2/uploads/trigger-download/": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["create_upload_trigger_download_2"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v2/users/{username}/": { parameters: { query?: never; @@ -8472,6 +8504,7 @@ export interface components { /** @default pending */ import_status: components["schemas"]["ImportStatusEnum"]; privacy_level?: components["schemas"]["LibraryPrivacyLevelEnum"]; + third_party_provider?: string | null; import_metadata?: components["schemas"]["ImportMetadataRequest"]; import_reference?: string; source?: string | null; @@ -8893,6 +8926,7 @@ export interface components { /** @default pending */ import_status: components["schemas"]["ImportStatusEnum"]; privacy_level?: components["schemas"]["LibraryPrivacyLevelEnum"]; + third_party_provider?: string | null; readonly import_details: unknown; import_metadata?: components["schemas"]["ImportMetadata"]; import_reference?: string; @@ -8907,6 +8941,7 @@ export interface components { /** @default pending */ import_status: components["schemas"]["ImportStatusEnum"]; privacy_level?: components["schemas"]["LibraryPrivacyLevelEnum"]; + third_party_provider?: string | null; import_metadata?: components["schemas"]["ImportMetadataRequest"]; import_reference?: string; source?: string | null; @@ -10590,6 +10625,7 @@ export interface operations { [name: string]: unknown; }; content: { + "application/activity+json": components["schemas"]["Fetch"]; "application/json": components["schemas"]["Fetch"]; }; }; @@ -10612,6 +10648,7 @@ export interface operations { [name: string]: unknown; }; content: { + "application/activity+json": components["schemas"]["Fetch"]; "application/json": components["schemas"]["Fetch"]; }; }; @@ -15756,6 +15793,32 @@ export interface operations { }; }; }; + create_upload_trigger_download: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UploadForOwnerRequest"]; + "application/x-www-form-urlencoded": components["schemas"]["UploadForOwnerRequest"]; + "multipart/form-data": components["schemas"]["UploadForOwnerRequest"]; + "application/activity+json": components["schemas"]["UploadForOwnerRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UploadForOwner"]; + }; + }; + }; + }; update_user: { parameters: { query?: never; @@ -17537,6 +17600,7 @@ export interface operations { [name: string]: unknown; }; content: { + "application/activity+json": components["schemas"]["Fetch"]; "application/json": components["schemas"]["Fetch"]; }; }; @@ -17559,6 +17623,7 @@ export interface operations { [name: string]: unknown; }; content: { + "application/activity+json": components["schemas"]["Fetch"]; "application/json": components["schemas"]["Fetch"]; }; }; @@ -22913,6 +22978,32 @@ export interface operations { }; }; }; + create_upload_trigger_download_2: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UploadForOwnerRequest"]; + "application/x-www-form-urlencoded": components["schemas"]["UploadForOwnerRequest"]; + "multipart/form-data": components["schemas"]["UploadForOwnerRequest"]; + "application/activity+json": components["schemas"]["UploadForOwnerRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UploadForOwner"]; + }; + }; + }; + }; update_user_3: { parameters: { query?: never; diff --git a/front/src/locales/en_US.json b/front/src/locales/en_US.json index 43094e895..20674d59b 100644 --- a/front/src/locales/en_US.json +++ b/front/src/locales/en_US.json @@ -492,7 +492,10 @@ "playTracks": "Play tracks", "report": "Report…", "startRadio": "Play similar songs", - "trackDetails": "Track details" + "trackDetails": "Track details", + "thirdPartyDownload": "Try to download from third-party service", + "thirdPartyDownloadTooltip": "This relies on installed plugins and may not work on all pods.", + "thirdPartyDownloadTriggered": "Download triggered, come back later" }, "title": { "more": "More…",