Merge branch '2452-fetch-third-party-metadata' into 'develop'
draft : import tracks and albums from musicbrainz into Funkwhale Closes #2452 See merge request funkwhale/funkwhale!2926
This commit is contained in:
commit
7c55e1cdbd
|
@ -2462,6 +2462,9 @@ paths:
|
||||||
responses:
|
responses:
|
||||||
'201':
|
'201':
|
||||||
content:
|
content:
|
||||||
|
application/activity+json:
|
||||||
|
schema:
|
||||||
|
$ref: '#/components/schemas/Fetch'
|
||||||
application/json:
|
application/json:
|
||||||
schema:
|
schema:
|
||||||
$ref: '#/components/schemas/Fetch'
|
$ref: '#/components/schemas/Fetch'
|
||||||
|
@ -2484,6 +2487,9 @@ paths:
|
||||||
responses:
|
responses:
|
||||||
'200':
|
'200':
|
||||||
content:
|
content:
|
||||||
|
application/activity+json:
|
||||||
|
schema:
|
||||||
|
$ref: '#/components/schemas/Fetch'
|
||||||
application/json:
|
application/json:
|
||||||
schema:
|
schema:
|
||||||
$ref: '#/components/schemas/Fetch'
|
$ref: '#/components/schemas/Fetch'
|
||||||
|
@ -11972,6 +11978,9 @@ paths:
|
||||||
responses:
|
responses:
|
||||||
'201':
|
'201':
|
||||||
content:
|
content:
|
||||||
|
application/activity+json:
|
||||||
|
schema:
|
||||||
|
$ref: '#/components/schemas/Fetch'
|
||||||
application/json:
|
application/json:
|
||||||
schema:
|
schema:
|
||||||
$ref: '#/components/schemas/Fetch'
|
$ref: '#/components/schemas/Fetch'
|
||||||
|
@ -11994,6 +12003,9 @@ paths:
|
||||||
responses:
|
responses:
|
||||||
'200':
|
'200':
|
||||||
content:
|
content:
|
||||||
|
application/activity+json:
|
||||||
|
schema:
|
||||||
|
$ref: '#/components/schemas/Fetch'
|
||||||
application/json:
|
application/json:
|
||||||
schema:
|
schema:
|
||||||
$ref: '#/components/schemas/Fetch'
|
$ref: '#/components/schemas/Fetch'
|
||||||
|
|
|
@ -13,7 +13,9 @@ from funkwhale_api.audio import models as audio_models
|
||||||
from funkwhale_api.audio import serializers as audio_serializers
|
from funkwhale_api.audio import serializers as audio_serializers
|
||||||
from funkwhale_api.common import serializers as common_serializers
|
from funkwhale_api.common import serializers as common_serializers
|
||||||
from funkwhale_api.music import models as music_models
|
from funkwhale_api.music import models as music_models
|
||||||
|
from funkwhale_api.music import serializers as music_serializers
|
||||||
from funkwhale_api.playlists import models as playlists_models
|
from funkwhale_api.playlists import models as playlists_models
|
||||||
|
from funkwhale_api.playlists import serializers as playlist_serializers
|
||||||
from funkwhale_api.users import serializers as users_serializers
|
from funkwhale_api.users import serializers as users_serializers
|
||||||
|
|
||||||
from . import filters, models
|
from . import filters, models
|
||||||
|
@ -197,10 +199,19 @@ OBJECT_SERIALIZER_MAPPING = {
|
||||||
music_models.Artist: federation_serializers.ArtistSerializer,
|
music_models.Artist: federation_serializers.ArtistSerializer,
|
||||||
music_models.Album: federation_serializers.AlbumSerializer,
|
music_models.Album: federation_serializers.AlbumSerializer,
|
||||||
music_models.Track: federation_serializers.TrackSerializer,
|
music_models.Track: federation_serializers.TrackSerializer,
|
||||||
|
music_models.Library: federation_serializers.LibrarySerializer,
|
||||||
models.Actor: federation_serializers.APIActorSerializer,
|
models.Actor: federation_serializers.APIActorSerializer,
|
||||||
audio_models.Channel: audio_serializers.ChannelSerializer,
|
audio_models.Channel: audio_serializers.ChannelSerializer,
|
||||||
playlists_models.Playlist: federation_serializers.PlaylistSerializer,
|
playlists_models.Playlist: federation_serializers.PlaylistSerializer,
|
||||||
}
|
}
|
||||||
|
OBJECT_MUSIC_SERIALIZER_MAPPING = {
|
||||||
|
music_models.Artist: music_serializers.ArtistSerializer,
|
||||||
|
music_models.Album: music_serializers.AlbumSerializer,
|
||||||
|
music_models.Track: music_serializers.TrackSerializer,
|
||||||
|
models.Actor: federation_serializers.APIActorSerializer,
|
||||||
|
audio_models.Channel: audio_serializers.ChannelSerializer,
|
||||||
|
playlists_models.Playlist: playlist_serializers.PlaylistSerializer,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def convert_url_to_webfinger(url):
|
def convert_url_to_webfinger(url):
|
||||||
|
@ -283,6 +294,9 @@ class FetchSerializer(serializers.ModelSerializer):
|
||||||
return value
|
return value
|
||||||
return f"webfinger://{value}"
|
return f"webfinger://{value}"
|
||||||
|
|
||||||
|
# to do : this is incomplete, schema conflict because
|
||||||
|
# federation serializers have the same name than musi serializer -> upgrade fed serializers to new names
|
||||||
|
# and add the new object here
|
||||||
@extend_schema_field(
|
@extend_schema_field(
|
||||||
{
|
{
|
||||||
"oneOf": [
|
"oneOf": [
|
||||||
|
@ -300,7 +314,12 @@ class FetchSerializer(serializers.ModelSerializer):
|
||||||
if obj is None:
|
if obj is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
serializer_class = OBJECT_SERIALIZER_MAPPING.get(type(obj))
|
media_type = self.context.get("media_type")
|
||||||
|
|
||||||
|
if media_type == "application/activity+json":
|
||||||
|
serializer_class = OBJECT_SERIALIZER_MAPPING.get(type(obj))
|
||||||
|
else:
|
||||||
|
serializer_class = OBJECT_MUSIC_SERIALIZER_MAPPING.get(type(obj))
|
||||||
if serializer_class:
|
if serializer_class:
|
||||||
return serializer_class(obj).data
|
return serializer_class(obj).data
|
||||||
return None
|
return None
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import requests.exceptions
|
import requests.exceptions
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.db import transaction
|
from django.db import transaction
|
||||||
|
@ -5,10 +7,13 @@ from django.db.models import Count, Q
|
||||||
from drf_spectacular.utils import extend_schema, extend_schema_view
|
from drf_spectacular.utils import extend_schema, extend_schema_view
|
||||||
from rest_framework import decorators, mixins, permissions, response, viewsets
|
from rest_framework import decorators, mixins, permissions, response, viewsets
|
||||||
from rest_framework.exceptions import NotFound as RestNotFound
|
from rest_framework.exceptions import NotFound as RestNotFound
|
||||||
|
from rest_framework.negotiation import DefaultContentNegotiation
|
||||||
|
from rest_framework.renderers import JSONRenderer
|
||||||
|
|
||||||
from funkwhale_api.common import preferences
|
from funkwhale_api.common import preferences
|
||||||
from funkwhale_api.common import utils as common_utils
|
from funkwhale_api.common import utils as common_utils
|
||||||
from funkwhale_api.common.permissions import ConditionalAuthentication
|
from funkwhale_api.common.permissions import ConditionalAuthentication
|
||||||
|
from funkwhale_api.common.renderers import ActivityStreamRenderer
|
||||||
from funkwhale_api.music import models as music_models
|
from funkwhale_api.music import models as music_models
|
||||||
from funkwhale_api.music import serializers as music_serializers
|
from funkwhale_api.music import serializers as music_serializers
|
||||||
from funkwhale_api.music import views as music_views
|
from funkwhale_api.music import views as music_views
|
||||||
|
@ -243,20 +248,41 @@ class FetchViewSet(
|
||||||
serializer_class = api_serializers.FetchSerializer
|
serializer_class = api_serializers.FetchSerializer
|
||||||
permission_classes = [permissions.IsAuthenticated]
|
permission_classes = [permissions.IsAuthenticated]
|
||||||
throttling_scopes = {"create": {"authenticated": "fetch"}}
|
throttling_scopes = {"create": {"authenticated": "fetch"}}
|
||||||
|
renderer_classes = [ActivityStreamRenderer, JSONRenderer]
|
||||||
|
|
||||||
def get_queryset(self):
|
def get_queryset(self):
|
||||||
return super().get_queryset().filter(actor=self.request.user.actor)
|
return super().get_queryset().filter(actor=self.request.user.actor)
|
||||||
|
|
||||||
|
def get_serializer_context(self):
|
||||||
|
context = super().get_serializer_context()
|
||||||
|
|
||||||
|
negotiator = DefaultContentNegotiation()
|
||||||
|
try:
|
||||||
|
renderer, media_type = negotiator.select_renderer(
|
||||||
|
self.request, self.get_renderers()
|
||||||
|
)
|
||||||
|
context["media_type"] = media_type
|
||||||
|
except Exception:
|
||||||
|
context["media_type"] = None
|
||||||
|
return context
|
||||||
|
|
||||||
def perform_create(self, serializer):
|
def perform_create(self, serializer):
|
||||||
fetch = serializer.save(actor=self.request.user.actor)
|
fetch = serializer.save(actor=self.request.user.actor)
|
||||||
if fetch.status == "finished":
|
if fetch.status == "finished":
|
||||||
# a duplicate was returned, no need to fetch again
|
# a duplicate was returned, no need to fetch again
|
||||||
return
|
return
|
||||||
if settings.FEDERATION_SYNCHRONOUS_FETCH:
|
|
||||||
tasks.fetch(fetch_id=fetch.pk)
|
parsed_url = urlparse(fetch.url)
|
||||||
|
domain = parsed_url.netloc
|
||||||
|
if domain in fetch.supported_services:
|
||||||
|
tasks.third_party_fetch(fetch_id=fetch.pk)
|
||||||
fetch.refresh_from_db()
|
fetch.refresh_from_db()
|
||||||
else:
|
else:
|
||||||
common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk)
|
if settings.FEDERATION_SYNCHRONOUS_FETCH:
|
||||||
|
tasks.fetch(fetch_id=fetch.pk)
|
||||||
|
fetch.refresh_from_db()
|
||||||
|
else:
|
||||||
|
common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk)
|
||||||
|
|
||||||
|
|
||||||
class DomainViewSet(
|
class DomainViewSet(
|
||||||
|
|
|
@ -18,6 +18,7 @@ from funkwhale_api.common import session
|
||||||
from funkwhale_api.common import utils as common_utils
|
from funkwhale_api.common import utils as common_utils
|
||||||
from funkwhale_api.common import validators as common_validators
|
from funkwhale_api.common import validators as common_validators
|
||||||
from funkwhale_api.music import utils as music_utils
|
from funkwhale_api.music import utils as music_utils
|
||||||
|
from funkwhale_api.musicbrainz import serializers as musicbrainz_serializers
|
||||||
|
|
||||||
from . import utils as federation_utils
|
from . import utils as federation_utils
|
||||||
|
|
||||||
|
@ -411,8 +412,15 @@ class Fetch(models.Model):
|
||||||
contexts.AS.Organization: [serializers.ActorSerializer],
|
contexts.AS.Organization: [serializers.ActorSerializer],
|
||||||
contexts.AS.Service: [serializers.ActorSerializer],
|
contexts.AS.Service: [serializers.ActorSerializer],
|
||||||
contexts.AS.Application: [serializers.ActorSerializer],
|
contexts.AS.Application: [serializers.ActorSerializer],
|
||||||
|
# for mb the key must be the api namespace
|
||||||
|
"recordings": [musicbrainz_serializers.RecordingSerializer],
|
||||||
|
"releases": [musicbrainz_serializers.ReleaseSerializer],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def supported_services(self):
|
||||||
|
return ["musicbrainz.org"]
|
||||||
|
|
||||||
|
|
||||||
class InboxItem(models.Model):
|
class InboxItem(models.Model):
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -1066,6 +1066,7 @@ class LibrarySerializer(PaginatedCollectionSerializer):
|
||||||
privacy = {"": "me", "./": "me", None: "me", contexts.AS.Public: "everyone"}
|
privacy = {"": "me", "./": "me", None: "me", contexts.AS.Public: "everyone"}
|
||||||
library, created = music_models.Library.objects.update_or_create(
|
library, created = music_models.Library.objects.update_or_create(
|
||||||
fid=validated_data["id"],
|
fid=validated_data["id"],
|
||||||
|
uuid=validated_data["id"].rstrip("/").split("/")[-1],
|
||||||
actor=actor,
|
actor=actor,
|
||||||
defaults={
|
defaults={
|
||||||
"uploads_count": validated_data["totalItems"],
|
"uploads_count": validated_data["totalItems"],
|
||||||
|
@ -1449,7 +1450,7 @@ class AlbumSerializer(MusicEntitySerializer):
|
||||||
acs.append(
|
acs.append(
|
||||||
utils.retrieve_ap_object(
|
utils.retrieve_ap_object(
|
||||||
ac["id"],
|
ac["id"],
|
||||||
actor=self.context.get("fetch_actor"),
|
actor=self.context.get("_actor"),
|
||||||
queryset=music_models.ArtistCredit,
|
queryset=music_models.ArtistCredit,
|
||||||
serializer_class=ArtistCreditSerializer,
|
serializer_class=ArtistCreditSerializer,
|
||||||
)
|
)
|
||||||
|
|
|
@ -2,9 +2,12 @@ import datetime
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import uuid
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
from django.contrib.contenttypes.models import ContentType
|
||||||
from django.core.cache import cache
|
from django.core.cache import cache
|
||||||
from django.db import transaction
|
from django.db import transaction
|
||||||
from django.db.models import F, Q
|
from django.db.models import F, Q
|
||||||
|
@ -13,6 +16,8 @@ from django.utils import timezone
|
||||||
from dynamic_preferences.registries import global_preferences_registry
|
from dynamic_preferences.registries import global_preferences_registry
|
||||||
from requests.exceptions import RequestException
|
from requests.exceptions import RequestException
|
||||||
|
|
||||||
|
from config import plugins
|
||||||
|
from funkwhale_api import musicbrainz
|
||||||
from funkwhale_api.audio import models as audio_models
|
from funkwhale_api.audio import models as audio_models
|
||||||
from funkwhale_api.common import models as common_models
|
from funkwhale_api.common import models as common_models
|
||||||
from funkwhale_api.common import preferences, session
|
from funkwhale_api.common import preferences, session
|
||||||
|
@ -449,7 +454,6 @@ def fetch(fetch_obj):
|
||||||
max_pages=settings.FEDERATION_COLLECTION_MAX_PAGES - 1,
|
max_pages=settings.FEDERATION_COLLECTION_MAX_PAGES - 1,
|
||||||
is_page=True,
|
is_page=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
fetch_obj.object = obj
|
fetch_obj.object = obj
|
||||||
fetch_obj.status = "finished"
|
fetch_obj.status = "finished"
|
||||||
fetch_obj.fetch_date = timezone.now()
|
fetch_obj.fetch_date = timezone.now()
|
||||||
|
@ -458,6 +462,167 @@ def fetch(fetch_obj):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def trigger_third_party_upload_hook(fetch):
|
||||||
|
if fetch.status == "finished" and fetch.object:
|
||||||
|
if fetch.object_content_type == ContentType.objects.get_for_model(
|
||||||
|
music_models.Track
|
||||||
|
):
|
||||||
|
if not music_models.Track.objects.filter(pk=fetch.object.pk).playable_by(
|
||||||
|
fetch.actor
|
||||||
|
):
|
||||||
|
plugins.trigger_hook(
|
||||||
|
plugins.TRIGGER_THIRD_PARTY_UPLOAD,
|
||||||
|
track=fetch.object,
|
||||||
|
)
|
||||||
|
if fetch.object_content_type == ContentType.objects.get_for_model(
|
||||||
|
music_models.Album
|
||||||
|
):
|
||||||
|
for track in fetch.object.tracks.all():
|
||||||
|
if not music_models.Track.objects.filter(
|
||||||
|
pk=fetch.object.pk
|
||||||
|
).playable_by(fetch.actor):
|
||||||
|
plugins.trigger_hook(
|
||||||
|
plugins.TRIGGER_THIRD_PARTY_UPLOAD,
|
||||||
|
track=track,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def musicbrainz_type_handler(fetch):
|
||||||
|
url = fetch.url
|
||||||
|
path_parts = urlparse(url).path.strip("/").split("/")
|
||||||
|
type_ = path_parts[0] + "s"
|
||||||
|
mbid = path_parts[1]
|
||||||
|
try:
|
||||||
|
uuid.UUID(mbid)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError(f"could no get mbid from url {url}")
|
||||||
|
return type_, mbid
|
||||||
|
|
||||||
|
|
||||||
|
def musicbrainz_metadata_handler(type_, id):
|
||||||
|
def replace_hyphens_in_keys(obj):
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
return {
|
||||||
|
k.replace("-", "_"): replace_hyphens_in_keys(v) for k, v in obj.items()
|
||||||
|
}
|
||||||
|
elif isinstance(obj, list):
|
||||||
|
return [replace_hyphens_in_keys(item) for item in obj]
|
||||||
|
else:
|
||||||
|
return obj
|
||||||
|
|
||||||
|
if type_ == "recordings":
|
||||||
|
includes = ["tags", "artists", "releases"]
|
||||||
|
elif type_ == "releases":
|
||||||
|
includes = ["tags", "artists", "recordings"]
|
||||||
|
|
||||||
|
result = replace_hyphens_in_keys(
|
||||||
|
getattr(musicbrainz.api, type_).get(id=id, includes=includes)
|
||||||
|
)
|
||||||
|
|
||||||
|
existing = (
|
||||||
|
music_models.Track.objects.filter(mbid=id).first()
|
||||||
|
if music_models.Track.objects.filter(mbid=id).exists()
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
return result, existing
|
||||||
|
|
||||||
|
|
||||||
|
type_and_id_from_third_party = {"musicbrainz.org": musicbrainz_type_handler}
|
||||||
|
metadata_from_third_party_ = {"musicbrainz.org": musicbrainz_metadata_handler}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.app.task(name="third_party_fetch")
|
||||||
|
@transaction.atomic
|
||||||
|
@celery.require_instance(
|
||||||
|
models.Fetch.objects.filter(status="pending").select_related("actor"),
|
||||||
|
"fetch_obj",
|
||||||
|
"fetch_id",
|
||||||
|
)
|
||||||
|
def third_party_fetch(fetch_obj):
|
||||||
|
def error(code, **kwargs):
|
||||||
|
fetch_obj.status = "errored"
|
||||||
|
fetch_obj.fetch_date = timezone.now()
|
||||||
|
fetch_obj.detail = {"error_code": code}
|
||||||
|
fetch_obj.detail.update(kwargs)
|
||||||
|
fetch_obj.save(update_fields=["fetch_date", "status", "detail"])
|
||||||
|
|
||||||
|
def check_url(url):
|
||||||
|
if not url.startswith("webfinger://"):
|
||||||
|
payload, updated = mrf.inbox.apply({"id": url})
|
||||||
|
if not payload:
|
||||||
|
return error("blocked", message="Blocked by MRF")
|
||||||
|
|
||||||
|
parsed_url = urlparse(url)
|
||||||
|
service = parsed_url.netloc
|
||||||
|
if service not in fetch_obj.supported_services:
|
||||||
|
return error("invalid_url", message=f"Unsupported domain {service}")
|
||||||
|
return service
|
||||||
|
|
||||||
|
url = fetch_obj.url
|
||||||
|
actor = fetch_obj.actor
|
||||||
|
service = check_url(url)
|
||||||
|
|
||||||
|
try:
|
||||||
|
type_, id = type_and_id_from_third_party[service](fetch_obj)
|
||||||
|
logger.debug("Parsed URL %s into type %s and id %s", url, type_, id)
|
||||||
|
except ValueError as e:
|
||||||
|
return error("url_parse_error", message=str(e))
|
||||||
|
|
||||||
|
try:
|
||||||
|
result, existing = metadata_from_third_party_[service](type_, id)
|
||||||
|
logger.debug(
|
||||||
|
f"Remote answered with {result} and we found {existing} in database"
|
||||||
|
)
|
||||||
|
except requests.exceptions.HTTPError as e:
|
||||||
|
return error(
|
||||||
|
"http",
|
||||||
|
status_code=e.response.status_code if e.response else None,
|
||||||
|
message=e.response.text,
|
||||||
|
)
|
||||||
|
except requests.exceptions.Timeout:
|
||||||
|
return error("timeout")
|
||||||
|
except requests.exceptions.ConnectionError as e:
|
||||||
|
return error("connection", message=str(e))
|
||||||
|
except requests.RequestException as e:
|
||||||
|
return error("request", message=str(e))
|
||||||
|
except Exception as e:
|
||||||
|
return error("unhandled", message=str(e))
|
||||||
|
|
||||||
|
try:
|
||||||
|
serializer_classes = fetch_obj.serializers.get(type_)
|
||||||
|
except (KeyError, AttributeError):
|
||||||
|
fetch_obj.status = "skipped"
|
||||||
|
fetch_obj.fetch_date = timezone.now()
|
||||||
|
fetch_obj.detail = {"reason": "unhandled_type", "type": type_}
|
||||||
|
return fetch_obj.save(update_fields=["fetch_date", "status", "detail"])
|
||||||
|
|
||||||
|
serializer = None
|
||||||
|
for serializer_class in serializer_classes:
|
||||||
|
serializer = serializer_class(
|
||||||
|
existing, data=result, context={"fetch_actor": actor}
|
||||||
|
)
|
||||||
|
if not serializer.is_valid():
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
if serializer.errors:
|
||||||
|
return error("validation", validation_errors=serializer.errors)
|
||||||
|
try:
|
||||||
|
obj = serializer.save()
|
||||||
|
except Exception as e:
|
||||||
|
error("save", message=str(e))
|
||||||
|
raise
|
||||||
|
|
||||||
|
fetch_obj.object = obj
|
||||||
|
fetch_obj.status = "finished"
|
||||||
|
fetch_obj.fetch_date = timezone.now()
|
||||||
|
trigger_third_party_upload_hook(fetch_obj)
|
||||||
|
fetch_obj.save(
|
||||||
|
update_fields=["fetch_date", "status", "object_id", "object_content_type"]
|
||||||
|
)
|
||||||
|
return fetch_obj
|
||||||
|
|
||||||
|
|
||||||
class PreserveSomeDataCollector(Collector):
|
class PreserveSomeDataCollector(Collector):
|
||||||
"""
|
"""
|
||||||
We need to delete everything related to an actor. Well… Almost everything.
|
We need to delete everything related to an actor. Well… Almost everything.
|
||||||
|
|
|
@ -0,0 +1,251 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from rest_framework import serializers
|
||||||
|
|
||||||
|
from funkwhale_api import musicbrainz
|
||||||
|
from funkwhale_api.tags import models as tags_models
|
||||||
|
|
||||||
|
from . import client
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class ArtistSerializer(serializers.Serializer):
|
||||||
|
"""
|
||||||
|
Serializer for Musicbrainz artist data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
id = serializers.CharField()
|
||||||
|
name = serializers.CharField()
|
||||||
|
|
||||||
|
def create(self, validated_data):
|
||||||
|
from funkwhale_api.music.models import Artist
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"name": validated_data["name"],
|
||||||
|
"mbid": validated_data["id"],
|
||||||
|
}
|
||||||
|
artist, created = Artist.objects.get_or_create(**data)
|
||||||
|
return artist
|
||||||
|
|
||||||
|
|
||||||
|
class ArtistCreditSerializer(serializers.Serializer):
|
||||||
|
"""
|
||||||
|
Serializer for Musicbrainz artist data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
name = serializers.CharField()
|
||||||
|
joinphrase = serializers.CharField(allow_blank=True)
|
||||||
|
artist = ArtistSerializer()
|
||||||
|
|
||||||
|
def create(self, validated_data):
|
||||||
|
from funkwhale_api.music.models import ArtistCredit
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"credit": validated_data["name"],
|
||||||
|
"joinphrase": validated_data.get("joinphrase", ""),
|
||||||
|
"artist": ArtistSerializer().create(validated_data["artist"]),
|
||||||
|
}
|
||||||
|
artist_credit, created = ArtistCredit.objects.get_or_create(**data)
|
||||||
|
return artist_credit
|
||||||
|
|
||||||
|
|
||||||
|
class ReleaseForRecordingSerializer(serializers.Serializer):
|
||||||
|
"""
|
||||||
|
Serializer for Musicbrainz release data when returned in a recording object.
|
||||||
|
"""
|
||||||
|
|
||||||
|
id = serializers.CharField()
|
||||||
|
title = serializers.CharField()
|
||||||
|
artist_credit = ArtistCreditSerializer(many=True)
|
||||||
|
tags = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||||
|
date = serializers.DateField(input_formats=["%Y", "%Y/%m/%d", "%Y-%m-%d"])
|
||||||
|
|
||||||
|
def create(self, validated_data):
|
||||||
|
from funkwhale_api.music.models import Album
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"title": validated_data["title"],
|
||||||
|
"mbid": validated_data["id"],
|
||||||
|
"release_date": validated_data.get("date", None),
|
||||||
|
}
|
||||||
|
album, created = Album.objects.get_or_create(**data)
|
||||||
|
artist_credit = ArtistCreditSerializer(many=True).create(
|
||||||
|
validated_data["artist_credit"]
|
||||||
|
)
|
||||||
|
album.artist_credit.set(artist_credit)
|
||||||
|
album.save()
|
||||||
|
tags_models.add_tags(album, *validated_data.get("tags", []))
|
||||||
|
return album
|
||||||
|
|
||||||
|
def update(self, instance, validated_data):
|
||||||
|
instance.title = validated_data["title"]
|
||||||
|
instance.release_date = validated_data.get("date")
|
||||||
|
instance.save()
|
||||||
|
tags_models.add_tags(instance, *validated_data.get("tags", []))
|
||||||
|
|
||||||
|
return instance
|
||||||
|
|
||||||
|
|
||||||
|
class RecordingSerializer(serializers.Serializer):
|
||||||
|
"""
|
||||||
|
Serializer for Musicbrainz track data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
id = serializers.CharField()
|
||||||
|
title = serializers.CharField()
|
||||||
|
artist_credit = ArtistCreditSerializer(many=True)
|
||||||
|
releases = ReleaseForRecordingSerializer(many=True, required=False)
|
||||||
|
tags = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||||
|
position = serializers.IntegerField(required=False, allow_null=True)
|
||||||
|
|
||||||
|
def create(self, validated_data):
|
||||||
|
from funkwhale_api.music.models import Track
|
||||||
|
|
||||||
|
data = {"mbid": validated_data["id"]}
|
||||||
|
defaults = {
|
||||||
|
"title": validated_data["title"],
|
||||||
|
"mbid": validated_data["id"],
|
||||||
|
# In mb a recording can have various releases, we take the fist one
|
||||||
|
"album": (
|
||||||
|
ReleaseForRecordingSerializer(many=True).create(
|
||||||
|
validated_data["releases"]
|
||||||
|
)[0]
|
||||||
|
if validated_data.get("releases")
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
# this will be none if the recording is not fetched from the release endpoint
|
||||||
|
"position": validated_data.get("position", None),
|
||||||
|
}
|
||||||
|
|
||||||
|
if defaults["album"] is not None and defaults["position"] is None:
|
||||||
|
result = client.api.releases.get(
|
||||||
|
id=validated_data["releases"][0]["id"],
|
||||||
|
includes=["tags", "artists", "recordings"],
|
||||||
|
)
|
||||||
|
tracks = result["media"][0]["tracks"]
|
||||||
|
defaults["position"] = next(
|
||||||
|
(o for o in tracks if o["recording"]["id"] == data["mbid"]), {}
|
||||||
|
).get("position", None)
|
||||||
|
|
||||||
|
track, created = Track.objects.get_or_create(**data, defaults=defaults)
|
||||||
|
artist_credit = ArtistCreditSerializer(many=True).create(
|
||||||
|
validated_data["artist_credit"]
|
||||||
|
)
|
||||||
|
track.artist_credit.set(artist_credit)
|
||||||
|
track.save()
|
||||||
|
|
||||||
|
tags_models.add_tags(track, *validated_data.get("tags", []))
|
||||||
|
|
||||||
|
return track
|
||||||
|
|
||||||
|
def update(self, instance, validated_data):
|
||||||
|
instance.title = validated_data["title"]
|
||||||
|
instance.save()
|
||||||
|
tags_models.add_tags(instance, *validated_data.get("tags", []))
|
||||||
|
|
||||||
|
return instance
|
||||||
|
|
||||||
|
|
||||||
|
class RecordingForReleaseSerializer(serializers.Serializer):
|
||||||
|
id = serializers.CharField()
|
||||||
|
title = serializers.CharField()
|
||||||
|
# not in Musicbrainz recording object, but used to store the position of the track in the album
|
||||||
|
position = serializers.IntegerField(required=False, allow_null=True)
|
||||||
|
|
||||||
|
def create(self, validated_data):
|
||||||
|
def replace_hyphens_in_keys(obj):
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
return {
|
||||||
|
k.replace("-", "_"): replace_hyphens_in_keys(v)
|
||||||
|
for k, v in obj.items()
|
||||||
|
}
|
||||||
|
elif isinstance(obj, list):
|
||||||
|
return [replace_hyphens_in_keys(item) for item in obj]
|
||||||
|
else:
|
||||||
|
return obj
|
||||||
|
|
||||||
|
recordings_data = musicbrainz.api.recordings.get(
|
||||||
|
id=validated_data["id"], includes=["tags", "artists"]
|
||||||
|
)
|
||||||
|
recordings_data = replace_hyphens_in_keys(recordings_data)
|
||||||
|
recordings_data["position"] = validated_data.get("position", None)
|
||||||
|
serializer = RecordingSerializer(data=recordings_data)
|
||||||
|
serializer.is_valid(raise_exception=True)
|
||||||
|
track = serializer.save()
|
||||||
|
track.album = validated_data["album"]
|
||||||
|
track.save()
|
||||||
|
return track
|
||||||
|
|
||||||
|
def update(self, instance, validated_data):
|
||||||
|
instance.title = validated_data["title"]
|
||||||
|
tags_models.add_tags(instance, *validated_data.get("tags", []))
|
||||||
|
instance.album = validated_data["album"]
|
||||||
|
instance.save()
|
||||||
|
return instance
|
||||||
|
|
||||||
|
|
||||||
|
class TrackSerializer(serializers.Serializer):
|
||||||
|
recording = RecordingForReleaseSerializer()
|
||||||
|
position = serializers.IntegerField()
|
||||||
|
|
||||||
|
|
||||||
|
class MediaSerializer(serializers.Serializer):
|
||||||
|
tracks = TrackSerializer(many=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ReleaseSerializer(serializers.Serializer):
|
||||||
|
"""
|
||||||
|
Serializer for Musicbrainz release data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
id = serializers.CharField()
|
||||||
|
title = serializers.CharField()
|
||||||
|
artist_credit = ArtistCreditSerializer(many=True)
|
||||||
|
tags = serializers.ListField(child=serializers.CharField(), allow_empty=True)
|
||||||
|
date = serializers.DateField(input_formats=["%Y", "%Y/%m/%d", "%Y-%m-%d"])
|
||||||
|
media = serializers.ListField(child=MediaSerializer())
|
||||||
|
|
||||||
|
def create(self, validated_data):
|
||||||
|
from funkwhale_api.music.models import Album
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"title": validated_data["title"],
|
||||||
|
"mbid": validated_data["id"],
|
||||||
|
"release_date": validated_data.get("date"),
|
||||||
|
}
|
||||||
|
album, created = Album.objects.get_or_create(**data)
|
||||||
|
artist_credit = ArtistCreditSerializer(many=True).create(
|
||||||
|
validated_data["artist_credit"]
|
||||||
|
)
|
||||||
|
album.artist_credit.set(artist_credit)
|
||||||
|
album.save()
|
||||||
|
tags_models.add_tags(album, *validated_data.get("tags", []))
|
||||||
|
|
||||||
|
# an album can have various media/physical representation, we take the first one
|
||||||
|
tracks = [t for t in validated_data["media"][0]["tracks"]]
|
||||||
|
recordings = []
|
||||||
|
for t in tracks:
|
||||||
|
t["recording"]["position"] = t["position"]
|
||||||
|
recordings.append(t["recording"])
|
||||||
|
|
||||||
|
for r in recordings:
|
||||||
|
r["album"] = album
|
||||||
|
RecordingForReleaseSerializer().create(r)
|
||||||
|
|
||||||
|
return album
|
||||||
|
|
||||||
|
# this will never be used while FetchViewSet and third_party_fetch filter out finished fetch
|
||||||
|
# Would be nice to have a way to manually update releases from Musicbrainz
|
||||||
|
def update(self, instance, validated_data):
|
||||||
|
logger.info(f"Updating release {instance} with data: {validated_data}")
|
||||||
|
instance.title = validated_data["title"]
|
||||||
|
instance.release_date = validated_data.get("date")
|
||||||
|
instance.save()
|
||||||
|
tags_models.add_tags(instance, *validated_data.get("tags", []))
|
||||||
|
|
||||||
|
recordings = [t["recording"] for t in validated_data["media"][0]["tracks"]]
|
||||||
|
for r in recordings:
|
||||||
|
r["album"] = instance
|
||||||
|
RecordingForReleaseSerializer().update(r)
|
||||||
|
return instance
|
|
@ -173,7 +173,12 @@ def test_fetch_serializer_with_object(
|
||||||
"actor": serializers.APIActorSerializer(fetch.actor).data,
|
"actor": serializers.APIActorSerializer(fetch.actor).data,
|
||||||
}
|
}
|
||||||
|
|
||||||
assert api_serializers.FetchSerializer(fetch).data == expected
|
assert (
|
||||||
|
api_serializers.FetchSerializer(
|
||||||
|
fetch, context={"media_type": "application/activity+json"}
|
||||||
|
).data
|
||||||
|
== expected
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_fetch_serializer_unhandled_obj(factories, to_api_date):
|
def test_fetch_serializer_unhandled_obj(factories, to_api_date):
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
Import tracks and albums from Musicbrainz into Funkwhale though the search bar (#2926)
|
|
@ -0,0 +1,22 @@
|
||||||
|
# Collections
|
||||||
|
|
||||||
|
## The issue
|
||||||
|
|
||||||
|
Has a user I want to be able to get metadata from third party services (to add tracks to my favorites or to a playlist)
|
||||||
|
Has a user I want to clone a third party playslit into Funkwhale
|
||||||
|
|
||||||
|
## Solution
|
||||||
|
|
||||||
|
paste the audio object (track, album, playlist) url into the search bar. Funkwhale will get the mmetadata and create the objects in db. First implementation with Musicbrainz :
|
||||||
|
|
||||||
|
- track : https://musicbrainz.org/release/{mbid}
|
||||||
|
- release : https://musicbrainz.org/recording/{mbid}
|
||||||
|
|
||||||
|
## Implementation
|
||||||
|
|
||||||
|
third_party_fetch in federation.tasks
|
||||||
|
musicbrainz.serializers
|
||||||
|
|
||||||
|
## Call for developers
|
||||||
|
|
||||||
|
This is time consuming. The main logic is implemented. Now everybody can add support for any third party service.
|
|
@ -73,7 +73,6 @@ watch(() => props.url, () => {
|
||||||
/>
|
/>
|
||||||
<Alert
|
<Alert
|
||||||
v-if="!isLoading && libraries.length === 0"
|
v-if="!isLoading && libraries.length === 0"
|
||||||
blue
|
|
||||||
style="grid-column: 1 / -1;"
|
style="grid-column: 1 / -1;"
|
||||||
>
|
>
|
||||||
{{ t('components.federation.LibraryWidget.empty.noMatch') }}
|
{{ t('components.federation.LibraryWidget.empty.noMatch') }}
|
||||||
|
|
|
@ -210,13 +210,19 @@ export default (props: PlayOptionsProps) => {
|
||||||
if (!id) {
|
if (!id) {
|
||||||
throw new Error("Library id not found in response.");
|
throw new Error("Library id not found in response.");
|
||||||
}
|
}
|
||||||
const fetchResponse = await axios.post('federation/fetches',
|
const fetchResponse = await axios.post(
|
||||||
{ object: id }
|
'federation/fetches',
|
||||||
|
{ object_uri: id },
|
||||||
|
{
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/activity+json'
|
||||||
|
}
|
||||||
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
const response = await axios.post(
|
const response = await axios.post(
|
||||||
'federation/follows/library',
|
'federation/follows/library',
|
||||||
{ target: fetchResponse.data.object.uuid }
|
{ target: fetchResponse.data.object.id.split('/').pop() }
|
||||||
);
|
);
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
|
|
|
@ -10590,6 +10590,7 @@ export interface operations {
|
||||||
[name: string]: unknown;
|
[name: string]: unknown;
|
||||||
};
|
};
|
||||||
content: {
|
content: {
|
||||||
|
"application/activity+json": components["schemas"]["Fetch"];
|
||||||
"application/json": components["schemas"]["Fetch"];
|
"application/json": components["schemas"]["Fetch"];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
@ -10612,6 +10613,7 @@ export interface operations {
|
||||||
[name: string]: unknown;
|
[name: string]: unknown;
|
||||||
};
|
};
|
||||||
content: {
|
content: {
|
||||||
|
"application/activity+json": components["schemas"]["Fetch"];
|
||||||
"application/json": components["schemas"]["Fetch"];
|
"application/json": components["schemas"]["Fetch"];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
@ -17537,6 +17539,7 @@ export interface operations {
|
||||||
[name: string]: unknown;
|
[name: string]: unknown;
|
||||||
};
|
};
|
||||||
content: {
|
content: {
|
||||||
|
"application/activity+json": components["schemas"]["Fetch"];
|
||||||
"application/json": components["schemas"]["Fetch"];
|
"application/json": components["schemas"]["Fetch"];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
@ -17559,6 +17562,7 @@ export interface operations {
|
||||||
[name: string]: unknown;
|
[name: string]: unknown;
|
||||||
};
|
};
|
||||||
content: {
|
content: {
|
||||||
|
"application/activity+json": components["schemas"]["Fetch"];
|
||||||
"application/json": components["schemas"]["Fetch"];
|
"application/json": components["schemas"]["Fetch"];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
Loading…
Reference in New Issue