Merge branch '2405-front-buttont-trigger-third-party-hook' into 'develop'

Draft: trigger plugins hook manually from the frontend

Closes #2405

See merge request funkwhale/funkwhale!2933
This commit is contained in:
petitminion 2025-07-10 15:55:45 +00:00
commit 6c23c49d1f
19 changed files with 776 additions and 25 deletions

View File

@ -2462,6 +2462,9 @@ paths:
responses: responses:
'201': '201':
content: content:
application/activity+json:
schema:
$ref: '#/components/schemas/Fetch'
application/json: application/json:
schema: schema:
$ref: '#/components/schemas/Fetch' $ref: '#/components/schemas/Fetch'
@ -2484,6 +2487,9 @@ paths:
responses: responses:
'200': '200':
content: content:
application/activity+json:
schema:
$ref: '#/components/schemas/Fetch'
application/json: application/json:
schema: schema:
$ref: '#/components/schemas/Fetch' $ref: '#/components/schemas/Fetch'
@ -9320,6 +9326,36 @@ paths:
schema: schema:
$ref: '#/components/schemas/UploadForOwner' $ref: '#/components/schemas/UploadForOwner'
description: '' description: ''
/api/v1/uploads/trigger-download/:
post:
operationId: create_upload_trigger_download
tags:
- uploads
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/UploadForOwnerRequest'
application/x-www-form-urlencoded:
schema:
$ref: '#/components/schemas/UploadForOwnerRequest'
multipart/form-data:
schema:
$ref: '#/components/schemas/UploadForOwnerRequest'
application/activity+json:
schema:
$ref: '#/components/schemas/UploadForOwnerRequest'
required: true
security:
- oauth2: []
- ApplicationToken: []
responses:
'200':
content:
application/json:
schema:
$ref: '#/components/schemas/UploadForOwner'
description: ''
/api/v1/users/{username}/: /api/v1/users/{username}/:
put: put:
operationId: update_user operationId: update_user
@ -11972,6 +12008,9 @@ paths:
responses: responses:
'201': '201':
content: content:
application/activity+json:
schema:
$ref: '#/components/schemas/Fetch'
application/json: application/json:
schema: schema:
$ref: '#/components/schemas/Fetch' $ref: '#/components/schemas/Fetch'
@ -11994,6 +12033,9 @@ paths:
responses: responses:
'200': '200':
content: content:
application/activity+json:
schema:
$ref: '#/components/schemas/Fetch'
application/json: application/json:
schema: schema:
$ref: '#/components/schemas/Fetch' $ref: '#/components/schemas/Fetch'
@ -18995,6 +19037,36 @@ paths:
schema: schema:
$ref: '#/components/schemas/UploadForOwner' $ref: '#/components/schemas/UploadForOwner'
description: '' description: ''
/api/v2/uploads/trigger-download/:
post:
operationId: create_upload_trigger_download_2
tags:
- uploads
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/UploadForOwnerRequest'
application/x-www-form-urlencoded:
schema:
$ref: '#/components/schemas/UploadForOwnerRequest'
multipart/form-data:
schema:
$ref: '#/components/schemas/UploadForOwnerRequest'
application/activity+json:
schema:
$ref: '#/components/schemas/UploadForOwnerRequest'
required: true
security:
- oauth2: []
- ApplicationToken: []
responses:
'200':
content:
application/json:
schema:
$ref: '#/components/schemas/UploadForOwner'
description: ''
/api/v2/users/{username}/: /api/v2/users/{username}/:
put: put:
operationId: update_user_3 operationId: update_user_3
@ -24755,6 +24827,10 @@ components:
default: pending default: pending
privacy_level: privacy_level:
$ref: '#/components/schemas/LibraryPrivacyLevelEnum' $ref: '#/components/schemas/LibraryPrivacyLevelEnum'
third_party_provider:
type: string
nullable: true
maxLength: 100
import_metadata: import_metadata:
$ref: '#/components/schemas/ImportMetadataRequest' $ref: '#/components/schemas/ImportMetadataRequest'
import_reference: import_reference:
@ -25896,6 +25972,10 @@ components:
default: pending default: pending
privacy_level: privacy_level:
$ref: '#/components/schemas/LibraryPrivacyLevelEnum' $ref: '#/components/schemas/LibraryPrivacyLevelEnum'
third_party_provider:
type: string
nullable: true
maxLength: 100
import_details: import_details:
readOnly: true readOnly: true
import_metadata: import_metadata:
@ -25939,6 +26019,10 @@ components:
default: pending default: pending
privacy_level: privacy_level:
$ref: '#/components/schemas/LibraryPrivacyLevelEnum' $ref: '#/components/schemas/LibraryPrivacyLevelEnum'
third_party_provider:
type: string
nullable: true
maxLength: 100
import_metadata: import_metadata:
$ref: '#/components/schemas/ImportMetadataRequest' $ref: '#/components/schemas/ImportMetadataRequest'
import_reference: import_reference:

View File

@ -9,5 +9,5 @@ logger = logging.getLogger(__name__)
@plugins.register_hook(plugins.TRIGGER_THIRD_PARTY_UPLOAD, PLUGIN) @plugins.register_hook(plugins.TRIGGER_THIRD_PARTY_UPLOAD, PLUGIN)
def lauch_download(track, conf={}): def lauch_download(track, actor, conf={}):
tasks.archive_download.delay(track_id=track.pk, conf=conf) tasks.archive_download.delay(track_id=track.pk, conf=conf)

View File

@ -53,7 +53,7 @@ def check_last_third_party_queries(track, count):
check_last_third_party_queries(track, count) check_last_third_party_queries(track, count)
def create_upload(url, track, files_data): def get_or_create_upload(url, track, files_data):
mimetype = f"audio/{files_data.get('format', 'unknown')}" mimetype = f"audio/{files_data.get('format', 'unknown')}"
duration = files_data.get("mtime", 0) duration = files_data.get("mtime", 0)
filesize = files_data.get("size", 0) filesize = files_data.get("size", 0)
@ -64,19 +64,23 @@ def create_upload(url, track, files_data):
actor=actors.get_service_actor(), actor=actors.get_service_actor(),
) )
return models.Upload.objects.create( defaults = {
"creation_date": timezone.now(),
"duration": duration,
"size": filesize,
"bitrate": bitrate,
"import_status": "pending",
}
upload, created = models.Upload.objects.get_or_create(
defaults=defaults,
mimetype=mimetype, mimetype=mimetype,
source=url, source=url,
third_party_provider="archive-dl", third_party_provider="archive-dl",
creation_date=timezone.now(),
track=track, track=track,
duration=duration,
size=filesize,
bitrate=bitrate,
library=service_library, library=service_library,
from_activity=None, from_activity=None,
import_status="pending",
) )
return upload
@celery.app.task(name="archivedl.archive_download") @celery.app.task(name="archivedl.archive_download")
@ -89,9 +93,11 @@ def archive_download(track, conf):
logger.error(e) logger.error(e)
return return
artist_name = utils.get_artist_credit_string(track) artist_name = utils.get_artist_credit_string(track)
# a lot of times this don't find anything, archive relies more on albums than tracks
query = f"mediatype:audio AND title:{track.title} AND creator:{artist_name}" query = f"mediatype:audio AND title:{track.title} AND creator:{artist_name}"
with requests.Session() as session: with requests.Session() as session:
url = get_search_url(query, page_size=1, page=1) url = get_search_url(query, page_size=1, page=1)
get_or_create_upload(url, track, files_data={})
page_data = fetch_json(url, session) page_data = fetch_json(url, session)
for obj in page_data["response"]["docs"]: for obj in page_data["response"]["docs"]:
logger.info(f"launching download item for {str(obj)}") logger.info(f"launching download item for {str(obj)}")
@ -123,7 +129,7 @@ def download_item(
) )
) )
url = f"https://archive.org/download/{item_data['identifier']}/{to_download[0]['name']}" url = f"https://archive.org/download/{item_data['identifier']}/{to_download[0]['name']}"
upload = create_upload(url, track, to_download[0]) upload = get_or_create_upload(url, track, to_download[0])
try: try:
with tempfile.TemporaryDirectory() as temp_dir: with tempfile.TemporaryDirectory() as temp_dir:
path = os.path.join(temp_dir, to_download[0]["name"]) path = os.path.join(temp_dir, to_download[0]["name"])

View File

@ -13,7 +13,9 @@ from funkwhale_api.audio import models as audio_models
from funkwhale_api.audio import serializers as audio_serializers from funkwhale_api.audio import serializers as audio_serializers
from funkwhale_api.common import serializers as common_serializers from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.music import models as music_models from funkwhale_api.music import models as music_models
from funkwhale_api.music import serializers as music_serializers
from funkwhale_api.playlists import models as playlists_models from funkwhale_api.playlists import models as playlists_models
from funkwhale_api.playlists import serializers as playlist_serializers
from funkwhale_api.users import serializers as users_serializers from funkwhale_api.users import serializers as users_serializers
from . import filters, models from . import filters, models
@ -197,10 +199,19 @@ OBJECT_SERIALIZER_MAPPING = {
music_models.Artist: federation_serializers.ArtistSerializer, music_models.Artist: federation_serializers.ArtistSerializer,
music_models.Album: federation_serializers.AlbumSerializer, music_models.Album: federation_serializers.AlbumSerializer,
music_models.Track: federation_serializers.TrackSerializer, music_models.Track: federation_serializers.TrackSerializer,
music_models.Library: federation_serializers.LibrarySerializer,
models.Actor: federation_serializers.APIActorSerializer, models.Actor: federation_serializers.APIActorSerializer,
audio_models.Channel: audio_serializers.ChannelSerializer, audio_models.Channel: audio_serializers.ChannelSerializer,
playlists_models.Playlist: federation_serializers.PlaylistSerializer, playlists_models.Playlist: federation_serializers.PlaylistSerializer,
} }
OBJECT_MUSIC_SERIALIZER_MAPPING = {
music_models.Artist: music_serializers.ArtistSerializer,
music_models.Album: music_serializers.AlbumSerializer,
music_models.Track: music_serializers.TrackSerializer,
models.Actor: federation_serializers.APIActorSerializer,
audio_models.Channel: audio_serializers.ChannelSerializer,
playlists_models.Playlist: playlist_serializers.PlaylistSerializer,
}
def convert_url_to_webfinger(url): def convert_url_to_webfinger(url):
@ -283,6 +294,9 @@ class FetchSerializer(serializers.ModelSerializer):
return value return value
return f"webfinger://{value}" return f"webfinger://{value}"
# to do : this is incomplete, schema conflict because
# federation serializers have the same name than musi serializer -> upgrade fed serializers to new names
# and add the new object here
@extend_schema_field( @extend_schema_field(
{ {
"oneOf": [ "oneOf": [
@ -300,7 +314,12 @@ class FetchSerializer(serializers.ModelSerializer):
if obj is None: if obj is None:
return None return None
serializer_class = OBJECT_SERIALIZER_MAPPING.get(type(obj)) media_type = self.context.get("media_type")
if media_type == "application/activity+json":
serializer_class = OBJECT_SERIALIZER_MAPPING.get(type(obj))
else:
serializer_class = OBJECT_MUSIC_SERIALIZER_MAPPING.get(type(obj))
if serializer_class: if serializer_class:
return serializer_class(obj).data return serializer_class(obj).data
return None return None

View File

@ -1,3 +1,5 @@
from urllib.parse import urlparse
import requests.exceptions import requests.exceptions
from django.conf import settings from django.conf import settings
from django.db import transaction from django.db import transaction
@ -5,10 +7,13 @@ from django.db.models import Count, Q
from drf_spectacular.utils import extend_schema, extend_schema_view from drf_spectacular.utils import extend_schema, extend_schema_view
from rest_framework import decorators, mixins, permissions, response, viewsets from rest_framework import decorators, mixins, permissions, response, viewsets
from rest_framework.exceptions import NotFound as RestNotFound from rest_framework.exceptions import NotFound as RestNotFound
from rest_framework.negotiation import DefaultContentNegotiation
from rest_framework.renderers import JSONRenderer
from funkwhale_api.common import preferences from funkwhale_api.common import preferences
from funkwhale_api.common import utils as common_utils from funkwhale_api.common import utils as common_utils
from funkwhale_api.common.permissions import ConditionalAuthentication from funkwhale_api.common.permissions import ConditionalAuthentication
from funkwhale_api.common.renderers import ActivityStreamRenderer
from funkwhale_api.music import models as music_models from funkwhale_api.music import models as music_models
from funkwhale_api.music import serializers as music_serializers from funkwhale_api.music import serializers as music_serializers
from funkwhale_api.music import views as music_views from funkwhale_api.music import views as music_views
@ -243,20 +248,41 @@ class FetchViewSet(
serializer_class = api_serializers.FetchSerializer serializer_class = api_serializers.FetchSerializer
permission_classes = [permissions.IsAuthenticated] permission_classes = [permissions.IsAuthenticated]
throttling_scopes = {"create": {"authenticated": "fetch"}} throttling_scopes = {"create": {"authenticated": "fetch"}}
renderer_classes = [ActivityStreamRenderer, JSONRenderer]
def get_queryset(self): def get_queryset(self):
return super().get_queryset().filter(actor=self.request.user.actor) return super().get_queryset().filter(actor=self.request.user.actor)
def get_serializer_context(self):
context = super().get_serializer_context()
negotiator = DefaultContentNegotiation()
try:
renderer, media_type = negotiator.select_renderer(
self.request, self.get_renderers()
)
context["media_type"] = media_type
except Exception:
context["media_type"] = None
return context
def perform_create(self, serializer): def perform_create(self, serializer):
fetch = serializer.save(actor=self.request.user.actor) fetch = serializer.save(actor=self.request.user.actor)
if fetch.status == "finished": if fetch.status == "finished":
# a duplicate was returned, no need to fetch again # a duplicate was returned, no need to fetch again
return return
if settings.FEDERATION_SYNCHRONOUS_FETCH:
tasks.fetch(fetch_id=fetch.pk) parsed_url = urlparse(fetch.url)
domain = parsed_url.netloc
if domain in fetch.supported_services:
tasks.third_party_fetch(fetch_id=fetch.pk)
fetch.refresh_from_db() fetch.refresh_from_db()
else: else:
common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk) if settings.FEDERATION_SYNCHRONOUS_FETCH:
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
else:
common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk)
class DomainViewSet( class DomainViewSet(

View File

@ -18,6 +18,7 @@ from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils from funkwhale_api.common import utils as common_utils
from funkwhale_api.common import validators as common_validators from funkwhale_api.common import validators as common_validators
from funkwhale_api.music import utils as music_utils from funkwhale_api.music import utils as music_utils
from funkwhale_api.musicbrainz import serializers as musicbrainz_serializers
from . import utils as federation_utils from . import utils as federation_utils
@ -411,8 +412,15 @@ class Fetch(models.Model):
contexts.AS.Organization: [serializers.ActorSerializer], contexts.AS.Organization: [serializers.ActorSerializer],
contexts.AS.Service: [serializers.ActorSerializer], contexts.AS.Service: [serializers.ActorSerializer],
contexts.AS.Application: [serializers.ActorSerializer], contexts.AS.Application: [serializers.ActorSerializer],
# for mb the key must be the api namespace
"recordings": [musicbrainz_serializers.RecordingSerializer],
"releases": [musicbrainz_serializers.ReleaseSerializer],
} }
@property
def supported_services(self):
return ["musicbrainz.org"]
class InboxItem(models.Model): class InboxItem(models.Model):
""" """

View File

@ -1066,6 +1066,7 @@ class LibrarySerializer(PaginatedCollectionSerializer):
privacy = {"": "me", "./": "me", None: "me", contexts.AS.Public: "everyone"} privacy = {"": "me", "./": "me", None: "me", contexts.AS.Public: "everyone"}
library, created = music_models.Library.objects.update_or_create( library, created = music_models.Library.objects.update_or_create(
fid=validated_data["id"], fid=validated_data["id"],
uuid=validated_data["id"].rstrip("/").split("/")[-1],
actor=actor, actor=actor,
defaults={ defaults={
"uploads_count": validated_data["totalItems"], "uploads_count": validated_data["totalItems"],
@ -1449,7 +1450,7 @@ class AlbumSerializer(MusicEntitySerializer):
acs.append( acs.append(
utils.retrieve_ap_object( utils.retrieve_ap_object(
ac["id"], ac["id"],
actor=self.context.get("fetch_actor"), actor=self.context.get("_actor"),
queryset=music_models.ArtistCredit, queryset=music_models.ArtistCredit,
serializer_class=ArtistCreditSerializer, serializer_class=ArtistCreditSerializer,
) )

View File

@ -2,9 +2,12 @@ import datetime
import json import json
import logging import logging
import os import os
import uuid
from urllib.parse import urlparse
import requests import requests
from django.conf import settings from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache from django.core.cache import cache
from django.db import transaction from django.db import transaction
from django.db.models import F, Q from django.db.models import F, Q
@ -13,6 +16,8 @@ from django.utils import timezone
from dynamic_preferences.registries import global_preferences_registry from dynamic_preferences.registries import global_preferences_registry
from requests.exceptions import RequestException from requests.exceptions import RequestException
from config import plugins
from funkwhale_api import musicbrainz
from funkwhale_api.audio import models as audio_models from funkwhale_api.audio import models as audio_models
from funkwhale_api.common import models as common_models from funkwhale_api.common import models as common_models
from funkwhale_api.common import preferences, session from funkwhale_api.common import preferences, session
@ -449,7 +454,6 @@ def fetch(fetch_obj):
max_pages=settings.FEDERATION_COLLECTION_MAX_PAGES - 1, max_pages=settings.FEDERATION_COLLECTION_MAX_PAGES - 1,
is_page=True, is_page=True,
) )
fetch_obj.object = obj fetch_obj.object = obj
fetch_obj.status = "finished" fetch_obj.status = "finished"
fetch_obj.fetch_date = timezone.now() fetch_obj.fetch_date = timezone.now()
@ -458,6 +462,167 @@ def fetch(fetch_obj):
) )
def trigger_third_party_upload_hook(fetch):
if fetch.status == "finished" and fetch.object:
if fetch.object_content_type == ContentType.objects.get_for_model(
music_models.Track
):
if not music_models.Track.objects.filter(pk=fetch.object.pk).playable_by(
fetch.actor
):
plugins.trigger_hook(
plugins.TRIGGER_THIRD_PARTY_UPLOAD,
track=fetch.object,
)
if fetch.object_content_type == ContentType.objects.get_for_model(
music_models.Album
):
for track in fetch.object.tracks.all():
if not music_models.Track.objects.filter(
pk=fetch.object.pk
).playable_by(fetch.actor):
plugins.trigger_hook(
plugins.TRIGGER_THIRD_PARTY_UPLOAD,
track=track,
)
def musicbrainz_type_handler(fetch):
url = fetch.url
path_parts = urlparse(url).path.strip("/").split("/")
type_ = path_parts[0] + "s"
mbid = path_parts[1]
try:
uuid.UUID(mbid)
except ValueError:
raise ValueError(f"could no get mbid from url {url}")
return type_, mbid
def musicbrainz_metadata_handler(type_, id):
def replace_hyphens_in_keys(obj):
if isinstance(obj, dict):
return {
k.replace("-", "_"): replace_hyphens_in_keys(v) for k, v in obj.items()
}
elif isinstance(obj, list):
return [replace_hyphens_in_keys(item) for item in obj]
else:
return obj
if type_ == "recordings":
includes = ["tags", "artists", "releases"]
elif type_ == "releases":
includes = ["tags", "artists", "recordings"]
result = replace_hyphens_in_keys(
getattr(musicbrainz.api, type_).get(id=id, includes=includes)
)
existing = (
music_models.Track.objects.filter(mbid=id).first()
if music_models.Track.objects.filter(mbid=id).exists()
else None
)
return result, existing
type_and_id_from_third_party = {"musicbrainz.org": musicbrainz_type_handler}
metadata_from_third_party_ = {"musicbrainz.org": musicbrainz_metadata_handler}
@celery.app.task(name="third_party_fetch")
@transaction.atomic
@celery.require_instance(
models.Fetch.objects.filter(status="pending").select_related("actor"),
"fetch_obj",
"fetch_id",
)
def third_party_fetch(fetch_obj):
def error(code, **kwargs):
fetch_obj.status = "errored"
fetch_obj.fetch_date = timezone.now()
fetch_obj.detail = {"error_code": code}
fetch_obj.detail.update(kwargs)
fetch_obj.save(update_fields=["fetch_date", "status", "detail"])
def check_url(url):
if not url.startswith("webfinger://"):
payload, updated = mrf.inbox.apply({"id": url})
if not payload:
return error("blocked", message="Blocked by MRF")
parsed_url = urlparse(url)
service = parsed_url.netloc
if service not in fetch_obj.supported_services:
return error("invalid_url", message=f"Unsupported domain {service}")
return service
url = fetch_obj.url
actor = fetch_obj.actor
service = check_url(url)
try:
type_, id = type_and_id_from_third_party[service](fetch_obj)
logger.debug("Parsed URL %s into type %s and id %s", url, type_, id)
except ValueError as e:
return error("url_parse_error", message=str(e))
try:
result, existing = metadata_from_third_party_[service](type_, id)
logger.debug(
f"Remote answered with {result} and we found {existing} in database"
)
except requests.exceptions.HTTPError as e:
return error(
"http",
status_code=e.response.status_code if e.response else None,
message=e.response.text,
)
except requests.exceptions.Timeout:
return error("timeout")
except requests.exceptions.ConnectionError as e:
return error("connection", message=str(e))
except requests.RequestException as e:
return error("request", message=str(e))
except Exception as e:
return error("unhandled", message=str(e))
try:
serializer_classes = fetch_obj.serializers.get(type_)
except (KeyError, AttributeError):
fetch_obj.status = "skipped"
fetch_obj.fetch_date = timezone.now()
fetch_obj.detail = {"reason": "unhandled_type", "type": type_}
return fetch_obj.save(update_fields=["fetch_date", "status", "detail"])
serializer = None
for serializer_class in serializer_classes:
serializer = serializer_class(
existing, data=result, context={"fetch_actor": actor}
)
if not serializer.is_valid():
continue
else:
break
if serializer.errors:
return error("validation", validation_errors=serializer.errors)
try:
obj = serializer.save()
except Exception as e:
error("save", message=str(e))
raise
fetch_obj.object = obj
fetch_obj.status = "finished"
fetch_obj.fetch_date = timezone.now()
trigger_third_party_upload_hook(fetch_obj)
fetch_obj.save(
update_fields=["fetch_date", "status", "object_id", "object_content_type"]
)
return fetch_obj
class PreserveSomeDataCollector(Collector): class PreserveSomeDataCollector(Collector):
""" """
We need to delete everything related to an actor. Well Almost everything. We need to delete everything related to an actor. Well Almost everything.

View File

@ -401,6 +401,7 @@ class UploadSerializer(serializers.ModelSerializer):
"import_date", "import_date",
"import_status", "import_status",
"privacy_level", "privacy_level",
"third_party_provider",
] ]
read_only_fields = [ read_only_fields = [

View File

@ -18,6 +18,7 @@ from rest_framework import views, viewsets
from rest_framework.decorators import action from rest_framework.decorators import action
from rest_framework.response import Response from rest_framework.response import Response
from config import plugins
from funkwhale_api.common import decorators as common_decorators from funkwhale_api.common import decorators as common_decorators
from funkwhale_api.common import permissions as common_permissions from funkwhale_api.common import permissions as common_permissions
from funkwhale_api.common import preferences from funkwhale_api.common import preferences
@ -821,6 +822,27 @@ class UploadViewSet(
status=200, status=200,
) )
@action(methods=["post"], detail=False, url_path="trigger-download")
def trigger_download(self, request, *args, **kwargs):
qs = self.get_queryset()
track = models.Track.objects.get(pk=request.data["track"])
actor = utils.get_actor_from_request(self.request)
tp_upload = models.Upload.objects.filter(
track=track, third_party_provider__isnull=False
)
if tp_upload.exists():
return Response(
serializers.UploadSerializer(tp_upload.first()).data,
status=200,
)
if not qs.filter(track=track).exists():
plugins.trigger_hook(
plugins.TRIGGER_THIRD_PARTY_UPLOAD,
track=track,
actor=actor,
)
return Response(status=404)
@action(methods=["post"], detail=False) @action(methods=["post"], detail=False)
def action(self, request, *args, **kwargs): def action(self, request, *args, **kwargs):
queryset = self.get_queryset() queryset = self.get_queryset()

View File

@ -0,0 +1,251 @@
import logging
from rest_framework import serializers
from funkwhale_api import musicbrainz
from funkwhale_api.tags import models as tags_models
from . import client
logger = logging.getLogger(__name__)
class ArtistSerializer(serializers.Serializer):
"""
Serializer for Musicbrainz artist data.
"""
id = serializers.CharField()
name = serializers.CharField()
def create(self, validated_data):
from funkwhale_api.music.models import Artist
data = {
"name": validated_data["name"],
"mbid": validated_data["id"],
}
artist, created = Artist.objects.get_or_create(**data)
return artist
class ArtistCreditSerializer(serializers.Serializer):
"""
Serializer for Musicbrainz artist data.
"""
name = serializers.CharField()
joinphrase = serializers.CharField(allow_blank=True)
artist = ArtistSerializer()
def create(self, validated_data):
from funkwhale_api.music.models import ArtistCredit
data = {
"credit": validated_data["name"],
"joinphrase": validated_data.get("joinphrase", ""),
"artist": ArtistSerializer().create(validated_data["artist"]),
}
artist_credit, created = ArtistCredit.objects.get_or_create(**data)
return artist_credit
class ReleaseForRecordingSerializer(serializers.Serializer):
"""
Serializer for Musicbrainz release data when returned in a recording object.
"""
id = serializers.CharField()
title = serializers.CharField()
artist_credit = ArtistCreditSerializer(many=True)
tags = serializers.ListField(child=serializers.CharField(), allow_empty=True)
date = serializers.DateField(input_formats=["%Y", "%Y/%m/%d", "%Y-%m-%d"])
def create(self, validated_data):
from funkwhale_api.music.models import Album
data = {
"title": validated_data["title"],
"mbid": validated_data["id"],
"release_date": validated_data.get("date", None),
}
album, created = Album.objects.get_or_create(**data)
artist_credit = ArtistCreditSerializer(many=True).create(
validated_data["artist_credit"]
)
album.artist_credit.set(artist_credit)
album.save()
tags_models.add_tags(album, *validated_data.get("tags", []))
return album
def update(self, instance, validated_data):
instance.title = validated_data["title"]
instance.release_date = validated_data.get("date")
instance.save()
tags_models.add_tags(instance, *validated_data.get("tags", []))
return instance
class RecordingSerializer(serializers.Serializer):
"""
Serializer for Musicbrainz track data.
"""
id = serializers.CharField()
title = serializers.CharField()
artist_credit = ArtistCreditSerializer(many=True)
releases = ReleaseForRecordingSerializer(many=True, required=False)
tags = serializers.ListField(child=serializers.CharField(), allow_empty=True)
position = serializers.IntegerField(required=False, allow_null=True)
def create(self, validated_data):
from funkwhale_api.music.models import Track
data = {"mbid": validated_data["id"]}
defaults = {
"title": validated_data["title"],
"mbid": validated_data["id"],
# In mb a recording can have various releases, we take the fist one
"album": (
ReleaseForRecordingSerializer(many=True).create(
validated_data["releases"]
)[0]
if validated_data.get("releases")
else None
),
# this will be none if the recording is not fetched from the release endpoint
"position": validated_data.get("position", None),
}
if defaults["album"] is not None and defaults["position"] is None:
result = client.api.releases.get(
id=validated_data["releases"][0]["id"],
includes=["tags", "artists", "recordings"],
)
tracks = result["media"][0]["tracks"]
defaults["position"] = next(
(o for o in tracks if o["recording"]["id"] == data["mbid"]), {}
).get("position", None)
track, created = Track.objects.get_or_create(**data, defaults=defaults)
artist_credit = ArtistCreditSerializer(many=True).create(
validated_data["artist_credit"]
)
track.artist_credit.set(artist_credit)
track.save()
tags_models.add_tags(track, *validated_data.get("tags", []))
return track
def update(self, instance, validated_data):
instance.title = validated_data["title"]
instance.save()
tags_models.add_tags(instance, *validated_data.get("tags", []))
return instance
class RecordingForReleaseSerializer(serializers.Serializer):
id = serializers.CharField()
title = serializers.CharField()
# not in Musicbrainz recording object, but used to store the position of the track in the album
position = serializers.IntegerField(required=False, allow_null=True)
def create(self, validated_data):
def replace_hyphens_in_keys(obj):
if isinstance(obj, dict):
return {
k.replace("-", "_"): replace_hyphens_in_keys(v)
for k, v in obj.items()
}
elif isinstance(obj, list):
return [replace_hyphens_in_keys(item) for item in obj]
else:
return obj
recordings_data = musicbrainz.api.recordings.get(
id=validated_data["id"], includes=["tags", "artists"]
)
recordings_data = replace_hyphens_in_keys(recordings_data)
recordings_data["position"] = validated_data.get("position", None)
serializer = RecordingSerializer(data=recordings_data)
serializer.is_valid(raise_exception=True)
track = serializer.save()
track.album = validated_data["album"]
track.save()
return track
def update(self, instance, validated_data):
instance.title = validated_data["title"]
tags_models.add_tags(instance, *validated_data.get("tags", []))
instance.album = validated_data["album"]
instance.save()
return instance
class TrackSerializer(serializers.Serializer):
recording = RecordingForReleaseSerializer()
position = serializers.IntegerField()
class MediaSerializer(serializers.Serializer):
tracks = TrackSerializer(many=True)
class ReleaseSerializer(serializers.Serializer):
"""
Serializer for Musicbrainz release data.
"""
id = serializers.CharField()
title = serializers.CharField()
artist_credit = ArtistCreditSerializer(many=True)
tags = serializers.ListField(child=serializers.CharField(), allow_empty=True)
date = serializers.DateField(input_formats=["%Y", "%Y/%m/%d", "%Y-%m-%d"])
media = serializers.ListField(child=MediaSerializer())
def create(self, validated_data):
from funkwhale_api.music.models import Album
data = {
"title": validated_data["title"],
"mbid": validated_data["id"],
"release_date": validated_data.get("date"),
}
album, created = Album.objects.get_or_create(**data)
artist_credit = ArtistCreditSerializer(many=True).create(
validated_data["artist_credit"]
)
album.artist_credit.set(artist_credit)
album.save()
tags_models.add_tags(album, *validated_data.get("tags", []))
# an album can have various media/physical representation, we take the first one
tracks = [t for t in validated_data["media"][0]["tracks"]]
recordings = []
for t in tracks:
t["recording"]["position"] = t["position"]
recordings.append(t["recording"])
for r in recordings:
r["album"] = album
RecordingForReleaseSerializer().create(r)
return album
# this will never be used while FetchViewSet and third_party_fetch filter out finished fetch
# Would be nice to have a way to manually update releases from Musicbrainz
def update(self, instance, validated_data):
logger.info(f"Updating release {instance} with data: {validated_data}")
instance.title = validated_data["title"]
instance.release_date = validated_data.get("date")
instance.save()
tags_models.add_tags(instance, *validated_data.get("tags", []))
recordings = [t["recording"] for t in validated_data["media"][0]["tracks"]]
for r in recordings:
r["album"] = instance
RecordingForReleaseSerializer().update(r)
return instance

View File

@ -173,7 +173,12 @@ def test_fetch_serializer_with_object(
"actor": serializers.APIActorSerializer(fetch.actor).data, "actor": serializers.APIActorSerializer(fetch.actor).data,
} }
assert api_serializers.FetchSerializer(fetch).data == expected assert (
api_serializers.FetchSerializer(
fetch, context={"media_type": "application/activity+json"}
).data
== expected
)
def test_fetch_serializer_unhandled_obj(factories, to_api_date): def test_fetch_serializer_unhandled_obj(factories, to_api_date):

View File

@ -0,0 +1 @@
Import tracks and albums from Musicbrainz into Funkwhale though the search bar (#2926)

22
docs/specs/fetch-third-party/index.md vendored Normal file
View File

@ -0,0 +1,22 @@
# Collections
## The issue
Has a user I want to be able to get metadata from third party services (to add tracks to my favorites or to a playlist)
Has a user I want to clone a third party playslit into Funkwhale
## Solution
paste the audio object (track, album, playlist) url into the search bar. Funkwhale will get the mmetadata and create the objects in db. First implementation with Musicbrainz :
- track : https://musicbrainz.org/release/{mbid}
- release : https://musicbrainz.org/recording/{mbid}
## Implementation
third_party_fetch in federation.tasks
musicbrainz.serializers
## Call for developers
This is time consuming. The main logic is implemented. Now everybody can add support for any third party service.

View File

@ -3,7 +3,7 @@ import type { Track, Artist, Album, Playlist, Library, Channel, Actor } from '~/
import type { components } from '~/generated/types' import type { components } from '~/generated/types'
import type { PlayOptionsProps } from '~/composables/audio/usePlayOptions' import type { PlayOptionsProps } from '~/composables/audio/usePlayOptions'
import { computed, ref } from 'vue' import { computed, ref, watch } from 'vue'
import { useI18n } from 'vue-i18n' import { useI18n } from 'vue-i18n'
import usePlayOptions from '~/composables/audio/usePlayOptions' import usePlayOptions from '~/composables/audio/usePlayOptions'
import useReport from '~/composables/moderation/useReport' import useReport from '~/composables/moderation/useReport'
@ -14,6 +14,7 @@ import Button from '~/components/ui/Button.vue'
import OptionsButton from '~/components/ui/button/Options.vue' import OptionsButton from '~/components/ui/button/Options.vue'
import Popover from '~/components/ui/Popover.vue' import Popover from '~/components/ui/Popover.vue'
import PopoverItem from '~/components/ui/popover/PopoverItem.vue' import PopoverItem from '~/components/ui/popover/PopoverItem.vue'
import axios from 'axios'
interface Props extends PlayOptionsProps { interface Props extends PlayOptionsProps {
split?: boolean split?: boolean
@ -103,10 +104,36 @@ const labels = computed(() => ({
PlaylistUploadGranted: t('components.audio.PlayButton.button.PlaylistUploadGranted'), PlaylistUploadGranted: t('components.audio.PlayButton.button.PlaylistUploadGranted'),
PlaylistUploadPending:t('components.audio.PlayButton.button.PlaylistUploadPending'), PlaylistUploadPending:t('components.audio.PlayButton.button.PlaylistUploadPending'),
PlaylistUploadNotRequest: t('components.audio.PlayButton.button.PlaylistUploadNotRequest'), PlaylistUploadNotRequest: t('components.audio.PlayButton.button.PlaylistUploadNotRequest'),
PlaylistUploadTooltip: t('components.audio.PlayButton.button.PlaylistUploadTooltip') PlaylistUploadTooltip: t('components.audio.PlayButton.button.PlaylistUploadTooltip'),
thirdPartyDownload: t('components.audio.PlayButton.button.thirdPartyDownload'),
thirdPartyDownloadTooltip: t('components.audio.PlayButton.button.thirdPartyDownload'),
thirdPartyDownloadTriggered: t('components.audio.PlayButton.button.thirdPartyDownloadTriggered')
})) }))
const isOpen = ref(false) const isOpen = ref(false)
const uploads = ref<components['schemas']['UploadForOwner'][]>([])
const thirdPartyUpload = computed(() => {
if (uploads.value?.find(upload => 'third_party_provider' in upload)) { return false } else { return true }
})
const triggerThirdPartyHook = async () => {
const response = await axios.post(
'uploads/trigger-download',
{ track: props.track?.id }
);
if (response.status === 404) {
uploads.value.push({ third_party_upload: "triggered" });
return null;
} else if (response.status === 200) {
console.log("uploads 400 (response.dat ", response.data)
uploads.value.push(response.data)
return response.data;
}
};
triggerThirdPartyHook()
const playlistLibraryFollowInfo = computed(() => { const playlistLibraryFollowInfo = computed(() => {
const playlist = props.playlist; const playlist = props.playlist;
@ -246,7 +273,22 @@ const playlistLibraryFollowInfo = computed(() => {
{{ t('components.audio.PlayButton.button.trackDetails') }} {{ t('components.audio.PlayButton.button.trackDetails') }}
</span> </span>
</PopoverItem> </PopoverItem>
<PopoverItem
v-if="!playable && thirdPartyUpload"
:title="labels.thirdPartyDownloadTooltip"
icon="bi-cloud-download"
@click.stop.prevent="triggerThirdPartyHook();"
>
{{ labels.thirdPartyDownload }}
</PopoverItem>
<PopoverItem
v-else-if="!playable && !thirdPartyUpload"
:title="labels.thirdPartyDownloadTooltip"
:disabled="true"
icon="bi-cloud-download"
>
{{ labels.thirdPartyDownloadTriggered }}
</PopoverItem>
<hr v-if="filterableArtist || Object.keys(getReportableObjects({ track, album, artist, playlist, account, channel })).length > 0"> <hr v-if="filterableArtist || Object.keys(getReportableObjects({ track, album, artist, playlist, account, channel })).length > 0">
<PopoverItem <PopoverItem

View File

@ -73,7 +73,6 @@ watch(() => props.url, () => {
/> />
<Alert <Alert
v-if="!isLoading && libraries.length === 0" v-if="!isLoading && libraries.length === 0"
blue
style="grid-column: 1 / -1;" style="grid-column: 1 / -1;"
> >
{{ t('components.federation.LibraryWidget.empty.noMatch') }} {{ t('components.federation.LibraryWidget.empty.noMatch') }}

View File

@ -210,19 +210,24 @@ export default (props: PlayOptionsProps) => {
if (!id) { if (!id) {
throw new Error("Library id not found in response."); throw new Error("Library id not found in response.");
} }
const fetchResponse = await axios.post('federation/fetches', const fetchResponse = await axios.post(
{ object: id } 'federation/fetches',
{ object_uri: id },
{
headers: {
Accept: 'application/activity+json'
}
}
); );
const response = await axios.post( const response = await axios.post(
'federation/follows/library', 'federation/follows/library',
{ target: fetchResponse.data.object.uuid } { target: fetchResponse.data.object.id.split('/').pop() }
); );
return response; return response;
}; };
return { return {
playable, playable,
filterableArtist, filterableArtist,

View File

@ -2895,6 +2895,22 @@ export interface paths {
patch: operations["partial_update_upload_bulk_update"]; patch: operations["partial_update_upload_bulk_update"];
trace?: never; trace?: never;
}; };
"/api/v1/uploads/trigger-download/": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
post: operations["create_upload_trigger_download"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v1/users/{username}/": { "/api/v1/users/{username}/": {
parameters: { parameters: {
query?: never; query?: never;
@ -6019,6 +6035,22 @@ export interface paths {
patch: operations["partial_update_upload_bulk_update_2"]; patch: operations["partial_update_upload_bulk_update_2"];
trace?: never; trace?: never;
}; };
"/api/v2/uploads/trigger-download/": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
post: operations["create_upload_trigger_download_2"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/api/v2/users/{username}/": { "/api/v2/users/{username}/": {
parameters: { parameters: {
query?: never; query?: never;
@ -8472,6 +8504,7 @@ export interface components {
/** @default pending */ /** @default pending */
import_status: components["schemas"]["ImportStatusEnum"]; import_status: components["schemas"]["ImportStatusEnum"];
privacy_level?: components["schemas"]["LibraryPrivacyLevelEnum"]; privacy_level?: components["schemas"]["LibraryPrivacyLevelEnum"];
third_party_provider?: string | null;
import_metadata?: components["schemas"]["ImportMetadataRequest"]; import_metadata?: components["schemas"]["ImportMetadataRequest"];
import_reference?: string; import_reference?: string;
source?: string | null; source?: string | null;
@ -8893,6 +8926,7 @@ export interface components {
/** @default pending */ /** @default pending */
import_status: components["schemas"]["ImportStatusEnum"]; import_status: components["schemas"]["ImportStatusEnum"];
privacy_level?: components["schemas"]["LibraryPrivacyLevelEnum"]; privacy_level?: components["schemas"]["LibraryPrivacyLevelEnum"];
third_party_provider?: string | null;
readonly import_details: unknown; readonly import_details: unknown;
import_metadata?: components["schemas"]["ImportMetadata"]; import_metadata?: components["schemas"]["ImportMetadata"];
import_reference?: string; import_reference?: string;
@ -8907,6 +8941,7 @@ export interface components {
/** @default pending */ /** @default pending */
import_status: components["schemas"]["ImportStatusEnum"]; import_status: components["schemas"]["ImportStatusEnum"];
privacy_level?: components["schemas"]["LibraryPrivacyLevelEnum"]; privacy_level?: components["schemas"]["LibraryPrivacyLevelEnum"];
third_party_provider?: string | null;
import_metadata?: components["schemas"]["ImportMetadataRequest"]; import_metadata?: components["schemas"]["ImportMetadataRequest"];
import_reference?: string; import_reference?: string;
source?: string | null; source?: string | null;
@ -10590,6 +10625,7 @@ export interface operations {
[name: string]: unknown; [name: string]: unknown;
}; };
content: { content: {
"application/activity+json": components["schemas"]["Fetch"];
"application/json": components["schemas"]["Fetch"]; "application/json": components["schemas"]["Fetch"];
}; };
}; };
@ -10612,6 +10648,7 @@ export interface operations {
[name: string]: unknown; [name: string]: unknown;
}; };
content: { content: {
"application/activity+json": components["schemas"]["Fetch"];
"application/json": components["schemas"]["Fetch"]; "application/json": components["schemas"]["Fetch"];
}; };
}; };
@ -15756,6 +15793,32 @@ export interface operations {
}; };
}; };
}; };
create_upload_trigger_download: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["UploadForOwnerRequest"];
"application/x-www-form-urlencoded": components["schemas"]["UploadForOwnerRequest"];
"multipart/form-data": components["schemas"]["UploadForOwnerRequest"];
"application/activity+json": components["schemas"]["UploadForOwnerRequest"];
};
};
responses: {
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["UploadForOwner"];
};
};
};
};
update_user: { update_user: {
parameters: { parameters: {
query?: never; query?: never;
@ -17537,6 +17600,7 @@ export interface operations {
[name: string]: unknown; [name: string]: unknown;
}; };
content: { content: {
"application/activity+json": components["schemas"]["Fetch"];
"application/json": components["schemas"]["Fetch"]; "application/json": components["schemas"]["Fetch"];
}; };
}; };
@ -17559,6 +17623,7 @@ export interface operations {
[name: string]: unknown; [name: string]: unknown;
}; };
content: { content: {
"application/activity+json": components["schemas"]["Fetch"];
"application/json": components["schemas"]["Fetch"]; "application/json": components["schemas"]["Fetch"];
}; };
}; };
@ -22913,6 +22978,32 @@ export interface operations {
}; };
}; };
}; };
create_upload_trigger_download_2: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["UploadForOwnerRequest"];
"application/x-www-form-urlencoded": components["schemas"]["UploadForOwnerRequest"];
"multipart/form-data": components["schemas"]["UploadForOwnerRequest"];
"application/activity+json": components["schemas"]["UploadForOwnerRequest"];
};
};
responses: {
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["UploadForOwner"];
};
};
};
};
update_user_3: { update_user_3: {
parameters: { parameters: {
query?: never; query?: never;

View File

@ -492,7 +492,10 @@
"playTracks": "Play tracks", "playTracks": "Play tracks",
"report": "Report…", "report": "Report…",
"startRadio": "Play similar songs", "startRadio": "Play similar songs",
"trackDetails": "Track details" "trackDetails": "Track details",
"thirdPartyDownload": "Try to download from third-party service",
"thirdPartyDownloadTooltip": "This relies on installed plugins and may not work on all pods.",
"thirdPartyDownloadTriggered": "Download triggered, come back later"
}, },
"title": { "title": {
"more": "More…", "more": "More…",