From cdc617be27901403994009819e19a62179c7e316 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 18 Apr 2019 14:37:17 +0200 Subject: [PATCH] Logic to refetch remote entities --- api/funkwhale_api/federation/admin.py | 8 + .../federation/api_serializers.py | 16 ++ api/funkwhale_api/federation/api_urls.py | 1 + api/funkwhale_api/federation/api_views.py | 8 + api/funkwhale_api/federation/decorators.py | 49 ++++++ api/funkwhale_api/federation/factories.py | 10 +- api/funkwhale_api/federation/filters.py | 11 ++ .../federation/migrations/0018_fetch.py | 33 ++++ api/funkwhale_api/federation/models.py | 49 ++++++ api/funkwhale_api/federation/serializers.py | 4 + api/funkwhale_api/federation/tasks.py | 83 +++++++++ api/funkwhale_api/federation/utils.py | 10 ++ api/funkwhale_api/music/models.py | 8 +- api/funkwhale_api/music/views.py | 7 +- api/tests/federation/test_api_views.py | 12 ++ api/tests/federation/test_decorators.py | 83 +++++++++ api/tests/federation/test_models.py | 9 + api/tests/federation/test_tasks.py | 58 +++++++ .../src/components/federation/FetchButton.vue | 157 ++++++++++++++++++ front/src/components/semantic/Modal.vue | 1 + front/src/views/admin/library/AlbumDetail.vue | 8 + .../src/views/admin/library/ArtistDetail.vue | 8 + front/src/views/admin/library/TrackDetail.vue | 8 + 23 files changed, 632 insertions(+), 9 deletions(-) create mode 100644 api/funkwhale_api/federation/decorators.py create mode 100644 api/funkwhale_api/federation/migrations/0018_fetch.py create mode 100644 api/tests/federation/test_decorators.py create mode 100644 front/src/components/federation/FetchButton.vue diff --git a/api/funkwhale_api/federation/admin.py b/api/funkwhale_api/federation/admin.py index 8c9bbe31c..263af80cb 100644 --- a/api/funkwhale_api/federation/admin.py +++ b/api/funkwhale_api/federation/admin.py @@ -30,6 +30,14 @@ class DomainAdmin(admin.ModelAdmin): search_fields = ["name"] +@admin.register(models.Fetch) +class FetchAdmin(admin.ModelAdmin): + list_display = ["url", "actor", "status", "creation_date", "fetch_date", "detail"] + search_fields = ["url", "actor__username"] + list_filter = ["status"] + list_select_related = True + + @admin.register(models.Activity) class ActivityAdmin(admin.ModelAdmin): list_display = ["type", "fid", "url", "actor", "creation_date"] diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py index 9041ed28a..dbc655a47 100644 --- a/api/funkwhale_api/federation/api_serializers.py +++ b/api/funkwhale_api/federation/api_serializers.py @@ -144,3 +144,19 @@ class InboxItemActionSerializer(common_serializers.ActionSerializer): def handle_read(self, objects): return objects.update(is_read=True) + + +class FetchSerializer(serializers.ModelSerializer): + actor = federation_serializers.APIActorSerializer() + + class Meta: + model = models.Fetch + fields = [ + "id", + "url", + "actor", + "status", + "detail", + "creation_date", + "fetch_date", + ] diff --git a/api/funkwhale_api/federation/api_urls.py b/api/funkwhale_api/federation/api_urls.py index e1e451bff..bd2258de9 100644 --- a/api/funkwhale_api/federation/api_urls.py +++ b/api/funkwhale_api/federation/api_urls.py @@ -3,6 +3,7 @@ from rest_framework import routers from . import api_views router = routers.SimpleRouter() +router.register(r"fetches", api_views.FetchViewSet, "fetches") router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows") router.register(r"inbox", api_views.InboxItemViewSet, "inbox") router.register(r"libraries", api_views.LibraryViewSet, "libraries") diff --git a/api/funkwhale_api/federation/api_views.py b/api/funkwhale_api/federation/api_views.py index 0fe044ec1..5f6f50d8f 100644 --- a/api/funkwhale_api/federation/api_views.py +++ b/api/funkwhale_api/federation/api_views.py @@ -5,6 +5,7 @@ from django.db.models import Count from rest_framework import decorators from rest_framework import mixins +from rest_framework import permissions from rest_framework import response from rest_framework import viewsets @@ -189,3 +190,10 @@ class InboxItemViewSet( serializer.is_valid(raise_exception=True) result = serializer.save() return response.Response(result, status=200) + + +class FetchViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + + queryset = models.Fetch.objects.select_related("actor") + serializer_class = api_serializers.FetchSerializer + permission_classes = [permissions.IsAuthenticated] diff --git a/api/funkwhale_api/federation/decorators.py b/api/funkwhale_api/federation/decorators.py new file mode 100644 index 000000000..3d2d62567 --- /dev/null +++ b/api/funkwhale_api/federation/decorators.py @@ -0,0 +1,49 @@ +from django.db import transaction + +from rest_framework import decorators +from rest_framework import permissions +from rest_framework import response +from rest_framework import status + +from funkwhale_api.common import utils as common_utils + +from . import api_serializers +from . import filters +from . import models +from . import tasks +from . import utils + + +def fetches_route(): + @transaction.atomic + def fetches(self, request, *args, **kwargs): + obj = self.get_object() + if request.method == "GET": + queryset = models.Fetch.objects.get_for_object(obj).select_related("actor") + queryset = queryset.order_by("-creation_date") + filterset = filters.FetchFilter(request.GET, queryset=queryset) + page = self.paginate_queryset(filterset.qs) + if page is not None: + serializer = api_serializers.FetchSerializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = api_serializers.FetchSerializer(queryset, many=True) + return response.Response(serializer.data) + if request.method == "POST": + if utils.is_local(obj.fid): + return response.Response( + {"detail": "Cannot fetch a local object"}, status=400 + ) + + fetch = models.Fetch.objects.create( + url=obj.fid, actor=request.user.actor, object=obj + ) + common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk) + serializer = api_serializers.FetchSerializer(fetch) + return response.Response(serializer.data, status=status.HTTP_201_CREATED) + + return decorators.action( + methods=["get", "post"], + detail=True, + permission_classes=[permissions.IsAuthenticated], + )(fetches) diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index cf9546447..14bb4e8c9 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -166,7 +166,7 @@ class MusicLibraryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): @registry.register -class LibraryScan(NoUpdateOnCreate, factory.django.DjangoModelFactory): +class LibraryScanFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): library = factory.SubFactory(MusicLibraryFactory) actor = factory.SubFactory(ActorFactory) total_files = factory.LazyAttribute(lambda o: o.library.uploads_count) @@ -175,6 +175,14 @@ class LibraryScan(NoUpdateOnCreate, factory.django.DjangoModelFactory): model = "music.LibraryScan" +@registry.register +class FetchFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): + actor = factory.SubFactory(ActorFactory) + + class Meta: + model = "federation.Fetch" + + @registry.register class ActivityFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): actor = factory.SubFactory(ActorFactory) diff --git a/api/funkwhale_api/federation/filters.py b/api/funkwhale_api/federation/filters.py index 3a8b76cee..bfc48bcfb 100644 --- a/api/funkwhale_api/federation/filters.py +++ b/api/funkwhale_api/federation/filters.py @@ -46,3 +46,14 @@ class InboxItemFilter(django_filters.FilterSet): def filter_before(self, queryset, field_name, value): return queryset.filter(pk__lte=value) + + +class FetchFilter(django_filters.FilterSet): + ordering = django_filters.OrderingFilter( + # tuple-mapping retains order + fields=(("creation_date", "creation_date"), ("fetch_date", "fetch_date")) + ) + + class Meta: + model = models.Fetch + fields = ["status", "object_id", "url"] diff --git a/api/funkwhale_api/federation/migrations/0018_fetch.py b/api/funkwhale_api/federation/migrations/0018_fetch.py new file mode 100644 index 000000000..11789024f --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0018_fetch.py @@ -0,0 +1,33 @@ +# Generated by Django 2.1.7 on 2019-04-17 14:57 + +import django.contrib.postgres.fields.jsonb +import django.core.serializers.json +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import funkwhale_api.federation.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('federation', '0017_auto_20190130_0926'), + ] + + operations = [ + migrations.CreateModel( + name='Fetch', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.URLField(db_index=True, max_length=500)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('fetch_date', models.DateTimeField(blank=True, null=True)), + ('object_id', models.IntegerField(null=True)), + ('status', models.CharField(choices=[('pending', 'Pending'), ('errored', 'Errored'), ('finished', 'Finished'), ('skipped', 'Skipped')], default='pending', max_length=20)), + ('detail', django.contrib.postgres.fields.jsonb.JSONField(default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000)), + ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='fetches', to='federation.Actor')), + ('object_content_type', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + ), + ] diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index f465ea3ac..7d3d5639d 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -288,6 +288,55 @@ class Actor(models.Model): ) +FETCH_STATUSES = [ + ("pending", "Pending"), + ("errored", "Errored"), + ("finished", "Finished"), + ("skipped", "Skipped"), +] + + +class FetchQuerySet(models.QuerySet): + def get_for_object(self, object): + content_type = ContentType.objects.get_for_model(object) + return self.filter(object_content_type=content_type, object_id=object.pk) + + +class Fetch(models.Model): + url = models.URLField(max_length=500, db_index=True) + creation_date = models.DateTimeField(default=timezone.now) + fetch_date = models.DateTimeField(null=True, blank=True) + object_id = models.IntegerField(null=True) + object_content_type = models.ForeignKey( + ContentType, null=True, on_delete=models.CASCADE + ) + object = GenericForeignKey("object_content_type", "object_id") + status = models.CharField(default="pending", choices=FETCH_STATUSES, max_length=20) + detail = JSONField(default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder) + actor = models.ForeignKey(Actor, related_name="fetches", on_delete=models.CASCADE) + + objects = FetchQuerySet.as_manager() + + def save(self, **kwargs): + if not self.url and self.object: + self.url = self.object.fid + + super().save(**kwargs) + + @property + def serializers(self): + from . import contexts + from . import serializers + + return { + contexts.FW.Artist: serializers.ArtistSerializer, + contexts.FW.Album: serializers.AlbumSerializer, + contexts.FW.Track: serializers.TrackSerializer, + contexts.AS.Audio: serializers.UploadSerializer, + contexts.FW.Library: serializers.LibrarySerializer, + } + + class InboxItem(models.Model): """ Store activities binding to local actors, with read/unread status. diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index b32c09bdb..3e7618c9c 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -771,6 +771,7 @@ class ArtistSerializer(MusicEntitySerializer): ] class Meta: + model = music_models.Artist jsonld_mapping = MUSIC_ENTITY_JSONLD_MAPPING def to_representation(self, instance): @@ -804,6 +805,7 @@ class AlbumSerializer(MusicEntitySerializer): ] class Meta: + model = music_models.Album jsonld_mapping = funkwhale_utils.concat_dicts( MUSIC_ENTITY_JSONLD_MAPPING, { @@ -863,6 +865,7 @@ class TrackSerializer(MusicEntitySerializer): ] class Meta: + model = music_models.Track jsonld_mapping = funkwhale_utils.concat_dicts( MUSIC_ENTITY_JSONLD_MAPPING, { @@ -970,6 +973,7 @@ class UploadSerializer(jsonld.JsonLdSerializer): track = TrackSerializer(required=True) class Meta: + model = music_models.Upload jsonld_mapping = { "track": jsonld.first_obj(contexts.FW.track), "library": jsonld.first_id(contexts.FW.library), diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index 38e8eb677..7a1c7d92b 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -1,9 +1,11 @@ import datetime +import json import logging import os import requests from django.conf import settings +from django.db import transaction from django.db.models import Q, F from django.utils import timezone from dynamic_preferences.registries import global_preferences_registry @@ -16,6 +18,7 @@ from funkwhale_api.music import models as music_models from funkwhale_api.taskapp import celery from . import actors +from . import jsonld from . import keys from . import models, signing from . import serializers @@ -278,3 +281,83 @@ def rotate_actor_key(actor): actor.private_key = pair[0].decode() actor.public_key = pair[1].decode() actor.save(update_fields=["private_key", "public_key"]) + + +@celery.app.task(name="federation.fetch") +@transaction.atomic +@celery.require_instance( + models.Fetch.objects.filter(status="pending").select_related("actor"), "fetch" +) +def fetch(fetch): + actor = fetch.actor + auth = signing.get_auth(actor.private_key, actor.private_key_id) + + def error(code, **kwargs): + fetch.status = "errored" + fetch.fetch_date = timezone.now() + fetch.detail = {"error_code": code} + fetch.detail.update(kwargs) + fetch.save(update_fields=["fetch_date", "status", "detail"]) + + try: + response = session.get_session().get( + auth=auth, + url=fetch.url, + timeout=5, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, + headers={"Content-Type": "application/activity+json"}, + ) + logger.debug("Remote answered with %s", response.status_code) + response.raise_for_status() + except requests.exceptions.HTTPError as e: + return error("http", status_code=e.response.status_code if e.response else None) + except requests.exceptions.Timeout: + return error("timeout") + except requests.exceptions.ConnectionError as e: + return error("connection", message=str(e)) + except requests.RequestException as e: + return error("request", message=str(e)) + except Exception as e: + return error("unhandled", message=str(e)) + + try: + payload = response.json() + except json.decoder.JSONDecodeError: + return error("invalid_json") + + try: + doc = jsonld.expand(payload) + except ValueError: + return error("invalid_jsonld") + + try: + type = doc.get("@type", [])[0] + except IndexError: + return error("missing_jsonld_type") + try: + serializer_class = fetch.serializers[type] + model = serializer_class.Meta.model + except (KeyError, AttributeError): + fetch.status = "skipped" + fetch.fetch_date = timezone.now() + fetch.detail = {"reason": "unhandled_type", "type": type} + return fetch.save(update_fields=["fetch_date", "status", "detail"]) + try: + id = doc.get("@id") + except IndexError: + existing = None + else: + existing = model.objects.filter(fid=id).first() + + serializer = serializer_class(existing, data=payload) + if not serializer.is_valid(): + return error("validation", validation_errors=serializer.errors) + try: + serializer.save() + except Exception as e: + error("save", message=str(e)) + raise + + fetch.status = "finished" + fetch.fetch_date = timezone.now() + return fetch.save(update_fields=["fetch_date", "status"]) diff --git a/api/funkwhale_api/federation/utils.py b/api/funkwhale_api/federation/utils.py index 2bbfdf7fa..8f73c5735 100644 --- a/api/funkwhale_api/federation/utils.py +++ b/api/funkwhale_api/federation/utils.py @@ -121,3 +121,13 @@ def get_domain_query_from_url(domain, url_field="fid"): **{"{}__startswith".format(url_field): "https://{}/".format(domain)} ) return query + + +def is_local(url): + if not url: + return True + + d = settings.FEDERATION_HOSTNAME + return url.startswith("http://{}/".format(d)) or url.startswith( + "https://{}/".format(d) + ) diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index 4b166be7c..7ad88d45f 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -117,13 +117,7 @@ class APIModelMixin(models.Model): @property def is_local(self): - if not self.fid: - return True - - d = settings.FEDERATION_HOSTNAME - return self.fid.startswith("http://{}/".format(d)) or self.fid.startswith( - "https://{}/".format(d) - ) + return federation_utils.is_local(self.fid) @property def domain_name(self): diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index b6df22143..336a87ce0 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -22,6 +22,7 @@ from funkwhale_api.common import views as common_views from funkwhale_api.federation.authentication import SignatureAuthentication from funkwhale_api.federation import actors from funkwhale_api.federation import api_serializers as federation_api_serializers +from funkwhale_api.federation import decorators as federation_decorators from funkwhale_api.federation import routes from funkwhale_api.users.oauth import permissions as oauth_permissions @@ -70,6 +71,7 @@ class ArtistViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelV filterset_class = filters.ArtistFilter ordering_fields = ("id", "name", "creation_date") + fetches = federation_decorators.fetches_route() mutations = common_decorators.mutations_route(types=["update"]) def get_queryset(self): @@ -100,6 +102,7 @@ class AlbumViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelVi ordering_fields = ("creation_date", "release_date", "title") filterset_class = filters.AlbumFilter + fetches = federation_decorators.fetches_route() mutations = common_decorators.mutations_route(types=["update"]) def get_queryset(self): @@ -201,7 +204,7 @@ class TrackViewSet( "disc_number", "artist__name", ) - + fetches = federation_decorators.fetches_route() mutations = common_decorators.mutations_route(types=["update"]) def get_queryset(self): @@ -437,6 +440,8 @@ class UploadViewSet( "artist__name", ) + fetches = federation_decorators.fetches_route() + def get_queryset(self): qs = super().get_queryset() return qs.filter(library__actor=self.request.user.actor) diff --git a/api/tests/federation/test_api_views.py b/api/tests/federation/test_api_views.py index 75579d39a..c34c5e99a 100644 --- a/api/tests/federation/test_api_views.py +++ b/api/tests/federation/test_api_views.py @@ -167,3 +167,15 @@ def test_user_can_update_read_status_of_inbox_item(factories, logged_in_api_clie ii.refresh_from_db() assert ii.is_read is True + + +def test_can_detail_fetch(logged_in_api_client, factories): + fetch = factories["federation.Fetch"](url="http://test.object") + url = reverse("api:v1:federation:fetches-detail", kwargs={"pk": fetch.pk}) + + response = logged_in_api_client.get(url) + + expected = api_serializers.FetchSerializer(fetch).data + + assert response.status_code == 200 + assert response.data == expected diff --git a/api/tests/federation/test_decorators.py b/api/tests/federation/test_decorators.py new file mode 100644 index 000000000..fa50f5674 --- /dev/null +++ b/api/tests/federation/test_decorators.py @@ -0,0 +1,83 @@ +from rest_framework import viewsets + +from funkwhale_api.music import models as music_models + +from funkwhale_api.federation import api_serializers +from funkwhale_api.federation import decorators +from funkwhale_api.federation import models +from funkwhale_api.federation import tasks + + +class V(viewsets.ModelViewSet): + queryset = music_models.Track.objects.all() + fetches = decorators.fetches_route() + permission_classes = [] + + +def test_fetches_route_create(factories, api_request, mocker): + on_commit = mocker.patch("funkwhale_api.common.utils.on_commit") + user = factories["users.User"]() + actor = user.create_actor() + track = factories["music.Track"]() + view = V.as_view({"post": "fetches"}) + + request = api_request.post("/", format="json") + setattr(request, "user", user) + setattr(request, "session", {}) + response = view(request, pk=track.pk) + + assert response.status_code == 201 + + fetch = models.Fetch.objects.get_for_object(track).latest("id") + on_commit.assert_called_once_with(tasks.fetch.delay, fetch_id=fetch.pk) + + assert fetch.url == track.fid + assert fetch.object == track + assert fetch.status == "pending" + assert fetch.actor == actor + + expected = api_serializers.FetchSerializer(fetch).data + assert response.data == expected + + +def test_fetches_route_create_local(factories, api_request, mocker, settings): + user = factories["users.User"]() + user.create_actor() + track = factories["music.Track"]( + fid="https://{}/test".format(settings.FEDERATION_HOSTNAME) + ) + view = V.as_view({"post": "fetches"}) + + request = api_request.post("/", format="json") + setattr(request, "user", user) + setattr(request, "session", {}) + response = view(request, pk=track.pk) + + assert response.status_code == 400 + + +def test_fetches_route_list(factories, api_request, mocker): + user = factories["users.User"]() + user.create_actor() + track = factories["music.Track"]() + fetches = [ + factories["federation.Fetch"](object=track), + factories["federation.Fetch"](object=track), + ] + view = V.as_view({"get": "fetches"}) + + request = api_request.get("/", format="json") + setattr(request, "user", user) + setattr(request, "session", {}) + expected = { + "next": None, + "previous": None, + "count": 2, + "results": api_serializers.FetchSerializer(reversed(fetches), many=True).data, + } + + request = api_request.get("/") + response = view(request, pk=track.pk) + + assert response.status_code == 200 + assert response.data == expected diff --git a/api/tests/federation/test_models.py b/api/tests/federation/test_models.py index 68a457142..d6f862bb3 100644 --- a/api/tests/federation/test_models.py +++ b/api/tests/federation/test_models.py @@ -164,3 +164,12 @@ def test_actor_can_manage_domain_service_actor(mocker, factories): obj = mocker.Mock(fid="https://{}/hello".format(actor.domain_id)) assert actor.can_manage(obj) is True + + +def test_can_create_fetch_for_object(factories): + track = factories["music.Track"](fid="http://test.domain") + fetch = factories["federation.Fetch"](object=track) + assert fetch.url == "http://test.domain" + assert fetch.status == "pending" + assert fetch.detail == {} + assert fetch.object == track diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py index 4428484b9..5cfd228e5 100644 --- a/api/tests/federation/test_tasks.py +++ b/api/tests/federation/test_tasks.py @@ -5,6 +5,7 @@ import pytest from django.utils import timezone +from funkwhale_api.federation import jsonld from funkwhale_api.federation import models from funkwhale_api.federation import serializers from funkwhale_api.federation import tasks @@ -332,3 +333,60 @@ def test_rotate_actor_key(factories, settings, mocker): assert actor.public_key == "public" assert actor.private_key == "private" + + +def test_fetch_skipped(factories, r_mock): + url = "https://fetch.object" + fetch = factories["federation.Fetch"](url=url) + payload = {"@context": jsonld.get_default_context(), "type": "Unhandled"} + r_mock.get(url, json=payload) + + tasks.fetch(fetch_id=fetch.pk) + + fetch.refresh_from_db() + + assert fetch.status == "skipped" + assert fetch.detail["reason"] == "unhandled_type" + + +@pytest.mark.parametrize( + "r_mock_args, expected_error_code", + [ + ({"json": {"type": "Unhandled"}}, "invalid_jsonld"), + ({"json": {"@context": jsonld.get_default_context()}}, "invalid_jsonld"), + ({"text": "invalidjson"}, "invalid_json"), + ({"status_code": 404}, "http"), + ({"status_code": 500}, "http"), + ], +) +def test_fetch_errored(factories, r_mock_args, expected_error_code, r_mock): + url = "https://fetch.object" + fetch = factories["federation.Fetch"](url=url) + r_mock.get(url, **r_mock_args) + + tasks.fetch(fetch_id=fetch.pk) + + fetch.refresh_from_db() + + assert fetch.status == "errored" + assert fetch.detail["error_code"] == expected_error_code + + +def test_fetch_success(factories, r_mock, mocker): + artist = factories["music.Artist"]() + fetch = factories["federation.Fetch"](url=artist.fid) + payload = serializers.ArtistSerializer(artist).data + init = mocker.spy(serializers.ArtistSerializer, "__init__") + save = mocker.spy(serializers.ArtistSerializer, "save") + + r_mock.get(artist.fid, json=payload) + + tasks.fetch(fetch_id=fetch.pk) + + fetch.refresh_from_db() + payload["@context"].append("https://funkwhale.audio/ns") + assert fetch.status == "finished" + assert init.call_count == 1 + assert init.call_args[0][1] == artist + assert init.call_args[1]["data"] == payload + assert save.call_count == 1 diff --git a/front/src/components/federation/FetchButton.vue b/front/src/components/federation/FetchButton.vue new file mode 100644 index 000000000..5c7f4b66e --- /dev/null +++ b/front/src/components/federation/FetchButton.vue @@ -0,0 +1,157 @@ + + + diff --git a/front/src/components/semantic/Modal.vue b/front/src/components/semantic/Modal.vue index 75cc97a88..076b3e466 100644 --- a/front/src/components/semantic/Modal.vue +++ b/front/src/components/semantic/Modal.vue @@ -23,6 +23,7 @@ export default { if (this.control) { $(this.$el).modal('hide') } + $(this.$el).remove() }, methods: { initModal () { diff --git a/front/src/views/admin/library/AlbumDetail.vue b/front/src/views/admin/library/AlbumDetail.vue index 3215da101..b5d802d98 100644 --- a/front/src/views/admin/library/AlbumDetail.vue +++ b/front/src/views/admin/library/AlbumDetail.vue @@ -46,6 +46,10 @@ Open on MusicBrainz  + +   + Refresh from remote server  + Open remote profile  @@ -264,10 +268,14 @@