From 9e8983bb60e22a2859cc5b828ef4a8339e9e6708 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Wed, 8 Apr 2020 13:28:46 +0200 Subject: [PATCH] See #170: reel2bits compat --- api/funkwhale_api/federation/activity.py | 8 +- api/funkwhale_api/federation/jsonld.py | 18 +++- api/funkwhale_api/federation/renderers.py | 15 ++++ api/funkwhale_api/federation/routes.py | 24 +++--- api/funkwhale_api/federation/serializers.py | 34 ++++++-- api/funkwhale_api/federation/tasks.py | 30 ++++--- api/funkwhale_api/federation/views.py | 22 ++++- api/funkwhale_api/music/serializers.py | 2 +- api/tests/federation/test_activity.py | 22 +++++ api/tests/federation/test_routes.py | 14 +-- api/tests/federation/test_serializers.py | 12 ++- .../test_third_party_activitypub.py | 85 +++++++++++++++++++ api/tests/federation/test_views.py | 15 +++- 13 files changed, 248 insertions(+), 53 deletions(-) diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index 7d9d25a2d..7725953e6 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -164,14 +164,18 @@ def receive(activity, on_behalf_of, inbox_actor=None): ) return - local_to_recipients = get_actors_from_audience(activity.get("to", [])) + local_to_recipients = get_actors_from_audience( + serializer.validated_data.get("to", []) + ) local_to_recipients = local_to_recipients.local() local_to_recipients = local_to_recipients.values_list("pk", flat=True) local_to_recipients = list(local_to_recipients) if inbox_actor: local_to_recipients.append(inbox_actor.pk) - local_cc_recipients = get_actors_from_audience(activity.get("cc", [])) + local_cc_recipients = get_actors_from_audience( + serializer.validated_data.get("cc", []) + ) local_cc_recipients = local_cc_recipients.local() local_cc_recipients = local_cc_recipients.values_list("pk", flat=True) diff --git a/api/funkwhale_api/federation/jsonld.py b/api/funkwhale_api/federation/jsonld.py index 4a23ae418..f95b279f4 100644 --- a/api/funkwhale_api/federation/jsonld.py +++ b/api/funkwhale_api/federation/jsonld.py @@ -232,16 +232,18 @@ class JsonLdSerializer(serializers.Serializer): def __init__(self, *args, **kwargs): self.jsonld_expand = kwargs.pop("jsonld_expand", True) super().__init__(*args, **kwargs) + self.jsonld_context = [] def run_validation(self, data=empty): if data and data is not empty: + self.jsonld_context = data.get("@context", []) if self.context.get("expand", self.jsonld_expand): try: data = expand(data) - except ValueError: + except ValueError as e: raise serializers.ValidationError( - "{} is not a valid jsonld document".format(data) + "{} is not a valid jsonld document: {}".format(data, e) ) try: config = self.Meta.jsonld_mapping @@ -294,3 +296,15 @@ def first_obj(property, aliases=[]): def raw(property, aliases=[]): return {"property": property, "aliases": aliases} + + +def is_present_recursive(data, key): + if isinstance(data, (dict, list)): + for v in data: + if is_present_recursive(v, key): + return True + else: + if data == key: + return True + + return False diff --git a/api/funkwhale_api/federation/renderers.py b/api/funkwhale_api/federation/renderers.py index 5b58cf031..88004969d 100644 --- a/api/funkwhale_api/federation/renderers.py +++ b/api/funkwhale_api/federation/renderers.py @@ -1,3 +1,4 @@ +from rest_framework.negotiation import BaseContentNegotiation from rest_framework.renderers import JSONRenderer @@ -15,5 +16,19 @@ def get_ap_renderers(): ] +class IgnoreClientContentNegotiation(BaseContentNegotiation): + def select_parser(self, request, parsers): + """ + Select the first parser in the `.parser_classes` list. + """ + return parsers[0] + + def select_renderer(self, request, renderers, format_suffix): + """ + Select the first renderer in the `.renderer_classes` list. + """ + return (renderers[0], renderers[0].media_type) + + class WebfingerRenderer(JSONRenderer): media_type = "application/jrd+json" diff --git a/api/funkwhale_api/federation/routes.py b/api/funkwhale_api/federation/routes.py index 4177f76d5..4d32cb671 100644 --- a/api/funkwhale_api/federation/routes.py +++ b/api/funkwhale_api/federation/routes.py @@ -134,19 +134,19 @@ def outbox_follow(context): def outbox_create_audio(context): upload = context["upload"] channel = upload.library.get_channel() - upload_serializer = ( - serializers.ChannelUploadSerializer if channel else serializers.UploadSerializer - ) followers_target = channel.actor if channel else upload.library actor = channel.actor if channel else upload.library.actor - - serializer = serializers.ActivitySerializer( - { - "type": "Create", - "actor": actor.fid, - "object": upload_serializer(upload).data, - } - ) + if channel: + serializer = serializers.ChannelCreateUploadSerializer(upload) + else: + upload_serializer = serializers.UploadSerializer + serializer = serializers.ActivitySerializer( + { + "type": "Create", + "actor": actor.fid, + "object": upload_serializer(upload).data, + } + ) yield { "type": "Create", "actor": actor, @@ -163,7 +163,7 @@ def inbox_create_audio(payload, context): is_channel = "library" not in payload["object"] if is_channel: channel = context["actor"].get_channel() - serializer = serializers.ChannelUploadSerializer( + serializer = serializers.ChannelCreateUploadSerializer( data=payload["object"], context={"channel": channel}, ) else: diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 313f225f7..b1583f9ac 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -436,8 +436,8 @@ class ActorSerializer(jsonld.JsonLdSerializer): ) if rss_url: rss_url = rss_url["href"] - attributed_to = self.validated_data.get("attributedTo") - if rss_url and attributed_to: + attributed_to = self.validated_data.get("attributedTo", actor.fid) + if rss_url: # if the actor is attributed to another actor, and there is a RSS url, # then we consider it's a channel create_or_update_channel( @@ -533,6 +533,7 @@ class BaseActivitySerializer(serializers.Serializer): id = serializers.URLField(max_length=500, required=False) type = serializers.CharField(max_length=100) actor = serializers.URLField(max_length=500) + object = serializers.JSONField(required=False, allow_null=True) def validate_actor(self, v): expected = self.context.get("actor") @@ -555,17 +556,30 @@ class BaseActivitySerializer(serializers.Serializer): ) def validate(self, data): - data["recipients"] = self.validate_recipients(self.initial_data) + self.validate_recipients(data, self.initial_data) return super().validate(data) - def validate_recipients(self, payload): + def validate_recipients(self, data, payload): """ Ensure we have at least a to/cc field with valid actors """ - to = payload.get("to", []) - cc = payload.get("cc", []) + data["to"] = payload.get("to", []) + data["cc"] = payload.get("cc", []) - if not to and not cc and not self.context.get("recipients"): + if ( + not data["to"] + and data.get("type") in ["Follow", "Accept"] + and data.get("object") + ): + # there isn't always a to field for Accept/Follow + # in their follow activity, so we consider the recipient + # to be the follow object + if data["type"] == "Follow": + data["to"].append(str(data.get("object"))) + else: + data["to"].append(data.get("object", {}).get("actor")) + + if not data["to"] and not data["cc"] and not self.context.get("recipients"): raise serializers.ValidationError( "We cannot handle an activity with no recipient" ) @@ -1786,6 +1800,7 @@ class ChannelUploadSerializer(jsonld.JsonLdSerializer): content = TruncatedCharField( truncate_length=common_models.CONTENT_TEXT_MAX_LENGTH, required=False, + allow_blank=True, allow_null=True, ) @@ -1951,6 +1966,11 @@ class ChannelCreateUploadSerializer(jsonld.JsonLdSerializer): return { "@context": jsonld.get_default_context(), "type": "Create", + "id": utils.full_url( + reverse( + "federation:music:uploads-activity", kwargs={"uuid": upload.uuid} + ) + ), "actor": upload.library.channel.actor.fid, "object": ChannelUploadSerializer( upload, context={"include_ap_context": False} diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index b84e4c5f9..bbed4ce8e 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -404,19 +404,25 @@ def fetch(fetch_obj): if isinstance(obj, models.Actor) and obj.get_channel(): obj = obj.get_channel() if obj.actor.outbox_url: - # first page fetch is synchronous, so that at least some data is available - # in the UI after subscription - result = fetch_collection( - obj.actor.outbox_url, channel_id=obj.pk, max_pages=1, - ) - if result.get("next_page"): - # additional pages are fetched in the background - result = fetch_collection.delay( - result["next_page"], - channel_id=obj.pk, - max_pages=settings.FEDERATION_COLLECTION_MAX_PAGES - 1, - is_page=True, + try: + # first page fetch is synchronous, so that at least some data is available + # in the UI after subscription + result = fetch_collection( + obj.actor.outbox_url, channel_id=obj.pk, max_pages=1, ) + except Exception: + logger.exception( + "Error while fetching actor outbox: %s", obj.actor.outbox.url + ) + else: + if result.get("next_page"): + # additional pages are fetched in the background + result = fetch_collection.delay( + result["next_page"], + channel_id=obj.pk, + max_pages=settings.FEDERATION_COLLECTION_MAX_PAGES - 1, + is_page=True, + ) fetch_obj.object = obj fetch_obj.status = "finished" diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index cd8d0b45c..2a26555fa 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -52,7 +52,11 @@ class SharedViewSet(FederationMixin, viewsets.GenericViewSet): authentication_classes = [authentication.SignatureAuthentication] renderer_classes = renderers.get_ap_renderers() - @action(methods=["post"], detail=False) + @action( + methods=["post"], + detail=False, + content_negotiation_class=renderers.IgnoreClientContentNegotiation, + ) def inbox(self, request, *args, **kwargs): if request.method.lower() == "post" and request.actor is None: raise exceptions.AuthenticationFailed( @@ -88,7 +92,11 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV serializer = self.get_serializer(instance) return response.Response(serializer.data) - @action(methods=["get", "post"], detail=True) + @action( + methods=["get", "post"], + detail=True, + content_negotiation_class=renderers.IgnoreClientContentNegotiation, + ) def inbox(self, request, *args, **kwargs): inbox_actor = self.get_object() if request.method.lower() == "post" and request.actor is None: @@ -352,6 +360,16 @@ class MusicUploadViewSet( return serializers.ChannelUploadSerializer(obj) return super().get_serializer(obj) + @action( + methods=["get"], + detail=True, + content_negotiation_class=renderers.IgnoreClientContentNegotiation, + ) + def activity(self, request, *args, **kwargs): + object = self.get_object() + serializer = serializers.ChannelCreateUploadSerializer(object) + return response.Response(serializer.data) + class MusicArtistViewSet( FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index 756d18bfc..6abd6e262 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -659,7 +659,7 @@ class OembedSerializer(serializers.Serializer): if track.attachment_cover: data[ "thumbnail_url" - ] = track.album.attachment_cover.download_url_medium_square_crop + ] = track.attachment_cover.download_url_medium_square_crop data["thumbnail_width"] = 200 data["thumbnail_height"] = 200 elif track.album and track.album.attachment_cover: diff --git a/api/tests/federation/test_activity.py b/api/tests/federation/test_activity.py index 89a25b22a..2e4b62014 100644 --- a/api/tests/federation/test_activity.py +++ b/api/tests/federation/test_activity.py @@ -69,6 +69,28 @@ def test_receive_validates_basic_attributes_and_stores_activity( assert serializer_init.call_args[1]["data"] == a +def test_receive_uses_follow_object_if_no_audience_provided( + mrf_inbox_registry, factories, now, mocker +): + mocker.patch.object( + activity.InboxRouter, "get_matching_handlers", return_value=True + ) + mocker.patch("funkwhale_api.common.utils.on_commit") + local_to_actor = factories["users.User"]().create_actor() + remote_actor = factories["federation.Actor"]() + a = { + "@context": [], + "actor": remote_actor.fid, + "type": "Follow", + "id": "https://test.activity", + "object": local_to_actor.fid, + } + + activity.receive(activity=a, on_behalf_of=remote_actor, inbox_actor=None) + + assert models.InboxItem.objects.filter(actor=local_to_actor, type="to").exists() + + def test_receive_uses_mrf_returned_payload(mrf_inbox_registry, factories, now, mocker): mocker.patch.object( activity.InboxRouter, "get_matching_handlers", return_value=True diff --git a/api/tests/federation/test_routes.py b/api/tests/federation/test_routes.py index 63d8905ca..2da2bd238 100644 --- a/api/tests/federation/test_routes.py +++ b/api/tests/federation/test_routes.py @@ -305,13 +305,7 @@ def test_outbox_create_audio_channel(factories, mocker): channel = factories["audio.Channel"]() upload = factories["music.Upload"](library=channel.library) activity = list(routes.outbox_create_audio({"upload": upload}))[0] - serializer = serializers.ActivitySerializer( - { - "type": "Create", - "object": serializers.ChannelUploadSerializer(upload).data, - "actor": channel.actor.fid, - } - ) + serializer = serializers.ChannelCreateUploadSerializer(upload) expected = serializer.data expected["to"] = [{"type": "followers", "target": upload.library.channel.actor}] @@ -360,11 +354,11 @@ def test_inbox_create_audio_channel(factories, mocker): "@context": jsonld.get_default_context(), "type": "Create", "actor": channel.actor.fid, - "object": serializers.ChannelUploadSerializer(upload).data, + "object": serializers.ChannelCreateUploadSerializer(upload).data, } upload.delete() - init = mocker.spy(serializers.ChannelUploadSerializer, "__init__") - save = mocker.spy(serializers.ChannelUploadSerializer, "save") + init = mocker.spy(serializers.ChannelCreateUploadSerializer, "__init__") + save = mocker.spy(serializers.ChannelCreateUploadSerializer, "save") result = routes.inbox_create_audio( payload, context={"actor": channel.actor, "raise_exception": True, "activity": activity}, diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index bb66c9fc8..addf74b4a 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -3,6 +3,7 @@ import pytest import uuid from django.core.paginator import Paginator +from django.urls import reverse from django.utils import timezone from funkwhale_api.common import utils as common_utils @@ -1399,19 +1400,19 @@ def test_activity_serializer_validate_recipients_empty(db): s = serializers.BaseActivitySerializer() with pytest.raises(serializers.serializers.ValidationError): - s.validate_recipients({}) + s.validate_recipients({}, {}) with pytest.raises(serializers.serializers.ValidationError): - s.validate_recipients({"to": []}) + s.validate_recipients({"to": []}, {}) with pytest.raises(serializers.serializers.ValidationError): - s.validate_recipients({"cc": []}) + s.validate_recipients({"cc": []}, {}) def test_activity_serializer_validate_recipients_context(db): s = serializers.BaseActivitySerializer(context={"recipients": ["dummy"]}) - assert s.validate_recipients({}) is None + assert s.validate_recipients({}, {}) is None def test_track_serializer_update_license(factories): @@ -1879,6 +1880,9 @@ def test_channel_create_upload_serializer(factories): expected = { "@context": jsonld.get_default_context(), "type": "Create", + "id": utils.full_url( + reverse("federation:music:uploads-activity", kwargs={"uuid": upload.uuid}) + ), "actor": upload.library.channel.actor.fid, "object": serializers.ChannelUploadSerializer( upload, context={"include_ap_context": False} diff --git a/api/tests/federation/test_third_party_activitypub.py b/api/tests/federation/test_third_party_activitypub.py index 34b09c891..83377660a 100644 --- a/api/tests/federation/test_third_party_activitypub.py +++ b/api/tests/federation/test_third_party_activitypub.py @@ -56,3 +56,88 @@ def test_pleroma_actor_from_ap(factories): assert actor.private_key is None assert actor.public_key == payload["publicKey"]["publicKeyPem"] assert actor.domain_id == "test.federation" + + +def test_reel2bits_channel_from_actor_ap(db, mocker): + mocker.patch("funkwhale_api.federation.tasks.update_domain_nodeinfo") + payload = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "Hashtag": "as:Hashtag", + "PropertyValue": "schema:PropertyValue", + "artwork": "reel2bits:artwork", + "featured": "toot:featured", + "genre": "reel2bits:genre", + "licence": "reel2bits:licence", + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "reel2bits": "http://reel2bits.org/ns#", + "schema": "http://schema.org#", + "sensitive": "as:sensitive", + "tags": "reel2bits:tags", + "toot": "http://joinmastodon.org/ns#", + "transcode_url": "reel2bits:transcode_url", + "transcoded": "reel2bits:transcoded", + "value": "schema:value", + }, + ], + "endpoints": {"sharedInbox": "https://r2b.example/inbox"}, + "followers": "https://r2b.example/user/anna/followers", + "following": "https://r2b.example/user/anna/followings", + "icon": { + "type": "Image", + "url": "https://r2b.example/uploads/avatars/anna/f4930.jpg", + }, + "id": "https://r2b.example/user/anna", + "inbox": "https://r2b.example/user/anna/inbox", + "manuallyApprovesFollowers": False, + "name": "Anna", + "outbox": "https://r2b.example/user/anna/outbox", + "preferredUsername": "anna", + "publicKey": { + "id": "https://r2b.example/user/anna#main-key", + "owner": "https://r2b.example/user/anna", + "publicKeyPem": "MIIBIxaeikqh", + }, + "type": "Person", + "url": [ + { + "type": "Link", + "mediaType": "text/html", + "href": "https://r2b.example/@anna", + }, + { + "type": "Link", + "mediaType": "application/rss+xml", + "href": "https://r2b.example/@anna.rss", + }, + ], + } + + serializer = serializers.ActorSerializer(data=payload) + assert serializer.is_valid(raise_exception=True) + actor = serializer.save() + + assert actor.fid == payload["id"] + assert actor.url == payload["url"][0]["href"] + assert actor.inbox_url == payload["inbox"] + assert actor.shared_inbox_url == payload["endpoints"]["sharedInbox"] + assert actor.outbox_url is payload["outbox"] + assert actor.following_url == payload["following"] + assert actor.followers_url == payload["followers"] + assert actor.followers_url == payload["followers"] + assert actor.type == payload["type"] + assert actor.preferred_username == payload["preferredUsername"] + assert actor.name == payload["name"] + assert actor.manually_approves_followers is payload["manuallyApprovesFollowers"] + assert actor.private_key is None + assert actor.public_key == payload["publicKey"]["publicKeyPem"] + assert actor.domain_id == "r2b.example" + + channel = actor.get_channel() + + assert channel.attributed_to == actor + assert channel.rss_url == payload["url"][1]["href"] + assert channel.artist.name == actor.name + assert channel.artist.attributed_to == actor diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index 1f67f6f27..10da31b3c 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -260,7 +260,7 @@ def test_channel_outbox_retrieve_page(factories, api_client): def test_channel_upload_retrieve(factories, api_client): channel = factories["audio.Channel"](local=True) upload = factories["music.Upload"](library=channel.library, playable=True) - url = reverse("federation:music:uploads-detail", kwargs={"uuid": upload.uuid},) + url = reverse("federation:music:uploads-detail", kwargs={"uuid": upload.uuid}) expected = serializers.ChannelUploadSerializer(upload).data @@ -270,6 +270,19 @@ def test_channel_upload_retrieve(factories, api_client): assert response.data == expected +def test_channel_upload_retrieve_activity(factories, api_client): + channel = factories["audio.Channel"](local=True) + upload = factories["music.Upload"](library=channel.library, playable=True) + url = reverse("federation:music:uploads-activity", kwargs={"uuid": upload.uuid}) + + expected = serializers.ChannelCreateUploadSerializer(upload).data + + response = api_client.get(url) + + assert response.status_code == 200 + assert response.data == expected + + @pytest.mark.parametrize("privacy_level", ["me", "instance"]) def test_music_library_retrieve_page_private(factories, api_client, privacy_level): library = factories["music.Library"](privacy_level=privacy_level, actor__local=True)