See #170: fetching remote objects

api/config/settings/common.py

@@ -833,6 +833,10 @@ THROTTLING_RATES = {
         "rate": THROTTLING_USER_RATES.get("password-reset-confirm", "20/h"),
         "description": "Password reset confirmation",
     },
+    "fetch": {
+        "rate": THROTTLING_USER_RATES.get("fetch", "200/d"),
+        "description": "Fetch remote objects",
+    },
 }
 
 
@@ -906,7 +910,7 @@ ACCOUNT_USERNAME_BLACKLIST = [
 ] + env.list("ACCOUNT_USERNAME_BLACKLIST", default=[])
 
 EXTERNAL_REQUESTS_VERIFY_SSL = env.bool("EXTERNAL_REQUESTS_VERIFY_SSL", default=True)
-EXTERNAL_REQUESTS_TIMEOUT = env.int("EXTERNAL_REQUESTS_TIMEOUT", default=5)
+EXTERNAL_REQUESTS_TIMEOUT = env.int("EXTERNAL_REQUESTS_TIMEOUT", default=10)
 # XXX: deprecated, see #186
 API_AUTHENTICATION_REQUIRED = env.bool("API_AUTHENTICATION_REQUIRED", True)
 
@@ -955,7 +959,11 @@ FEDERATION_OBJECT_FETCH_DELAY = env.int(
 MODERATION_EMAIL_NOTIFICATIONS_ENABLED = env.bool(
     "MODERATION_EMAIL_NOTIFICATIONS_ENABLED", default=True
 )
-
+FEDERATION_AUTHENTIFY_FETCHES = True
+FEDERATION_SYNCHRONOUS_FETCH = env.bool("FEDERATION_SYNCHRONOUS_FETCH", default=True)
+FEDERATION_DUPLICATE_FETCH_DELAY = env.int(
+    "FEDERATION_DUPLICATE_FETCH_DELAY", default=60 * 50
+)
 # Delay in days after signup before we show the "support us" messages
 INSTANCE_SUPPORT_MESSAGE_DELAY = env.int("INSTANCE_SUPPORT_MESSAGE_DELAY", default=15)
 FUNKWHALE_SUPPORT_MESSAGE_DELAY = env.int("FUNKWHALE_SUPPORT_MESSAGE_DELAY", default=15)

api/funkwhale_api/common/utils.py

@@ -234,9 +234,11 @@ def get_updated_fields(conf, data, obj):
             data_value = data[data_field]
         except KeyError:
             continue
-
+        if obj.pk:
-        obj_value = getattr(obj, obj_field)
+            obj_value = getattr(obj, obj_field)
-        if obj_value != data_value:
+            if obj_value != data_value:
+                final_data[obj_field] = data_value
+        else:
             final_data[obj_field] = data_value
 
     return final_data

api/funkwhale_api/federation/api_serializers.py

@@ -1,7 +1,13 @@
+import datetime
+
+from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
+from django.core import validators
+from django.utils import timezone
 
 from rest_framework import serializers
 
+from funkwhale_api.common import fields as common_fields
 from funkwhale_api.common import serializers as common_serializers
 from funkwhale_api.music import models as music_models
 from funkwhale_api.users import serializers as users_serializers
@@ -158,8 +164,21 @@ class InboxItemActionSerializer(common_serializers.ActionSerializer):
         return objects.update(is_read=True)
 
 
+FETCH_OBJECT_CONFIG = {
+    "artist": {"queryset": music_models.Artist.objects.all()},
+    "album": {"queryset": music_models.Album.objects.all()},
+    "track": {"queryset": music_models.Track.objects.all()},
+    "library": {"queryset": music_models.Library.objects.all(), "id_attr": "uuid"},
+    "upload": {"queryset": music_models.Upload.objects.all(), "id_attr": "uuid"},
+    "account": {"queryset": models.Actor.objects.all(), "id_attr": "full_username"},
+}
+FETCH_OBJECT_FIELD = common_fields.GenericRelation(FETCH_OBJECT_CONFIG)
+
+
 class FetchSerializer(serializers.ModelSerializer):
-    actor = federation_serializers.APIActorSerializer()
+    actor = federation_serializers.APIActorSerializer(read_only=True)
+    object = serializers.CharField(write_only=True)
+    force = serializers.BooleanField(default=False, required=False, write_only=True)
 
     class Meta:
         model = models.Fetch
@@ -171,7 +190,63 @@ class FetchSerializer(serializers.ModelSerializer):
             "detail",
             "creation_date",
             "fetch_date",
+            "object",
+            "force",
         ]
+        read_only_fields = [
+            "id",
+            "url",
+            "actor",
+            "status",
+            "detail",
+            "creation_date",
+            "fetch_date",
+        ]
+
+    def validate_object(self, value):
+        # if value is a webginfer lookup, we craft a special url
+        if value.startswith("@"):
+            value = value.lstrip("@")
+        validator = validators.EmailValidator()
+        try:
+            validator(value)
+        except validators.ValidationError:
+            return value
+
+        return "webfinger://{}".format(value)
+
+    def create(self, validated_data):
+        check_duplicates = not validated_data.get("force", False)
+        if check_duplicates:
+            # first we check for duplicates
+            duplicate = (
+                validated_data["actor"]
+                .fetches.filter(
+                    status="finished",
+                    url=validated_data["object"],
+                    creation_date__gte=timezone.now()
+                    - datetime.timedelta(
+                        seconds=settings.FEDERATION_DUPLICATE_FETCH_DELAY
+                    ),
+                )
+                .order_by("-creation_date")
+                .first()
+            )
+            if duplicate:
+                return duplicate
+
+        fetch = models.Fetch.objects.create(
+            actor=validated_data["actor"], url=validated_data["object"]
+        )
+        return fetch
+
+    def to_representation(self, obj):
+        repr = super().to_representation(obj)
+        object_data = None
+        if obj.object:
+            object_data = FETCH_OBJECT_FIELD.to_representation(obj.object)
+        repr["object"] = object_data
+        return repr
 
 
 class FullActorSerializer(serializers.Serializer):

api/funkwhale_api/federation/api_views.py

@@ -1,5 +1,6 @@
 import requests.exceptions
 
+from django.conf import settings
 from django.db import transaction
 from django.db.models import Count
 
@@ -10,6 +11,7 @@ from rest_framework import response
 from rest_framework import viewsets
 
 from funkwhale_api.common import preferences
+from funkwhale_api.common import utils as common_utils
 from funkwhale_api.common.permissions import ConditionalAuthentication
 from funkwhale_api.music import models as music_models
 from funkwhale_api.music import views as music_views
@@ -22,6 +24,7 @@ from . import filters
 from . import models
 from . import routes
 from . import serializers
+from . import tasks
 from . import utils
 
 
@@ -195,11 +198,28 @@ class InboxItemViewSet(
         return response.Response(result, status=200)
 
 
-class FetchViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+class FetchViewSet(
+    mixins.CreateModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+):
 
     queryset = models.Fetch.objects.select_related("actor")
     serializer_class = api_serializers.FetchSerializer
     permission_classes = [permissions.IsAuthenticated]
+    throttling_scopes = {"create": {"authenticated": "fetch"}}
+
+    def get_queryset(self):
+        return super().get_queryset().filter(actor=self.request.user.actor)
+
+    def perform_create(self, serializer):
+        fetch = serializer.save(actor=self.request.user.actor)
+        if fetch.status == "finished":
+            # a duplicate was returned, no need to fetch again
+            return
+        if settings.FEDERATION_SYNCHRONOUS_FETCH:
+            tasks.fetch(fetch_id=fetch.pk)
+            fetch.refresh_from_db()
+        else:
+            common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk)
 
 
 class DomainViewSet(

api/funkwhale_api/federation/factories.py

@@ -21,7 +21,7 @@ class SignatureAuthFactory(factory.Factory):
     key = factory.LazyFunction(lambda: keys.get_key_pair()[0])
     key_id = factory.Faker("url")
     use_auth_header = False
-    headers = ["(request-target)", "user-agent", "host", "date", "content-type"]
+    headers = ["(request-target)", "user-agent", "host", "date", "accept"]
 
     class Meta:
         model = requests_http_signature.HTTPSignatureAuth
@@ -42,7 +42,7 @@ class SignedRequestFactory(factory.Factory):
             "User-Agent": "Test",
             "Host": "test.host",
             "Date": http_date(timezone.now().timestamp()),
-            "Content-Type": "application/activity+json",
+            "Accept": "application/activity+json",
         }
         if extracted:
             default_headers.update(extracted)

api/funkwhale_api/federation/library.py

@@ -9,9 +9,7 @@ def get_library_data(library_url, actor):
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
     try:
         response = session.get_session().get(
-            library_url,
+            library_url, auth=auth, headers={"Accept": "application/activity+json"},
-            auth=auth,
-            headers={"Content-Type": "application/activity+json"},
         )
     except requests.ConnectionError:
         return {"errors": ["This library is not reachable"]}
@@ -32,7 +30,7 @@ def get_library_data(library_url, actor):
 def get_library_page(library, page_url, actor):
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
     response = session.get_session().get(
-        page_url, auth=auth, headers={"Content-Type": "application/activity+json"},
+        page_url, auth=auth, headers={"Accept": "application/activity+json"},
     )
     serializer = serializers.CollectionPageSerializer(
         data=response.json(),

api/funkwhale_api/federation/models.py

@@ -372,7 +372,7 @@ class Fetch(models.Model):
     objects = FetchQuerySet.as_manager()
 
     def save(self, **kwargs):
-        if not self.url and self.object:
+        if not self.url and self.object and hasattr(self.object, "fid"):
             self.url = self.object.fid
 
         super().save(**kwargs)
@@ -388,6 +388,11 @@ class Fetch(models.Model):
             contexts.FW.Track: serializers.TrackSerializer,
             contexts.AS.Audio: serializers.UploadSerializer,
             contexts.FW.Library: serializers.LibrarySerializer,
+            contexts.AS.Group: serializers.ActorSerializer,
+            contexts.AS.Person: serializers.ActorSerializer,
+            contexts.AS.Organization: serializers.ActorSerializer,
+            contexts.AS.Service: serializers.ActorSerializer,
+            contexts.AS.Application: serializers.ActorSerializer,
         }
 
 
@@ -568,7 +573,7 @@ class LibraryTrack(models.Model):
             auth=auth,
             stream=True,
             timeout=20,
-            headers={"Content-Type": "application/activity+json"},
+            headers={"Accept": "application/activity+json"},
         )
         with remote_response as r:
             remote_response.raise_for_status()

api/funkwhale_api/federation/serializers.py

@@ -151,6 +151,10 @@ class ActorSerializer(jsonld.JsonLdSerializer):
     )
 
     class Meta:
+        # not strictly necessary because it's not a model serializer
+        # but used by tasks.py/fetch
+        model = models.Actor
+
         jsonld_mapping = {
             "outbox": jsonld.first_id(contexts.AS.outbox),
             "inbox": jsonld.first_id(contexts.LDP.inbox),
@@ -765,6 +769,10 @@ class LibrarySerializer(PaginatedCollectionSerializer):
     )
 
     class Meta:
+        # not strictly necessary because it's not a model serializer
+        # but used by tasks.py/fetch
+        model = music_models.Library
+
         jsonld_mapping = common_utils.concat_dicts(
             PAGINATED_COLLECTION_JSONLD_MAPPING,
             {
@@ -795,12 +803,15 @@ class LibrarySerializer(PaginatedCollectionSerializer):
         return r
 
     def create(self, validated_data):
-        actor = utils.retrieve_ap_object(
+        if self.instance:
-            validated_data["attributedTo"],
+            actor = self.instance.actor
-            actor=self.context.get("fetch_actor"),
+        else:
-            queryset=models.Actor,
+            actor = utils.retrieve_ap_object(
-            serializer_class=ActorSerializer,
+                validated_data["attributedTo"],
-        )
+                actor=self.context.get("fetch_actor"),
+                queryset=models.Actor,
+                serializer_class=ActorSerializer,
+            )
         privacy = {"": "me", "./": "me", None: "me", contexts.AS.Public: "everyone"}
         library, created = music_models.Library.objects.update_or_create(
             fid=validated_data["id"],
@@ -815,6 +826,9 @@ class LibrarySerializer(PaginatedCollectionSerializer):
         )
         return library
 
+    def update(self, instance, validated_data):
+        return self.create(validated_data)
+
 
 class CollectionPageSerializer(jsonld.JsonLdSerializer):
     type = serializers.ChoiceField(choices=[contexts.AS.CollectionPage])
@@ -968,8 +982,13 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
         allow_null=True,
     )
 
-    @transaction.atomic
     def update(self, instance, validated_data):
+        return self.update_or_create(validated_data)
+
+    @transaction.atomic
+    def update_or_create(self, validated_data):
+        instance = self.instance or self.Meta.model(fid=validated_data["id"])
+        creating = instance.pk is None
         attributed_to_fid = validated_data.get("attributedTo")
         if attributed_to_fid:
             validated_data["attributedTo"] = actors.get_actor(attributed_to_fid)
@@ -977,8 +996,11 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
             self.updateable_fields, validated_data, instance
         )
         updated_fields = self.validate_updated_data(instance, updated_fields)
-
+        if creating:
-        if updated_fields:
+            instance, created = self.Meta.model.objects.get_or_create(
+                fid=validated_data["id"], defaults=updated_fields
+            )
+        else:
             music_tasks.update_library_entity(instance, updated_fields)
 
         tags = [t["name"] for t in validated_data.get("tags", []) or []]
@@ -1064,6 +1086,8 @@ class ArtistSerializer(MusicEntitySerializer):
             d["@context"] = jsonld.get_default_context()
         return d
 
+    create = MusicEntitySerializer.update_or_create
+
 
 class AlbumSerializer(MusicEntitySerializer):
     released = serializers.DateField(allow_null=True, required=False)
@@ -1074,10 +1098,11 @@ class AlbumSerializer(MusicEntitySerializer):
     )
     updateable_fields = [
         ("name", "title"),
+        ("cover", "attachment_cover"),
         ("musicbrainzId", "mbid"),
         ("attributedTo", "attributed_to"),
         ("released", "release_date"),
-        ("cover", "attachment_cover"),
+        ("_artist", "artist"),
     ]
 
     class Meta:
@@ -1124,6 +1149,20 @@ class AlbumSerializer(MusicEntitySerializer):
             d["@context"] = jsonld.get_default_context()
         return d
 
+    def validate(self, data):
+        validated_data = super().validate(data)
+        if not self.parent:
+            validated_data["_artist"] = utils.retrieve_ap_object(
+                validated_data["artists"][0]["id"],
+                actor=self.context.get("fetch_actor"),
+                queryset=music_models.Artist,
+                serializer_class=ArtistSerializer,
+            )
+
+        return validated_data
+
+    create = MusicEntitySerializer.update_or_create
+
 
 class TrackSerializer(MusicEntitySerializer):
     position = serializers.IntegerField(min_value=0, allow_null=True, required=False)
@@ -1293,39 +1332,66 @@ class UploadSerializer(jsonld.JsonLdSerializer):
             return lb
 
         actor = self.context.get("actor")
-        kwargs = {}
+
-        if actor:
-            kwargs["actor"] = actor
         try:
-            return music_models.Library.objects.get(fid=v, **kwargs)
+            library = utils.retrieve_ap_object(
-        except music_models.Library.DoesNotExist:
+                v,
+                actor=self.context.get("fetch_actor"),
+                queryset=music_models.Library,
+                serializer_class=LibrarySerializer,
+            )
+        except Exception:
             raise serializers.ValidationError("Invalid library")
+        if actor and library.actor != actor:
+            raise serializers.ValidationError("Invalid library")
+        return library
 
+    def update(self, instance, validated_data):
+        return self.create(validated_data)
+
+    @transaction.atomic
     def create(self, validated_data):
-        try:
+        instance = self.instance or None
-            return music_models.Upload.objects.get(fid=validated_data["id"])
+        if not self.instance:
-        except music_models.Upload.DoesNotExist:
+            try:
-            pass
+                instance = music_models.Upload.objects.get(fid=validated_data["id"])
+            except music_models.Upload.DoesNotExist:
+                pass
 
-        track = TrackSerializer(
+        if instance:
-            context={"activity": self.context.get("activity")}
+            data = {
-        ).create(validated_data["track"])
+                "mimetype": validated_data["url"]["mediaType"],
+                "source": validated_data["url"]["href"],
+                "creation_date": validated_data["published"],
+                "modification_date": validated_data.get("updated"),
+                "duration": validated_data["duration"],
+                "size": validated_data["size"],
+                "bitrate": validated_data["bitrate"],
+                "import_status": "finished",
+            }
+            return music_models.Upload.objects.update_or_create(
+                fid=validated_data["id"], defaults=data
+            )[0]
+        else:
+            track = TrackSerializer(
+                context={"activity": self.context.get("activity")}
+            ).create(validated_data["track"])
 
-        data = {
+            data = {
-            "fid": validated_data["id"],
+                "fid": validated_data["id"],
-            "mimetype": validated_data["url"]["mediaType"],
+                "mimetype": validated_data["url"]["mediaType"],
-            "source": validated_data["url"]["href"],
+                "source": validated_data["url"]["href"],
-            "creation_date": validated_data["published"],
+                "creation_date": validated_data["published"],
-            "modification_date": validated_data.get("updated"),
+                "modification_date": validated_data.get("updated"),
-            "track": track,
+                "track": track,
-            "duration": validated_data["duration"],
+                "duration": validated_data["duration"],
-            "size": validated_data["size"],
+                "size": validated_data["size"],
-            "bitrate": validated_data["bitrate"],
+                "bitrate": validated_data["bitrate"],
-            "library": validated_data["library"],
+                "library": validated_data["library"],
-            "from_activity": self.context.get("activity"),
+                "from_activity": self.context.get("activity"),
-            "import_status": "finished",
+                "import_status": "finished",
-        }
+            }
-        return music_models.Upload.objects.create(**data)
+            return music_models.Upload.objects.create(**data)
 
     def to_representation(self, instance):
         track = instance.track

api/funkwhale_api/federation/tasks.py

@@ -14,6 +14,7 @@ from requests.exceptions import RequestException
 from funkwhale_api.common import preferences
 from funkwhale_api.common import session
 from funkwhale_api.common import utils as common_utils
+from funkwhale_api.moderation import mrf
 from funkwhale_api.music import models as music_models
 from funkwhale_api.taskapp import celery
 
@@ -24,6 +25,7 @@ from . import models, signing
 from . import serializers
 from . import routes
 from . import utils
+from . import webfinger
 
 logger = logging.getLogger(__name__)
 
@@ -285,24 +287,45 @@ def rotate_actor_key(actor):
 @celery.app.task(name="federation.fetch")
 @transaction.atomic
 @celery.require_instance(
-    models.Fetch.objects.filter(status="pending").select_related("actor"), "fetch"
+    models.Fetch.objects.filter(status="pending").select_related("actor"),
+    "fetch_obj",
+    "fetch_id",
 )
-def fetch(fetch):
+def fetch(fetch_obj):
-    actor = fetch.actor
-    auth = signing.get_auth(actor.private_key, actor.private_key_id)
-
     def error(code, **kwargs):
-        fetch.status = "errored"
+        fetch_obj.status = "errored"
-        fetch.fetch_date = timezone.now()
+        fetch_obj.fetch_date = timezone.now()
-        fetch.detail = {"error_code": code}
+        fetch_obj.detail = {"error_code": code}
-        fetch.detail.update(kwargs)
+        fetch_obj.detail.update(kwargs)
-        fetch.save(update_fields=["fetch_date", "status", "detail"])
+        fetch_obj.save(update_fields=["fetch_date", "status", "detail"])
 
+    url = fetch_obj.url
+    mrf_check_url = url
+    if not mrf_check_url.startswith("webfinger://"):
+        payload, updated = mrf.inbox.apply({"id": mrf_check_url})
+        if not payload:
+            return error("blocked", message="Blocked by MRF")
+
+    actor = fetch_obj.actor
+    if settings.FEDERATION_AUTHENTIFY_FETCHES:
+        auth = signing.get_auth(actor.private_key, actor.private_key_id)
+    else:
+        auth = None
     try:
+        if url.startswith("webfinger://"):
+            # we first grab the correpsonding webfinger representation
+            # to get the ActivityPub actor ID
+            webfinger_data = webfinger.get_resource(
+                "acct:" + url.replace("webfinger://", "")
+            )
+            url = webfinger.get_ap_url(webfinger_data["links"])
+            if not url:
+                return error("webfinger", message="Invalid or missing webfinger data")
+            payload, updated = mrf.inbox.apply({"id": url})
+            if not payload:
+                return error("blocked", message="Blocked by MRF")
         response = session.get_session().get(
-            auth=auth,
+            auth=auth, url=url, headers={"Accept": "application/activity+json"},
-            url=fetch.url,
-            headers={"Content-Type": "application/activity+json"},
         )
         logger.debug("Remote answered with %s", response.status_code)
         response.raise_for_status()
@@ -320,8 +343,19 @@ def fetch(fetch):
     try:
         payload = response.json()
     except json.decoder.JSONDecodeError:
+        # we attempt to extract a <link rel=alternate> that points
+        # to an activity pub resource, if possible, and retry with this URL
+        alternate_url = utils.find_alternate(response.text)
+        if alternate_url:
+            fetch_obj.url = alternate_url
+            fetch_obj.save(update_fields=["url"])
+            return fetch(fetch_id=fetch_obj.pk)
         return error("invalid_json")
 
+    payload, updated = mrf.inbox.apply(payload)
+    if not payload:
+        return error("blocked", message="Blocked by MRF")
+
     try:
         doc = jsonld.expand(payload)
     except ValueError:
@@ -332,13 +366,13 @@ def fetch(fetch):
     except IndexError:
         return error("missing_jsonld_type")
     try:
-        serializer_class = fetch.serializers[type]
+        serializer_class = fetch_obj.serializers[type]
         model = serializer_class.Meta.model
     except (KeyError, AttributeError):
-        fetch.status = "skipped"
+        fetch_obj.status = "skipped"
-        fetch.fetch_date = timezone.now()
+        fetch_obj.fetch_date = timezone.now()
-        fetch.detail = {"reason": "unhandled_type", "type": type}
+        fetch_obj.detail = {"reason": "unhandled_type", "type": type}
-        return fetch.save(update_fields=["fetch_date", "status", "detail"])
+        return fetch_obj.save(update_fields=["fetch_date", "status", "detail"])
     try:
         id = doc.get("@id")
     except IndexError:
@@ -350,11 +384,14 @@ def fetch(fetch):
     if not serializer.is_valid():
         return error("validation", validation_errors=serializer.errors)
     try:
-        serializer.save()
+        obj = serializer.save()
     except Exception as e:
         error("save", message=str(e))
         raise
 
-    fetch.status = "finished"
+    fetch_obj.object = obj
-    fetch.fetch_date = timezone.now()
+    fetch_obj.status = "finished"
-    return fetch.save(update_fields=["fetch_date", "status"])
+    fetch_obj.fetch_date = timezone.now()
+    return fetch_obj.save(
+        update_fields=["fetch_date", "status", "object_id", "object_content_type"]
+    )

api/funkwhale_api/federation/utils.py

@@ -1,3 +1,4 @@
+import html.parser
 import unicodedata
 import re
 from django.conf import settings
@@ -164,3 +165,39 @@ def get_actor_from_username_data_query(field, data):
                 "domain__name__iexact": data["domain"],
             }
         )
+
+
+class StopParsing(Exception):
+    pass
+
+
+class AlternateLinkParser(html.parser.HTMLParser):
+    def __init__(self, *args, **kwargs):
+        self.result = None
+        super().__init__(*args, **kwargs)
+
+    def handle_starttag(self, tag, attrs):
+        if tag != "link":
+            return
+
+        attrs_dict = dict(attrs)
+        if attrs_dict.get("rel") == "alternate" and attrs_dict.get(
+            "type", "application/activity+json"
+        ):
+            self.result = attrs_dict.get("href")
+            raise StopParsing()
+
+    def handle_endtag(self, tag):
+        if tag == "head":
+            raise StopParsing()
+
+
+def find_alternate(response_text):
+    if not response_text:
+        return
+
+    parser = AlternateLinkParser()
+    try:
+        parser.feed(response_text)
+    except StopParsing:
+        return parser.result

api/funkwhale_api/federation/webfinger.py

@@ -46,3 +46,12 @@ def get_resource(resource_string):
     serializer = serializers.ActorWebfingerSerializer(data=response.json())
     serializer.is_valid(raise_exception=True)
     return serializer.validated_data
+
+
+def get_ap_url(links):
+    for link in links:
+        if (
+            link.get("rel") == "self"
+            and link.get("type") == "application/activity+json"
+        ):
+            return link["href"]

api/funkwhale_api/moderation/management/commands/mrf_check.py

@@ -82,7 +82,7 @@ class Command(BaseCommand):
             content = models.Activity.objects.get(uuid=input).payload
         elif is_url(input):
             response = session.get_session().get(
-                input, headers={"Content-Type": "application/activity+json"},
+                input, headers={"Accept": "application/activity+json"},
             )
             response.raise_for_status()
             content = response.json()

api/funkwhale_api/music/serializers.py

@@ -324,6 +324,7 @@ class TrackSerializer(OptionalDescriptionMixin, serializers.Serializer):
 class LibraryForOwnerSerializer(serializers.ModelSerializer):
     uploads_count = serializers.SerializerMethodField()
     size = serializers.SerializerMethodField()
+    actor = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Library
@@ -336,6 +337,7 @@ class LibraryForOwnerSerializer(serializers.ModelSerializer):
             "uploads_count",
             "size",
             "creation_date",
+            "actor",
         ]
         read_only_fields = ["fid", "uuid", "creation_date", "actor"]
 
@@ -350,6 +352,12 @@ class LibraryForOwnerSerializer(serializers.ModelSerializer):
             {"type": "Update", "object": {"type": "Library"}}, context={"library": obj}
         )
 
+    def get_actor(self, o):
+        # Import at runtime to avoid a circular import issue
+        from funkwhale_api.federation import serializers as federation_serializers
+
+        return federation_serializers.APIActorSerializer(o.actor).data
+
 
 class UploadSerializer(serializers.ModelSerializer):
     track = TrackSerializer(required=False, allow_null=True)

api/funkwhale_api/music/views.py

@@ -249,6 +249,7 @@ class LibraryViewSet(
     queryset = (
         models.Library.objects.all()
         .filter(channel=None)
+        .select_related("actor")
         .order_by("-creation_date")
         .annotate(_uploads_count=Count("uploads"))
         .annotate(_size=Sum("uploads__size"))
@@ -261,11 +262,15 @@ class LibraryViewSet(
     required_scope = "libraries"
     anonymous_policy = "setting"
     owner_field = "actor.user"
-    owner_checks = ["read", "write"]
+    owner_checks = ["write"]
 
     def get_queryset(self):
         qs = super().get_queryset()
-        return qs.filter(actor=self.request.user.actor)
+        # allow retrieving a single library by uuid if request.user isn't
+        # the owner. Any other get should be from the owner only
+        if self.action != "retrieve":
+            qs = qs.filter(actor=self.request.user.actor)
+        return qs
 
     def perform_create(self, serializer):
         serializer.save(actor=self.request.user.actor)
@@ -599,7 +604,7 @@ class UploadViewSet(
         models.Upload.objects.all()
         .order_by("-creation_date")
         .prefetch_related(
-            "library",
+            "library__actor",
             "track__artist",
             "track__album__artist",
             "track__attachment_cover",
@@ -613,7 +618,7 @@ class UploadViewSet(
     required_scope = "libraries"
     anonymous_policy = "setting"
     owner_field = "library.actor.user"
-    owner_checks = ["read", "write"]
+    owner_checks = ["write"]
     filterset_class = filters.UploadFilter
     ordering_fields = (
         "creation_date",
@@ -628,7 +633,12 @@ class UploadViewSet(
         if self.action in ["update", "partial_update"]:
             # prevent updating an upload that is already processed
             qs = qs.filter(import_status="draft")
-        return qs.filter(library__actor=self.request.user.actor)
+        if self.action != "retrieve":
+            qs = qs.filter(library__actor=self.request.user.actor)
+        else:
+            actor = utils.get_actor_from_request(self.request)
+            qs = qs.playable_by(actor)
+        return qs
 
     @action(methods=["get"], detail=True, url_path="audio-file-metadata")
     def audio_file_metadata(self, request, *args, **kwargs):

api/requirements/local.txt

@@ -5,7 +5,7 @@ django_coverage_plugin>=1.6,<1.7
 factory_boy>=2.11.1
 
 # django-debug-toolbar that works with Django 1.5+
-django-debug-toolbar>=1.11,<1.12
+django-debug-toolbar>=2.2,<2.3
 
 # improved REPL
 ipdb==0.11

api/tests/federation/test_api_serializers.py

@@ -141,3 +141,65 @@ def test_api_full_actor_serializer(factories, to_api_date):
     serializer = api_serializers.FullActorSerializer(actor)
 
     assert serializer.data == expected
+
+
+def test_fetch_serializer_no_obj(factories, to_api_date):
+    fetch = factories["federation.Fetch"]()
+    expected = {
+        "id": fetch.pk,
+        "url": fetch.url,
+        "creation_date": to_api_date(fetch.creation_date),
+        "fetch_date": None,
+        "status": fetch.status,
+        "detail": fetch.detail,
+        "object": None,
+        "actor": serializers.APIActorSerializer(fetch.actor).data,
+    }
+
+    assert api_serializers.FetchSerializer(fetch).data == expected
+
+
+@pytest.mark.parametrize(
+    "object_factory, expected_type, expected_id",
+    [
+        ("music.Album", "album", "id"),
+        ("music.Artist", "artist", "id"),
+        ("music.Track", "track", "id"),
+        ("music.Library", "library", "uuid"),
+        ("music.Upload", "upload", "uuid"),
+        ("federation.Actor", "account", "full_username"),
+    ],
+)
+def test_fetch_serializer_with_object(
+    object_factory, expected_type, expected_id, factories, to_api_date
+):
+    obj = factories[object_factory]()
+    fetch = factories["federation.Fetch"](object=obj)
+    expected = {
+        "id": fetch.pk,
+        "url": fetch.url,
+        "creation_date": to_api_date(fetch.creation_date),
+        "fetch_date": None,
+        "status": fetch.status,
+        "detail": fetch.detail,
+        "object": {"type": expected_type, expected_id: getattr(obj, expected_id)},
+        "actor": serializers.APIActorSerializer(fetch.actor).data,
+    }
+
+    assert api_serializers.FetchSerializer(fetch).data == expected
+
+
+def test_fetch_serializer_unhandled_obj(factories, to_api_date):
+    fetch = factories["federation.Fetch"](object=factories["users.User"]())
+    expected = {
+        "id": fetch.pk,
+        "url": fetch.url,
+        "creation_date": to_api_date(fetch.creation_date),
+        "fetch_date": None,
+        "status": fetch.status,
+        "detail": fetch.detail,
+        "object": None,
+        "actor": serializers.APIActorSerializer(fetch.actor).data,
+    }
+
+    assert api_serializers.FetchSerializer(fetch).data == expected

api/tests/federation/test_api_views.py

@@ -1,9 +1,12 @@
+import datetime
+
 import pytest
 
 from django.urls import reverse
 
 from funkwhale_api.federation import api_serializers
 from funkwhale_api.federation import serializers
+from funkwhale_api.federation import tasks
 from funkwhale_api.federation import views
 
 
@@ -170,7 +173,8 @@ def test_user_can_update_read_status_of_inbox_item(factories, logged_in_api_clie
 
 
 def test_can_detail_fetch(logged_in_api_client, factories):
-    fetch = factories["federation.Fetch"](url="http://test.object")
+    actor = logged_in_api_client.user.create_actor()
+    fetch = factories["federation.Fetch"](url="http://test.object", actor=actor)
     url = reverse("api:v1:federation:fetches-detail", kwargs={"pk": fetch.pk})
 
     response = logged_in_api_client.get(url)
@@ -209,3 +213,76 @@ def test_can_retrieve_actor(factories, api_client, preferences):
 
     expected = api_serializers.FullActorSerializer(actor).data
     assert response.data == expected
+
+
+@pytest.mark.parametrize(
+    "object_id, expected_url",
+    [
+        ("https://fetch.url", "https://fetch.url"),
+        ("name@domain.tld", "webfinger://name@domain.tld"),
+        ("@name@domain.tld", "webfinger://name@domain.tld"),
+    ],
+)
+def test_can_fetch_using_url_synchronous(
+    object_id, expected_url, factories, logged_in_api_client, mocker, settings
+):
+    settings.FEDERATION_SYNCHRONOUS_FETCH = True
+    actor = logged_in_api_client.user.create_actor()
+
+    def fake_task(fetch_id):
+        actor.fetches.filter(id=fetch_id).update(status="finished")
+
+    fetch_task = mocker.patch.object(tasks, "fetch", side_effect=fake_task)
+
+    url = reverse("api:v1:federation:fetches-list")
+    data = {"object": object_id}
+    response = logged_in_api_client.post(url, data)
+    assert response.status_code == 201
+
+    fetch = actor.fetches.latest("id")
+
+    assert fetch.status == "finished"
+    assert fetch.url == expected_url
+    assert response.data == api_serializers.FetchSerializer(fetch).data
+    fetch_task.assert_called_once_with(fetch_id=fetch.pk)
+
+
+def test_fetch_duplicate(factories, logged_in_api_client, settings, now):
+    object_id = "http://example.test"
+    settings.FEDERATION_DUPLICATE_FETCH_DELAY = 60
+    actor = logged_in_api_client.user.create_actor()
+    duplicate = factories["federation.Fetch"](
+        actor=actor,
+        status="finished",
+        url=object_id,
+        creation_date=now - datetime.timedelta(seconds=59),
+    )
+    url = reverse("api:v1:federation:fetches-list")
+    data = {"object": object_id}
+    response = logged_in_api_client.post(url, data)
+    assert response.status_code == 201
+    assert response.data == api_serializers.FetchSerializer(duplicate).data
+
+
+def test_fetch_duplicate_bypass_with_force(
+    factories, logged_in_api_client, mocker, settings, now
+):
+    fetch_task = mocker.patch.object(tasks, "fetch")
+    object_id = "http://example.test"
+    settings.FEDERATION_DUPLICATE_FETCH_DELAY = 60
+    actor = logged_in_api_client.user.create_actor()
+    duplicate = factories["federation.Fetch"](
+        actor=actor,
+        status="finished",
+        url=object_id,
+        creation_date=now - datetime.timedelta(seconds=59),
+    )
+    url = reverse("api:v1:federation:fetches-list")
+    data = {"object": object_id, "force": True}
+    response = logged_in_api_client.post(url, data)
+
+    fetch = actor.fetches.latest("id")
+    assert fetch != duplicate
+    assert response.status_code == 201
+    assert response.data == api_serializers.FetchSerializer(fetch).data
+    fetch_task.assert_called_once_with(fetch_id=fetch.pk)

api/tests/federation/test_serializers.py

@@ -580,6 +580,37 @@ def test_music_library_serializer_from_private(factories, mocker):
     )
 
 
+def test_music_library_serializer_from_ap_update(factories, mocker):
+    actor = factories["federation.Actor"]()
+    library = factories["music.Library"]()
+
+    data = {
+        "@context": jsonld.get_default_context(),
+        "audience": "https://www.w3.org/ns/activitystreams#Public",
+        "name": "Hello",
+        "summary": "World",
+        "type": "Library",
+        "id": library.fid,
+        "followers": "https://library.id/followers",
+        "attributedTo": actor.fid,
+        "totalItems": 12,
+        "first": "https://library.id?page=1",
+        "last": "https://library.id?page=2",
+    }
+    serializer = serializers.LibrarySerializer(library, data=data)
+
+    assert serializer.is_valid(raise_exception=True)
+
+    serializer.save()
+    library.refresh_from_db()
+
+    assert library.uploads_count == data["totalItems"]
+    assert library.privacy_level == "everyone"
+    assert library.name == "Hello"
+    assert library.description == "World"
+    assert library.followers_url == data["followers"]
+
+
 def test_activity_pub_artist_serializer_to_ap(factories):
     content = factories["common.Content"]()
     artist = factories["music.Artist"](
@@ -610,6 +641,86 @@ def test_activity_pub_artist_serializer_to_ap(factories):
     assert serializer.data == expected
 
 
+def test_activity_pub_artist_serializer_from_ap_create(factories, faker, now, mocker):
+    actor = factories["federation.Actor"]()
+    mocker.patch(
+        "funkwhale_api.federation.utils.retrieve_ap_object", return_value=actor
+    )
+    payload = {
+        "@context": jsonld.get_default_context(),
+        "type": "Artist",
+        "id": "https://test.artist",
+        "name": "Art",
+        "musicbrainzId": faker.uuid4(),
+        "published": now.isoformat(),
+        "attributedTo": actor.fid,
+        "content": "Summary",
+        "image": {
+            "type": "Image",
+            "mediaType": "image/jpeg",
+            "url": "https://attachment.file",
+        },
+        "tag": [
+            {"type": "Hashtag", "name": "#Punk"},
+            {"type": "Hashtag", "name": "#Rock"},
+        ],
+    }
+    serializer = serializers.ArtistSerializer(data=payload)
+    assert serializer.is_valid(raise_exception=True) is True
+
+    artist = serializer.save()
+
+    assert artist.fid == payload["id"]
+    assert artist.attributed_to == actor
+    assert artist.name == payload["name"]
+    assert str(artist.mbid) == payload["musicbrainzId"]
+    assert artist.description.text == payload["content"]
+    assert artist.description.content_type == "text/html"
+    assert artist.attachment_cover.url == payload["image"]["url"]
+    assert artist.attachment_cover.mimetype == payload["image"]["mediaType"]
+    assert artist.get_tags() == ["Punk", "Rock"]
+
+
+def test_activity_pub_artist_serializer_from_ap_update(factories, faker, now, mocker):
+    artist = factories["music.Artist"]()
+    actor = factories["federation.Actor"]()
+    mocker.patch(
+        "funkwhale_api.federation.utils.retrieve_ap_object", return_value=actor
+    )
+    payload = {
+        "@context": jsonld.get_default_context(),
+        "type": "Artist",
+        "id": artist.fid,
+        "name": "Art",
+        "musicbrainzId": faker.uuid4(),
+        "published": now.isoformat(),
+        "attributedTo": actor.fid,
+        "content": "Summary",
+        "image": {
+            "type": "Image",
+            "mediaType": "image/jpeg",
+            "url": "https://attachment.file",
+        },
+        "tag": [
+            {"type": "Hashtag", "name": "#Punk"},
+            {"type": "Hashtag", "name": "#Rock"},
+        ],
+    }
+    serializer = serializers.ArtistSerializer(artist, data=payload)
+    assert serializer.is_valid(raise_exception=True) is True
+    serializer.save()
+    artist.refresh_from_db()
+
+    assert artist.attributed_to == actor
+    assert artist.name == payload["name"]
+    assert str(artist.mbid) == payload["musicbrainzId"]
+    assert artist.description.text == payload["content"]
+    assert artist.description.content_type == "text/html"
+    assert artist.attachment_cover.url == payload["image"]["url"]
+    assert artist.attachment_cover.mimetype == payload["image"]["mediaType"]
+    assert artist.get_tags() == ["Punk", "Rock"]
+
+
 def test_activity_pub_album_serializer_to_ap(factories):
     content = factories["common.Content"]()
     album = factories["music.Album"](
@@ -652,39 +763,42 @@ def test_activity_pub_album_serializer_to_ap(factories):
     assert serializer.data == expected
 
 
-def test_activity_pub_artist_serializer_from_ap_update(factories, faker):
+def test_activity_pub_album_serializer_from_ap_create(factories, faker, now):
-    artist = factories["music.Artist"](attributed=True)
+    actor = factories["federation.Actor"]()
+    artist = factories["music.Artist"]()
+    released = faker.date_object()
     payload = {
         "@context": jsonld.get_default_context(),
-        "type": "Artist",
+        "type": "Album",
-        "id": artist.fid,
+        "id": "https://album.example",
         "name": faker.sentence(),
+        "cover": {"type": "Link", "mediaType": "image/jpeg", "href": faker.url()},
         "musicbrainzId": faker.uuid4(),
-        "published": artist.creation_date.isoformat(),
+        "published": now.isoformat(),
-        "attributedTo": artist.attributed_to.fid,
+        "released": released.isoformat(),
-        "mediaType": "text/html",
+        "artists": [
-        "content": common_utils.render_html(faker.sentence(), "text/html"),
+            serializers.ArtistSerializer(
-        "image": {"type": "Image", "mediaType": "image/jpeg", "url": faker.url()},
+                artist, context={"include_ap_context": False}
+            ).data
+        ],
+        "attributedTo": actor.fid,
         "tag": [
             {"type": "Hashtag", "name": "#Punk"},
             {"type": "Hashtag", "name": "#Rock"},
         ],
     }
-
+    serializer = serializers.AlbumSerializer(data=payload)
-    serializer = serializers.ArtistSerializer(artist, data=payload)
     assert serializer.is_valid(raise_exception=True) is True
 
-    serializer.save()
+    album = serializer.save()
 
-    artist.refresh_from_db()
+    assert album.title == payload["name"]
-
+    assert str(album.mbid) == payload["musicbrainzId"]
-    assert artist.name == payload["name"]
+    assert album.release_date == released
-    assert str(artist.mbid) == payload["musicbrainzId"]
+    assert album.artist == artist
-    assert artist.attachment_cover.url == payload["image"]["url"]
+    assert album.attachment_cover.url == payload["cover"]["href"]
-    assert artist.attachment_cover.mimetype == payload["image"]["mediaType"]
+    assert album.attachment_cover.mimetype == payload["cover"]["mediaType"]
-    assert artist.description.text == payload["content"]
+    assert sorted(album.tagged_items.values_list("tag__name", flat=True)) == [
-    assert artist.description.content_type == "text/html"
-    assert sorted(artist.tagged_items.values_list("tag__name", flat=True)) == [
         "Punk",
         "Rock",
     ]
@@ -1062,6 +1176,43 @@ def test_activity_pub_upload_serializer_from_ap(factories, mocker, r_mock):
     assert upload.modification_date == updated
 
 
+def test_activity_pub_upload_serializer_from_ap_update(factories, mocker, now, r_mock):
+    library = factories["music.Library"]()
+    upload = factories["music.Upload"](library=library)
+
+    data = {
+        "@context": jsonld.get_default_context(),
+        "type": "Audio",
+        "id": upload.fid,
+        "name": "Ignored",
+        "published": now.isoformat(),
+        "updated": now.isoformat(),
+        "duration": 42,
+        "bitrate": 42,
+        "size": 66,
+        "url": {
+            "href": "https://audio.file/url",
+            "type": "Link",
+            "mediaType": "audio/mp3",
+        },
+        "library": library.fid,
+        "track": serializers.TrackSerializer(upload.track).data,
+    }
+    r_mock.get(data["track"]["album"]["cover"]["href"], body=io.BytesIO(b"coucou"))
+
+    serializer = serializers.UploadSerializer(upload, data=data)
+    assert serializer.is_valid(raise_exception=True)
+    serializer.save()
+    upload.refresh_from_db()
+
+    assert upload.fid == data["id"]
+    assert upload.duration == data["duration"]
+    assert upload.size == data["size"]
+    assert upload.bitrate == data["bitrate"]
+    assert upload.source == data["url"]["href"]
+    assert upload.mimetype == data["url"]["mediaType"]
+
+
 def test_activity_pub_upload_serializer_validtes_library_actor(factories, mocker):
     library = factories["music.Library"]()
     usurpator = factories["federation.Actor"]()
@@ -1201,7 +1352,7 @@ def test_track_serializer_update_license(factories):
 
     obj = factories["music.Track"](license=None)
 
-    serializer = serializers.TrackSerializer()
+    serializer = serializers.TrackSerializer(obj)
     serializer.update(obj, {"license": "http://creativecommons.org/licenses/by/2.0/"})
 
     obj.refresh_from_db()

api/tests/federation/test_tasks.py

@@ -395,3 +395,156 @@ def test_fetch_success(factories, r_mock, mocker):
     assert init.call_args[0][1] == artist
     assert init.call_args[1]["data"] == payload
     assert save.call_count == 1
+
+
+def test_fetch_webfinger(factories, r_mock, mocker):
+    actor = factories["federation.Actor"]()
+    fetch = factories["federation.Fetch"](
+        url="webfinger://{}".format(actor.full_username)
+    )
+    payload = serializers.ActorSerializer(actor).data
+    init = mocker.spy(serializers.ActorSerializer, "__init__")
+    save = mocker.spy(serializers.ActorSerializer, "save")
+    webfinger_payload = {
+        "subject": "acct:{}".format(actor.full_username),
+        "aliases": ["https://test.webfinger"],
+        "links": [
+            {"rel": "self", "type": "application/activity+json", "href": actor.fid}
+        ],
+    }
+    webfinger_url = "https://{}/.well-known/webfinger?resource={}".format(
+        actor.domain_id, webfinger_payload["subject"]
+    )
+    r_mock.get(actor.fid, json=payload)
+    r_mock.get(webfinger_url, json=webfinger_payload)
+
+    tasks.fetch(fetch_id=fetch.pk)
+
+    fetch.refresh_from_db()
+    payload["@context"].append("https://funkwhale.audio/ns")
+    assert fetch.status == "finished"
+    assert fetch.object == actor
+    assert init.call_count == 1
+    assert init.call_args[0][1] == actor
+    assert init.call_args[1]["data"] == payload
+    assert save.call_count == 1
+
+
+def test_fetch_rel_alternate(factories, r_mock, mocker):
+    actor = factories["federation.Actor"]()
+    fetch = factories["federation.Fetch"](url="http://example.page")
+    html_text = """
+    <html>
+        <head>
+            <link rel="alternate" type="application/activity+json" href="{}" />
+        </head>
+    </html>
+    """.format(
+        actor.fid
+    )
+    ap_payload = serializers.ActorSerializer(actor).data
+    init = mocker.spy(serializers.ActorSerializer, "__init__")
+    save = mocker.spy(serializers.ActorSerializer, "save")
+    r_mock.get(fetch.url, text=html_text)
+    r_mock.get(actor.fid, json=ap_payload)
+
+    tasks.fetch(fetch_id=fetch.pk)
+
+    fetch.refresh_from_db()
+    ap_payload["@context"].append("https://funkwhale.audio/ns")
+    assert fetch.status == "finished"
+    assert fetch.object == actor
+    assert init.call_count == 1
+    assert init.call_args[0][1] == actor
+    assert init.call_args[1]["data"] == ap_payload
+    assert save.call_count == 1
+
+
+@pytest.mark.parametrize(
+    "factory_name, serializer_class",
+    [
+        ("federation.Actor", serializers.ActorSerializer),
+        ("music.Library", serializers.LibrarySerializer),
+        ("music.Artist", serializers.ArtistSerializer),
+        ("music.Album", serializers.AlbumSerializer),
+        ("music.Track", serializers.TrackSerializer),
+    ],
+)
+def test_fetch_url(factory_name, serializer_class, factories, r_mock, mocker):
+    obj = factories[factory_name]()
+    fetch = factories["federation.Fetch"](url=obj.fid)
+    payload = serializer_class(obj).data
+    init = mocker.spy(serializer_class, "__init__")
+    save = mocker.spy(serializer_class, "save")
+
+    r_mock.get(obj.fid, json=payload)
+
+    tasks.fetch(fetch_id=fetch.pk)
+
+    fetch.refresh_from_db()
+    payload["@context"].append("https://funkwhale.audio/ns")
+    assert fetch.status == "finished"
+    assert fetch.object == obj
+    assert init.call_count == 1
+    assert init.call_args[0][1] == obj
+    assert init.call_args[1]["data"] == payload
+    assert save.call_count == 1
+
+
+def test_fetch_honor_instance_policy_domain(factories):
+    domain = factories["moderation.InstancePolicy"](
+        block_all=True, for_domain=True
+    ).target_domain
+    fid = "https://{}/test".format(domain.name)
+
+    fetch = factories["federation.Fetch"](url=fid)
+    tasks.fetch(fetch_id=fetch.pk)
+    fetch.refresh_from_db()
+
+    assert fetch.status == "errored"
+    assert fetch.detail["error_code"] == "blocked"
+
+
+def test_fetch_honor_mrf_inbox_before_http(mrf_inbox_registry, factories, mocker):
+    apply = mocker.patch.object(mrf_inbox_registry, "apply", return_value=(None, False))
+    fid = "http://domain/test"
+    fetch = factories["federation.Fetch"](url=fid)
+    tasks.fetch(fetch_id=fetch.pk)
+    fetch.refresh_from_db()
+
+    assert fetch.status == "errored"
+    assert fetch.detail["error_code"] == "blocked"
+    apply.assert_called_once_with({"id": fid})
+
+
+def test_fetch_honor_mrf_inbox_after_http(
+    r_mock, mrf_inbox_registry, factories, mocker
+):
+    apply = mocker.patch.object(
+        mrf_inbox_registry, "apply", side_effect=[(True, False), (None, False)]
+    )
+    payload = {"id": "http://domain/test", "actor": "hello"}
+    r_mock.get(payload["id"], json=payload)
+    fetch = factories["federation.Fetch"](url=payload["id"])
+    tasks.fetch(fetch_id=fetch.pk)
+    fetch.refresh_from_db()
+
+    assert fetch.status == "errored"
+    assert fetch.detail["error_code"] == "blocked"
+
+    apply.assert_any_call({"id": payload["id"]})
+    apply.assert_any_call(payload)
+
+
+def test_fetch_honor_instance_policy_different_url_and_id(r_mock, factories):
+    domain = factories["moderation.InstancePolicy"](
+        block_all=True, for_domain=True
+    ).target_domain
+    fid = "https://ok/test"
+    r_mock.get(fid, json={"id": "http://{}/test".format(domain.name)})
+    fetch = factories["federation.Fetch"](url=fid)
+    tasks.fetch(fetch_id=fetch.pk)
+    fetch.refresh_from_db()
+
+    assert fetch.status == "errored"
+    assert fetch.detail["error_code"] == "blocked"

api/tests/music/test_views.py

@@ -640,6 +640,16 @@ def test_user_can_list_their_library(factories, logged_in_api_client):
     assert response.data["results"][0]["uuid"] == str(library.uuid)
 
 
+def test_user_can_retrieve_another_user_library(factories, logged_in_api_client):
+    library = factories["music.Library"]()
+
+    url = reverse("api:v1:libraries-detail", kwargs={"uuid": library.uuid})
+    response = logged_in_api_client.get(url)
+
+    assert response.status_code == 200
+    assert response.data["uuid"] == str(library.uuid)
+
+
 def test_library_list_excludes_channel_library(factories, logged_in_api_client):
     actor = logged_in_api_client.user.create_actor()
     factories["audio.Channel"](attributed_to=actor)
@@ -670,9 +680,11 @@ def test_library_delete_via_api_triggers_outbox(factories, mocker):
     )
 
 
-def test_user_cannot_get_other_actors_uploads(factories, logged_in_api_client):
+def test_user_cannot_get_other_not_playable_uploads(factories, logged_in_api_client):
     logged_in_api_client.user.create_actor()
-    upload = factories["music.Upload"]()
+    upload = factories["music.Upload"](
+        import_status="finished", library__privacy_level="private"
+    )
 
     url = reverse("api:v1:uploads-detail", kwargs={"uuid": upload.uuid})
     response = logged_in_api_client.get(url)
@@ -680,6 +692,19 @@ def test_user_cannot_get_other_actors_uploads(factories, logged_in_api_client):
     assert response.status_code == 404
 
 
+def test_user_can_get_retrieve_playable_uploads(factories, logged_in_api_client):
+    logged_in_api_client.user.create_actor()
+    upload = factories["music.Upload"](
+        import_status="finished", library__privacy_level="everyone"
+    )
+
+    url = reverse("api:v1:uploads-detail", kwargs={"uuid": upload.uuid})
+    response = logged_in_api_client.get(url)
+
+    assert response.status_code == 200
+    assert response.data["uuid"] == str(upload.uuid)
+
+
 def test_user_cannot_delete_other_actors_uploads(factories, logged_in_api_client):
     logged_in_api_client.user.create_actor()
     upload = factories["music.Upload"]()

docs/conf.py

@@ -95,18 +95,16 @@ html_theme = "sphinx_rtd_theme"
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-html_theme_options = {
+html_theme_options = {"gitlab_url": "https://dev.funkwhale.audio/funkwhale/funkwhale"}
-    'gitlab_url': 'https://dev.funkwhale.audio/funkwhale/funkwhale'
-}
 html_context = {
-  'display_gitlab': True,
+    "display_gitlab": True,
-  'gitlab_host': 'dev.funkwhale.audio',
+    "gitlab_host": "dev.funkwhale.audio",
-  'gitlab_repo': 'funkwhale',
+    "gitlab_repo": "funkwhale",
-  'gitlab_user': 'funkwhale',
+    "gitlab_user": "funkwhale",
-  'gitlab_version': 'master',
+    "gitlab_version": "master",
-  'conf_py_path': '/docs/',
+    "conf_py_path": "/docs/",
 }
-html_logo = 'logo.svg'
+html_logo = "logo.svg"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -173,15 +171,13 @@ texinfo_documents = [
 # Define list of redirect files to be build in the Sphinx build process
 
 redirect_files = [
-
+    ("importing-music.html", "admin/importing-music.html"),
-    ('importing-music.html', 'admin/importing-music.html'),
+    ("architecture.html", "developers/architecture.html"),
-    ('architecture.html', 'developers/architecture.html'),
+    ("troubleshooting.html", "admin/troubleshooting.html"),
-    ('troubleshooting.html', 'admin/troubleshooting.html'),
+    ("configuration.html", "admin/configuration.html"),
-    ('configuration.html', 'admin/configuration.html'),
+    ("upgrading/index.html", "../admin/upgrading.html"),
-    ('upgrading/index.html', '../admin/upgrading.html'),
+    ("upgrading/0.17.html", "../admin/0.17.html"),
-    ('upgrading/0.17.html', '../admin/0.17.html'),
+    ("users/django.html", "../admin/django.html"),
-    ('users/django.html', '../admin/django.html'),
-
 ]
 
 # Generate redirect template
@@ -199,16 +195,17 @@ redirect_template = """\
 
 # Tell Sphinx to copy the files
 
+
 def copy_legacy_redirects(app, docname):
-    if app.builder.name == 'html':
+    if app.builder.name == "html":
         for html_src_path, new in redirect_files:
             page = redirect_template.format(new=new)
-            target_path = app.outdir + '/' + html_src_path
+            target_path = app.outdir + "/" + html_src_path
             if not os.path.exists(os.path.dirname(target_path)):
-               os.makedirs(os.path.dirname(target_path))
+                os.makedirs(os.path.dirname(target_path))
-            with open(target_path, 'w') as f:
+            with open(target_path, "w") as f:
-               f.write(page)
+                f.write(page)
 
 
 def setup(app):
-    app.connect('build-finished', copy_legacy_redirects)
+    app.connect("build-finished", copy_legacy_redirects)

docs/serve.py

@@ -1,13 +1,10 @@
 #!/usr/bin/env python
 from subprocess import call
+
 # initial make
 call(["python", "-m", "sphinx", ".", "/tmp/_build"])
 from livereload import Server, shell
 
 server = Server()
-server.watch('.', shell('python -m sphinx . /tmp/_build'))
+server.watch(".", shell("python -m sphinx . /tmp/_build"))
-server.serve(
+server.serve(root="/tmp/_build/", liveport=35730, port=8001, host="0.0.0.0")
-    root='/tmp/_build/',
-    liveport=35730,
-    port=8001,
-host='0.0.0.0')

front/scripts/print-duplicates-source.py

@@ -9,21 +9,17 @@ def print_duplicates(path):
     contexts_by_id = collections.defaultdict(list)
     for e in pofile:
         contexts_by_id[e.msgid].append(e.msgctxt)
-    count = collections.Counter(
+    count = collections.Counter([e.msgid for e in pofile])
-        [e.msgid for e in pofile]
+    duplicates = [(k, v) for k, v in count.items() if v > 1]
-    )
-    duplicates = [
-        (k, v) for k, v in count.items()
-        if v > 1
-    ]
     for k, v in sorted(duplicates, key=lambda r: r[1], reverse=True):
-        print('{} entries - {}:'.format(v, k))
+        print("{} entries - {}:".format(v, k))
         for ctx in contexts_by_id[k]:
-            print('  - {}'.format(ctx))
+            print("  - {}".format(ctx))
         print()
 
     total_duplicates = sum([v - 1 for _, v in duplicates])
-    print('{} total duplicates'.format(total_duplicates))
+    print("{} total duplicates".format(total_duplicates))
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()

front/src/components/audio/SearchBar.vue

@@ -15,6 +15,7 @@
 <script>
 import jQuery from 'jquery'
 import router from '@/router'
+import lodash from '@/lodash'
 import GlobalEvents from "@/components/utils/global-events"
 
 export default {
@@ -55,7 +56,11 @@ export default {
         noResults: this.$pgettext('Sidebar/Search/Error.Label', 'Sorry, there are no results for this search')
       },
       onSelect (result, response) {
+        jQuery(self.$el).search("set value", searchQuery)
+        console.log('SELECTEING', result)
         router.push(result.routerUrl)
+        jQuery(self.$el).search("hide results")
+        return false
       },
       onSearchQuery (query) {
         self.$emit('search')
@@ -70,9 +75,14 @@ export default {
           return xhrObject
         },
         onResponse: function (initialResponse) {
+          let objId = self.extractObjId(searchQuery)
           let results = {}
           let isEmptyResults = true
           let categories = [
+            {
+              code: 'federation',
+              name: self.$pgettext('*/*/*', 'Federation'),
+            },
             {
               code: 'artists',
               route: 'library.artists.detail',
@@ -139,21 +149,42 @@ export default {
               name: category.name,
               results: []
             }
-            initialResponse[category.code].forEach(result => {
+            if (category.code === 'federation') {
-              isEmptyResults = false
+
-              let id = category.getId(result)
+              if (objId) {
-              results[category.code].results.push({
+                isEmptyResults = false
-                title: category.getTitle(result),
+                let searchMessage = self.$pgettext('Search/*/*', 'Search on the fediverse')
-                id,
+                results['federation'] = {
-                routerUrl: {
+                  name: self.$pgettext('*/*/*', 'Federation'),
-                  name: category.route,
+                  results: [{
-                  params: {
+                    title: searchMessage,
-                    id
+                    routerUrl: {
-                  }
+                      name: 'search',
-                },
+                      query: {
-                description: category.getDescription(result)
+                        id: objId,
+                      }
+                    }
+                  }]
+                }
+              }
+            }
+            else {
+              initialResponse[category.code].forEach(result => {
+                isEmptyResults = false
+                let id = category.getId(result)
+                results[category.code].results.push({
+                  title: category.getTitle(result),
+                  id,
+                  routerUrl: {
+                    name: category.route,
+                    params: {
+                      id
+                    }
+                  },
+                  description: category.getDescription(result)
+                })
               })
-            })
+            }
           })
           return {
             results: isEmptyResults ? {} : results
@@ -167,6 +198,19 @@ export default {
     focusSearch () {
       this.$refs.search.focus()
     },
+    extractObjId (query) {
+      query = lodash.trim(query)
+      query = lodash.trim(query, '@')
+      if (query.indexOf(' ') > -1) {
+        return
+      }
+      if (query.startsWith('http://') || query.startsWith('https://')) {
+        return query
+      }
+      if (query.split('@').length > 1) {
+        return query
+      }
+    }
   }
 }
 </script>

front/src/components/common/ActorAvatar.vue

@@ -1,5 +1,6 @@
 <template>
-  <span :style="defaultAvatarStyle" class="ui avatar circular label">{{ actor.preferred_username[0]}}</span>
+  <img v-if="actor.icon && actor.icon.original" :src="actor.icon.small_square_crop" class="ui avatar circular image" />
+  <span v-else :style="defaultAvatarStyle" class="ui avatar circular label">{{ actor.preferred_username[0]}}</span>
 </template>
 
 <script>
@@ -20,7 +21,7 @@ export default {
 }
 </script>
 <style lang="scss">
-.ui.circular.avatar.label {
+.ui.circular.avatar {
   width: 28px;
   height: 28px;
   font-size: 1em !important;

front/src/components/common/ActorLink.vue

@@ -19,7 +19,7 @@ export default {
       if (this.actor.is_local) {
         return {name: 'profile.overview', params: {username: this.actor.preferred_username}}
       } else {
-        return {name: 'profile.overview', params: {username: this.actor.full_username}}
+        return {name: 'profile.full.overview', params: {username: this.actor.preferred_username, domain: this.actor.domain}}
       }
     },
     repr () {

front/src/lodash.js

@@ -16,4 +16,5 @@ export default {
   isEqual: require('lodash/isEqual'),
   sum: require('lodash/sum'),
   startCase: require('lodash/startCase'),
+  trim: require('lodash/trim'),
 }

front/src/router/index.js

@@ -74,6 +74,15 @@ export default new Router({
         defaultKey: route.query.key
       })
     },
+    {
+      path: "/search",
+      name: "search",
+      component: () =>
+        import(/* webpackChunkName: "core" */ "@/views/Search"),
+      props: route => ({
+        initialId: route.query.id
+      })
+    },
     {
       path: "/auth/password/reset/confirm",
       name: "auth.password-reset-confirm",
@@ -143,17 +152,17 @@ export default new Router({
         ),
       props: true
     },
-    ...['/@:username', '/@:username@:domain'].map((path) => {
+    ...[{suffix: '.full', path: '/@:username@:domain'}, {suffix: '', path: '/@:username'}].map((route) => {
       return {
-        path: path,
+        path: route.path,
-        name: "profile",
+        name: `profile${route.suffix}`,
         component: () =>
         import(/* webpackChunkName: "core" */ "@/views/auth/ProfileBase"),
         props: true,
         children: [
           {
             path: "",
-            name: "profile.overview",
+            name: `profile${route.suffix}.overview`,
             component: () =>
               import(
                 /* webpackChunkName: "core" */ "@/views/auth/ProfileOverview"
@@ -161,7 +170,7 @@ export default new Router({
           },
           {
             path: "activity",
-            name: "profile.activity",
+            name: `profile${route.suffix}.activity`,
             component: () =>
               import(
                 /* webpackChunkName: "core" */ "@/views/auth/ProfileActivity"

front/src/views/Search.vue

@@ -0,0 +1,207 @@
+<template>
+  <main class="main pusher" v-title="labels.title">
+    <section class="ui vertical stripe segment">
+      <div class="ui small text container">
+        <form :class="['ui', {loading: isLoading}, 'form']" @submit.stop.prevent="createFetch">
+          <h2><translate translate-context="Content/Fetch/Title">Retrieve a remote object</translate></h2>
+          <p>
+            <translate translate-context="Content/Fetch/Paragraph">Use this form to retrieve an object hosted somewhere else in the fediverse.</translate>
+          </p>
+          <div v-if="errors.length > 0" class="ui negative message">
+            <div class="header"><translate translate-context="Content/*/Error message.Title">Error while fetching object</translate></div>
+            <ul class="list">
+              <li v-for="error in errors">{{ error }}</li>
+            </ul>
+          </div>
+          <div class="ui required field">
+            <label for="object-id">
+              {{ labels.fieldLabel }}
+            </label>
+            <input type="text" name="object-id" id="object-id" v-model="id" required>
+          </div>
+          <button type="submit" :class="['ui', 'primary', {loading: isLoading}, 'button']" :disabled="isLoading || !id || id.length === 0">
+            <translate translate-context="Content/Search/Input.Label/Noun">Search</translate>
+          </button>
+        </form>
+        <div v-if="!isLoading && fetch && fetch.status === 'finished'">
+          <div class="ui hidden divider"></div>
+          <h2><translate translate-context="Content/Fetch/Title/Noun">Result</translate></h2>
+          <div class="ui hidden divider"></div>
+          <div v-if="objComponent" class="ui app-cards cards">
+            <component v-bind="objComponent.props" :is="objComponent.type"></component>
+          </div>
+          <div v-else class="ui warning message">
+            <p><translate translate-context="Content/*/Error message.Title">This kind of object isn't supported yet</translate></p>
+          </div>
+        </div>
+      </div>
+    </section>
+  </main>
+</template>
+
+<script>
+import axios from 'axios'
+
+
+import AlbumCard from '@/components/audio/album/Card'
+import ArtistCard from '@/components/audio/artist/Card'
+import LibraryCard from '@/views/content/remote/Card'
+import ChannelEntryCard from '@/components/audio/ChannelEntryCard'
+
+export default {
+  props: {
+    initialId: { type: String, required: false}
+  },
+  components: {
+    ActorLink:  () => import(/* webpackChunkName: "common" */ "@/components/common/ActorLink"),
+    ArtistCard,
+    AlbumCard,
+    LibraryCard,
+    ChannelEntryCard,
+  },
+  data () {
+    return {
+      id: this.initialId,
+      fetch: null,
+      obj: null,
+      isLoading: false,
+      errors: [],
+    }
+  },
+  created () {
+    if (this.id) {
+      this.createFetch()
+    }
+  },
+  computed: {
+    labels() {
+      return {
+        title: this.$pgettext('Head/Fetch/Title', "Search a remote object"),
+        fieldLabel: this.$pgettext('Head/Fetch/Field.Placeholder', "URL or @username"),
+      }
+    },
+    objInfo () {
+      if (this.fetch && this.fetch.status === 'finished') {
+        return this.fetch.object
+      }
+    },
+    objComponent () {
+      if (!this.obj) {
+        return
+      }
+      switch (this.objInfo.type) {
+        case "account":
+          return {
+            type: "actor-link",
+            props: {actor: this.obj}
+          }
+        case "library":
+          return {
+            type: "library-card",
+            props: {library: this.obj}
+          }
+        case "album":
+          return {
+            type: "album-card",
+            props: {album: this.obj}
+          }
+        case "artist":
+          return {
+            type: "artist-card",
+            props: {artist: this.obj}
+          }
+        case "upload":
+          return {
+            type: "channel-entry-card",
+            props: {entry: this.obj.track}
+          }
+
+        default:
+          return
+      }
+    }
+  },
+  methods: {
+    createFetch () {
+      if (!this.id) {
+        return
+      }
+      this.$router.replace({name: "search", query: {id: this.id}})
+      this.fetch = null
+      let self = this
+      self.errors = []
+      self.isLoading = true
+      let payload = {
+        object: this.id
+      }
+
+      axios.post('federation/fetches/', payload).then((response) => {
+        self.isLoading = false
+        self.fetch = response.data
+        if (self.fetch.status === 'errored' || self.fetch.status === 'skipped') {
+          self.errors.push(
+            self.$pgettext("Content/*/Error message.Title", "This object cannot be retrieved")
+          )
+        }
+      }, error => {
+        self.isLoading = false
+        self.errors = error.backendErrors
+      })
+    },
+    getObj (objInfo) {
+      if (!this.id) {
+        return
+      }
+      let self = this
+      self.isLoading = true
+      let url = null
+      switch (objInfo.type) {
+        case 'account':
+          url = `federation/actors/${objInfo.full_username}/`
+          break;
+        case 'library':
+          url = `libraries/${objInfo.uuid}/`
+          break;
+        case 'artist':
+          url = `artists/${objInfo.id}/`
+          break;
+        case 'album':
+          url = `albums/${objInfo.id}/`
+          break;
+        case 'upload':
+          url = `uploads/${objInfo.uuid}/`
+          break;
+
+        default:
+          break;
+      }
+      if (!url) {
+        this.errors.push(
+          self.$pgettext("Content/*/Error message.Title", "This kind of object isn't supported yet")
+        )
+        this.isLoading = false
+        return
+      }
+      axios.get(url).then((response) => {
+        self.obj = response.data
+        self.isLoading = false
+      }, error => {
+        self.isLoading = false
+        self.errors = error.backendErrors
+      })
+    }
+  },
+  watch: {
+    initialId (v) {
+      this.id = v
+      this.createFetch()
+    },
+    objInfo (v) {
+      this.obj = null
+      if (v) {
+        this.getObj(v)
+      }
+    }
+  }
+}
+</script>

front/src/views/auth/ProfileBase.vue

@@ -105,6 +105,7 @@ export default {
   methods: {
     fetch () {
       let self = this
+      self.object = null
       self.isLoading = true
       axios.get(`federation/actors/${this.fullUsername}/`).then((response) => {
         self.object = response.data
@@ -139,6 +140,14 @@ export default {
     displayName () {
       return this.object.name || this.object.preferred_username
     }
+  },
+  watch: {
+    domain () {
+      this.fetch()
+    },
+    username () {
+      this.fetch()
+    }
   }
 }
 </script>