Merge branch 'federation-follow-request' into 'develop'

Federation follow request

See merge request funkwhale/funkwhale!123

api/config/settings/common.py

@@ -30,7 +30,12 @@ FUNKWHALE_HOSTNAME = urlsplit(FUNKWHALE_URL).netloc
 
 FEDERATION_ENABLED = env.bool('FEDERATION_ENABLED', default=True)
 FEDERATION_HOSTNAME = env('FEDERATION_HOSTNAME', default=FUNKWHALE_HOSTNAME)
-
+FEDERATION_COLLECTION_PAGE_SIZE = env.int(
+    'FEDERATION_COLLECTION_PAGE_SIZE', default=50
+)
+FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
+    'FEDERATION_MUSIC_NEEDS_APPROVAL', default=True
+)
 ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
 
 # APP CONFIGURATION

api/funkwhale_api/common/utils.py

@@ -1,3 +1,4 @@
+from urllib.parse import urlencode, parse_qs, urlsplit, urlunsplit
 import os
 import shutil
 
@@ -25,3 +26,20 @@ def on_commit(f, *args, **kwargs):
     return transaction.on_commit(
         lambda: f(*args, **kwargs)
     )
+
+
+def set_query_parameter(url, **kwargs):
+    """Given a URL, set or replace a query parameter and return the
+    modified URL.
+
+    >>> set_query_parameter('http://example.com?foo=bar&biz=baz', 'foo', 'stuff')
+    'http://example.com?foo=stuff&biz=baz'
+    """
+    scheme, netloc, path, query_string, fragment = urlsplit(url)
+    query_params = parse_qs(query_string)
+
+    for param_name, param_value in kwargs.items():
+        query_params[param_name] = [param_value]
+    new_query_string = urlencode(query_params, doseq=True)
+
+    return urlunsplit((scheme, netloc, path, new_query_string, fragment))

api/funkwhale_api/federation/activity.py

@@ -2,7 +2,9 @@ import logging
 import json
 import requests
 import requests_http_signature
+import uuid
 
+from . import models
 from . import signing
 
 logger = logging.getLogger(__name__)
@@ -42,33 +44,26 @@ ACTIVITY_TYPES = [
 OBJECT_TYPES = [
     'Article',
     'Audio',
+    'Collection',
     'Document',
     'Event',
     'Image',
     'Note',
+    'OrderedCollection',
     'Page',
     'Place',
     'Profile',
     'Relationship',
     'Tombstone',
     'Video',
-]
+] + ACTIVITY_TYPES
+
 
 def deliver(activity, on_behalf_of, to=[]):
     from . import actors
     logger.info('Preparing activity delivery to %s', to)
-    auth = requests_http_signature.HTTPSignatureAuth(
+    auth = signing.get_auth(
-        use_auth_header=False,
+        on_behalf_of.private_key, on_behalf_of.private_key_id)
-        headers=[
-            '(request-target)',
-            'user-agent',
-            'host',
-            'date',
-            'content-type',],
-        algorithm='rsa-sha256',
-        key=on_behalf_of.private_key.encode('utf-8'),
-        key_id=on_behalf_of.private_key_id,
-    )
     for url in to:
         recipient_actor = actors.get_actor(url)
         logger.debug('delivering to %s', recipient_actor.inbox_url)
@@ -83,3 +78,68 @@ def deliver(activity, on_behalf_of, to=[]):
         )
         response.raise_for_status()
         logger.debug('Remote answered with %s', response.status_code)
+
+
+def get_follow(follow_id, follower, followed):
+    return {
+        '@context': [
+            'https://www.w3.org/ns/activitystreams',
+            'https://w3id.org/security/v1',
+            {}
+        ],
+        'actor': follower.url,
+        'id': follower.url + '#follows/{}'.format(follow_id),
+        'object': followed.url,
+        'type': 'Follow'
+    }
+
+
+def get_undo(id, actor, object):
+    return {
+        '@context': [
+            'https://www.w3.org/ns/activitystreams',
+            'https://w3id.org/security/v1',
+            {}
+        ],
+        'type': 'Undo',
+        'id': id + '/undo',
+        'actor': actor.url,
+        'object': object,
+    }
+
+
+def get_accept_follow(accept_id, accept_actor, follow, follow_actor):
+    return {
+        "@context": [
+            "https://www.w3.org/ns/activitystreams",
+            "https://w3id.org/security/v1",
+            {}
+        ],
+        "id": accept_actor.url + '#accepts/follows/{}'.format(
+            accept_id),
+        "type": "Accept",
+        "actor": accept_actor.url,
+        "object": {
+            "id": follow['id'],
+            "type": "Follow",
+            "actor": follow_actor.url,
+            "object": accept_actor.url
+        },
+    }
+
+
+def accept_follow(target, follow, actor):
+    accept_uuid = uuid.uuid4()
+    accept = get_accept_follow(
+        accept_id=accept_uuid,
+        accept_actor=target,
+        follow=follow,
+        follow_actor=actor)
+    deliver(
+        accept,
+        to=[actor.url],
+        on_behalf_of=target)
+    return models.Follow.objects.get_or_create(
+        actor=actor,
+        target=target,
+    )

api/funkwhale_api/federation/actors.py

@@ -1,8 +1,10 @@
 import logging
 import requests
+import uuid
 import xml
 
 from django.conf import settings
+from django.db import transaction
 from django.urls import reverse
 from django.utils import timezone
 
@@ -11,8 +13,10 @@ from rest_framework.exceptions import PermissionDenied
 from dynamic_preferences.registries import global_preferences_registry
 
 from . import activity
+from . import keys
 from . import models
 from . import serializers
+from . import signing
 from . import utils
 
 logger = logging.getLogger(__name__)
@@ -47,31 +51,48 @@ def get_actor(actor_url):
 
 class SystemActor(object):
     additional_attributes = {}
+    manually_approves_followers = False
+
+    def get_request_auth(self):
+        actor = self.get_actor_instance()
+        return signing.get_auth(
+            actor.private_key, actor.private_key_id)
+
+    def serialize(self):
+        actor = self.get_actor_instance()
+        serializer = serializers.ActorSerializer(actor)
+        return serializer.data
 
     def get_actor_instance(self):
-        a = models.Actor(
+        try:
-            **self.get_instance_argument(
+            return models.Actor.objects.get(url=self.get_actor_url())
-                self.id,
+        except models.Actor.DoesNotExist:
-                name=self.name,
+            pass
-                summary=self.summary,
+        private, public = keys.get_key_pair()
-                **self.additional_attributes
+        args = self.get_instance_argument(
-            )
+            self.id,
+            name=self.name,
+            summary=self.summary,
+            **self.additional_attributes
         )
-        a.pk = self.id
+        args['private_key'] = private.decode('utf-8')
-        return a
+        args['public_key'] = public.decode('utf-8')
+        return models.Actor.objects.create(**args)
+
+    def get_actor_url(self):
+        return utils.full_url(
+            reverse(
+                'federation:instance-actors-detail',
+                kwargs={'actor': self.id}))
 
     def get_instance_argument(self, id, name, summary, **kwargs):
-        preferences = global_preferences_registry.manager()
         p = {
             'preferred_username': id,
             'domain': settings.FEDERATION_HOSTNAME,
             'type': 'Person',
             'name': name.format(host=settings.FEDERATION_HOSTNAME),
             'manually_approves_followers': True,
-            'url': utils.full_url(
+            'url': self.get_actor_url(),
-                reverse(
-                    'federation:instance-actors-detail',
-                    kwargs={'actor': id})),
             'shared_inbox_url': utils.full_url(
                 reverse(
                     'federation:instance-actors-inbox',
@@ -84,8 +105,6 @@ class SystemActor(object):
                 reverse(
                     'federation:instance-actors-outbox',
                     kwargs={'actor': id})),
-            'public_key': preferences['federation__public_key'],
-            'private_key': preferences['federation__private_key'],
             'summary': summary.format(host=settings.FEDERATION_HOSTNAME)
         }
         p.update(kwargs)
@@ -95,7 +114,7 @@ class SystemActor(object):
         raise NotImplementedError
 
     def post_inbox(self, data, actor=None):
-        raise NotImplementedError
+        return self.handle(data, actor=actor)
 
     def get_outbox(self, data, actor=None):
         raise NotImplementedError
@@ -103,6 +122,62 @@ class SystemActor(object):
     def post_outbox(self, data, actor=None):
         raise NotImplementedError
 
+    def handle(self, data, actor=None):
+        """
+        Main entrypoint for handling activities posted to the
+        actor's inbox
+        """
+        logger.info('Received activity on %s inbox', self.id)
+
+        if actor is None:
+            raise PermissionDenied('Actor not authenticated')
+
+        serializer = serializers.ActivitySerializer(
+            data=data, context={'actor': actor})
+        serializer.is_valid(raise_exception=True)
+
+        ac = serializer.data
+        try:
+            handler = getattr(
+                self, 'handle_{}'.format(ac['type'].lower()))
+        except (KeyError, AttributeError):
+            logger.debug(
+                'No handler for activity %s', ac['type'])
+            return
+
+        return handler(data, actor)
+
+    def handle_follow(self, ac, sender):
+        system_actor = self.get_actor_instance()
+        if self.manually_approves_followers:
+            fr, created = models.FollowRequest.objects.get_or_create(
+                actor=sender,
+                target=system_actor,
+                approved=None,
+            )
+            return fr
+
+        return activity.accept_follow(
+            system_actor, ac, sender
+        )
+
+    def handle_undo_follow(self, ac, sender):
+        actor = self.get_actor_instance()
+        models.Follow.objects.filter(
+            actor=sender,
+            target=actor,
+        ).delete()
+
+    def handle_undo(self, ac, sender):
+        if ac['object']['type'] != 'Follow':
+            return
+
+        if ac['object']['actor'] != sender.url:
+            # not the same actor, permission issue
+            return
+
+        self.handle_undo_follow(ac, sender)
+
 
 class LibraryActor(SystemActor):
     id = 'library'
@@ -112,6 +187,62 @@ class LibraryActor(SystemActor):
         'manually_approves_followers': True
     }
 
+    def serialize(self):
+        data = super().serialize()
+        urls = data.setdefault('url', [])
+        urls.append({
+            'type': 'Link',
+            'mediaType': 'application/activity+json',
+            'name': 'library',
+            'href': utils.full_url(reverse('federation:music:files-list'))
+        })
+        return data
+
+    @property
+    def manually_approves_followers(self):
+        return settings.FEDERATION_MUSIC_NEEDS_APPROVAL
+
+    @transaction.atomic
+    def handle_create(self, ac, sender):
+        try:
+            remote_library = models.Library.objects.get(
+                actor=sender,
+                federation_enabled=True,
+            )
+        except models.Library.DoesNotExist:
+            logger.info(
+                'Skipping import, we\'re not following %s', sender.url)
+            return
+
+        if ac['object']['type'] != 'Collection':
+            return
+
+        if ac['object']['totalItems'] <= 0:
+            return
+
+        try:
+            items = ac['object']['items']
+        except KeyError:
+            logger.warning('No items in collection!')
+            return
+
+        item_serializers = [
+            serializers.AudioSerializer(
+                data=i, context={'library': remote_library})
+            for i in items
+        ]
+
+        valid_serializers = []
+        for s in item_serializers:
+            if s.is_valid():
+                valid_serializers.append(s)
+            else:
+                logger.debug(
+                    'Skipping invalid item %s, %s', s.initial_data, s.errors)
+
+        for s in valid_serializers:
+            s.save()
+
 
 class TestActor(SystemActor):
     id = 'test'
@@ -123,40 +254,24 @@ class TestActor(SystemActor):
     additional_attributes = {
         'manually_approves_followers': False
     }
+    manually_approves_followers = False
 
     def get_outbox(self, data, actor=None):
         return {
-        	"@context": [
+            "@context": [
-        		"https://www.w3.org/ns/activitystreams",
+                "https://www.w3.org/ns/activitystreams",
-        		"https://w3id.org/security/v1",
+                "https://w3id.org/security/v1",
-        		{}
+                {}
-        	],
+            ],
-        	"id": utils.full_url(
+            "id": utils.full_url(
                 reverse(
                     'federation:instance-actors-outbox',
                     kwargs={'actor': self.id})),
-        	"type": "OrderedCollection",
+            "type": "OrderedCollection",
-        	"totalItems": 0,
+            "totalItems": 0,
-        	"orderedItems": []
+            "orderedItems": []
         }
 
-    def post_inbox(self, data, actor=None):
-        if actor is None:
-            raise PermissionDenied('Actor not authenticated')
-
-        serializer = serializers.ActivitySerializer(
-            data=data, context={'actor': actor})
-        serializer.is_valid(raise_exception=True)
-
-        ac = serializer.validated_data
-        logger.info('Received activity on %s inbox', self.id)
-        if ac['type'] == 'Create' and ac['object']['type'] == 'Note':
-            # we received a toot \o/
-            command = self.parse_command(ac['object']['content'])
-            logger.debug('Parsed command: %s', command)
-            if command == 'ping':
-                self.handle_ping(ac, actor)
-
     def parse_command(self, message):
         """
         Remove any links or fancy markup to extract /command from
@@ -168,7 +283,16 @@ class TestActor(SystemActor):
         except IndexError:
             return
 
-    def handle_ping(self, ac, sender):
+    def handle_create(self, ac, sender):
+        if ac['object']['type'] != 'Note':
+            return
+
+        # we received a toot \o/
+        command = self.parse_command(ac['object']['content'])
+        logger.debug('Parsed command: %s', command)
+        if command != 'ping':
+            return
+
         now = timezone.now()
         test_actor = self.get_actor_instance()
         reply_url = 'https://{}/activities/note/{}'.format(
@@ -179,10 +303,10 @@ class TestActor(SystemActor):
         )
         reply_activity = {
             "@context": [
-        		"https://www.w3.org/ns/activitystreams",
+                "https://www.w3.org/ns/activitystreams",
-        		"https://w3id.org/security/v1",
+                "https://w3id.org/security/v1",
-        		{}
+                {}
-        	],
+            ],
             'type': 'Create',
             'actor': test_actor.url,
             'id': '{}/activity'.format(reply_url),
@@ -214,6 +338,43 @@ class TestActor(SystemActor):
             to=[ac['actor']],
             on_behalf_of=test_actor)
 
+    def handle_follow(self, ac, sender):
+        super().handle_follow(ac, sender)
+        # also, we follow back
+        test_actor = self.get_actor_instance()
+        follow_uuid = uuid.uuid4()
+        follow = activity.get_follow(
+            follow_id=follow_uuid,
+            follower=test_actor,
+            followed=sender)
+        activity.deliver(
+            follow,
+            to=[ac['actor']],
+            on_behalf_of=test_actor)
+
+    def handle_undo_follow(self, ac, sender):
+        super().handle_undo_follow(ac, sender)
+        actor = self.get_actor_instance()
+        # we also unfollow the sender, if possible
+        try:
+            follow = models.Follow.objects.get(
+                target=sender,
+                actor=actor,
+            )
+        except models.Follow.DoesNotExist:
+            return
+        undo = activity.get_undo(
+            id=follow.get_federation_url(),
+            actor=actor,
+            object=serializers.FollowSerializer(follow).data,
+        )
+        follow.delete()
+        activity.deliver(
+            undo,
+            to=[sender.url],
+            on_behalf_of=actor)
+
+
 SYSTEM_ACTORS = {
     'library': LibraryActor(),
     'test': TestActor(),

api/funkwhale_api/federation/authentication.py

@@ -7,6 +7,7 @@ from rest_framework import exceptions
 
 from . import actors
 from . import keys
+from . import models
 from . import serializers
 from . import signing
 from . import utils
@@ -42,11 +43,16 @@ class SignatureAuthentication(authentication.BaseAuthentication):
         except cryptography.exceptions.InvalidSignature:
             raise exceptions.AuthenticationFailed('Invalid signature')
 
-        return serializer.build()
+        try:
+            return models.Actor.objects.get(url=actor_data['id'])
+        except models.Actor.DoesNotExist:
+            return serializer.save()
 
     def authenticate(self, request):
         setattr(request, 'actor', None)
         actor = self.authenticate_actor(request)
+        if not actor:
+            return
         user = AnonymousUser()
         setattr(request, 'actor', actor)
         return (user, None)

api/funkwhale_api/federation/dynamic_preferences_registry.py

@@ -4,31 +4,3 @@ from dynamic_preferences import types
 from dynamic_preferences.registries import global_preferences_registry
 
 federation = types.Section('federation')
-
-
-@global_preferences_registry.register
-class FederationPrivateKey(types.StringPreference):
-    show_in_api = False
-    section = federation
-    name = 'private_key'
-    default = ''
-    help_text = (
-        'Instance private key, used for signing federation HTTP requests'
-    )
-    verbose_name = (
-        'Instance private key (keep it secret, do not change it)'
-    )
-
-
-@global_preferences_registry.register
-class FederationPublicKey(types.StringPreference):
-    show_in_api = False
-    section = federation
-    name = 'public_key'
-    default = ''
-    help_text = (
-        'Instance public key, used for signing federation HTTP requests'
-    )
-    verbose_name = (
-        'Instance public key (do not change it)'
-    )

api/funkwhale_api/federation/factories.py

@@ -1,8 +1,10 @@
 import factory
 import requests
 import requests_http_signature
+import uuid
 
 from django.utils import timezone
+from django.conf import settings
 
 from funkwhale_api.factories import registry
 
@@ -51,9 +53,23 @@ class SignedRequestFactory(factory.Factory):
         self.headers.update(default_headers)
 
 
+@registry.register(name='federation.Link')
+class LinkFactory(factory.Factory):
+    type = 'Link'
+    href = factory.Faker('url')
+    mediaType = 'text/html'
+
+    class Meta:
+        model = dict
+
+    class Params:
+        audio = factory.Trait(
+            mediaType=factory.Iterator(['audio/mp3', 'audio/ogg'])
+        )
+
+
 @registry.register
 class ActorFactory(factory.DjangoModelFactory):
-
     public_key = None
     private_key = None
     preferred_username = factory.Faker('user_name')
@@ -66,6 +82,12 @@ class ActorFactory(factory.DjangoModelFactory):
     class Meta:
         model = models.Actor
 
+    class Params:
+        local = factory.Trait(
+            domain=factory.LazyAttribute(
+                lambda o: settings.FEDERATION_HOSTNAME)
+        )
+
     @classmethod
     def _generate(cls, create, attrs):
         has_public = attrs.get('public_key') is not None
@@ -77,6 +99,102 @@ class ActorFactory(factory.DjangoModelFactory):
         return super()._generate(create, attrs)
 
 
+@registry.register
+class FollowFactory(factory.DjangoModelFactory):
+    target = factory.SubFactory(ActorFactory)
+    actor = factory.SubFactory(ActorFactory)
+
+    class Meta:
+        model = models.Follow
+
+    class Params:
+        local = factory.Trait(
+            actor=factory.SubFactory(ActorFactory, local=True)
+        )
+
+
+@registry.register
+class FollowRequestFactory(factory.DjangoModelFactory):
+    target = factory.SubFactory(ActorFactory)
+    actor = factory.SubFactory(ActorFactory)
+
+    class Meta:
+        model = models.FollowRequest
+
+
+@registry.register
+class LibraryFactory(factory.DjangoModelFactory):
+    actor = factory.SubFactory(ActorFactory)
+    url = factory.Faker('url')
+    federation_enabled = True
+    download_files = False
+    autoimport = False
+
+    class Meta:
+        model = models.Library
+
+
+class ArtistMetadataFactory(factory.Factory):
+    name = factory.Faker('name')
+
+    class Meta:
+        model = dict
+
+    class Params:
+        musicbrainz = factory.Trait(
+            musicbrainz_id=factory.Faker('uuid4')
+        )
+
+
+class ReleaseMetadataFactory(factory.Factory):
+    title = factory.Faker('sentence')
+
+    class Meta:
+        model = dict
+
+    class Params:
+        musicbrainz = factory.Trait(
+            musicbrainz_id=factory.Faker('uuid4')
+        )
+
+
+class RecordingMetadataFactory(factory.Factory):
+    title = factory.Faker('sentence')
+
+    class Meta:
+        model = dict
+
+    class Params:
+        musicbrainz = factory.Trait(
+            musicbrainz_id=factory.Faker('uuid4')
+        )
+
+
+@registry.register(name='federation.LibraryTrackMetadata')
+class LibraryTrackMetadataFactory(factory.Factory):
+    artist = factory.SubFactory(ArtistMetadataFactory)
+    recording = factory.SubFactory(RecordingMetadataFactory)
+    release = factory.SubFactory(ReleaseMetadataFactory)
+
+    class Meta:
+        model = dict
+
+
+@registry.register
+class LibraryTrackFactory(factory.DjangoModelFactory):
+    library = factory.SubFactory(LibraryFactory)
+    url = factory.Faker('url')
+    title = factory.Faker('sentence')
+    artist_name = factory.Faker('sentence')
+    album_title = factory.Faker('sentence')
+    audio_url = factory.Faker('url')
+    audio_mimetype = 'audio/ogg'
+    metadata = factory.SubFactory(LibraryTrackMetadataFactory)
+
+    class Meta:
+        model = models.LibraryTrack
+
+
 @registry.register(name='federation.Note')
 class NoteFactory(factory.Factory):
     type = 'Note'
@@ -89,3 +207,51 @@ class NoteFactory(factory.Factory):
 
     class Meta:
         model = dict
+
+
+@registry.register(name='federation.Activity')
+class ActivityFactory(factory.Factory):
+    type = 'Create'
+    id = factory.Faker('url')
+    published = factory.LazyFunction(
+        lambda: timezone.now().isoformat()
+    )
+    actor = factory.Faker('url')
+    object = factory.SubFactory(
+        NoteFactory,
+        actor=factory.SelfAttribute('..actor'),
+        published=factory.SelfAttribute('..published'))
+
+    class Meta:
+        model = dict
+
+
+@registry.register(name='federation.AudioMetadata')
+class AudioMetadataFactory(factory.Factory):
+    recording = factory.LazyAttribute(
+        lambda o: 'https://musicbrainz.org/recording/{}'.format(uuid.uuid4())
+    )
+    artist = factory.LazyAttribute(
+        lambda o: 'https://musicbrainz.org/artist/{}'.format(uuid.uuid4())
+    )
+    release = factory.LazyAttribute(
+        lambda o: 'https://musicbrainz.org/release/{}'.format(uuid.uuid4())
+    )
+
+    class Meta:
+        model = dict
+
+
+@registry.register(name='federation.Audio')
+class AudioFactory(factory.Factory):
+    type = 'Audio'
+    id = factory.Faker('url')
+    published = factory.LazyFunction(
+        lambda: timezone.now().isoformat()
+    )
+    actor = factory.Faker('url')
+    url = factory.SubFactory(LinkFactory, audio=True)
+    metadata = factory.SubFactory(LibraryTrackMetadataFactory)
+
+    class Meta:
+        model = dict

api/funkwhale_api/federation/management/commands/generate_keys.py

@@ -1,53 +0,0 @@
-from django.core.management.base import BaseCommand, CommandError
-from django.db import transaction
-
-from dynamic_preferences.registries import global_preferences_registry
-
-from funkwhale_api.federation import keys
-
-
-class Command(BaseCommand):
-    help = (
-        'Generate a public/private key pair for your instance,'
-        ' for federation purposes. If a key pair already exists, does nothing.'
-    )
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            '--replace',
-            action='store_true',
-            dest='replace',
-            default=False,
-            help='Replace existing key pair, if any',
-        )
-        parser.add_argument(
-            '--noinput', '--no-input', action='store_false', dest='interactive',
-            help="Do NOT prompt the user for input of any kind.",
-        )
-
-    @transaction.atomic
-    def handle(self, *args, **options):
-        preferences = global_preferences_registry.manager()
-        existing_public = preferences['federation__public_key']
-        existing_private = preferences['federation__public_key']
-
-        if existing_public or existing_private and not options['replace']:
-            raise CommandError(
-                'Keys are already present! '
-                'Replace them with --replace if you know what you are doing.')
-
-        if options['interactive']:
-            message = (
-                'Are you sure you want to do this?\n\n'
-                "Type 'yes' to continue, or 'no' to cancel: "
-            )
-            if input(''.join(message)) != 'yes':
-                raise CommandError("Operation cancelled.")
-        private, public = keys.get_key_pair()
-        preferences['federation__public_key'] = public.decode('utf-8')
-        preferences['federation__private_key'] = private.decode('utf-8')
-
-        self.stdout.write(
-            'Your new key pair was generated.'
-            'Your public key is now:\n\n{}'.format(public.decode('utf-8'))
-        )

api/funkwhale_api/federation/migrations/0002_auto_20180403_1620.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.0.3 on 2018-04-03 16:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='actor',
+            unique_together={('domain', 'preferred_username')},
+        ),
+    ]

api/funkwhale_api/federation/migrations/0003_auto_20180407_1010.py

@@ -0,0 +1,94 @@
+# Generated by Django 2.0.3 on 2018-04-07 10:10
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+def delete_system_actors(apps, schema_editor):
+    """Revert site domain and name to default."""
+    Actor = apps.get_model("federation", "Actor")
+    Actor.objects.filter(preferred_username__in=['test', 'library']).delete()
+
+
+def backward(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0002_auto_20180403_1620'),
+    ]
+
+    operations = [
+        migrations.RunPython(delete_system_actors, backward),
+        migrations.CreateModel(
+            name='Follow',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('modification_date', models.DateTimeField(auto_now=True)),
+                ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='emitted_follows', to='federation.Actor')),
+                ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='received_follows', to='federation.Actor')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='FollowRequest',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('modification_date', models.DateTimeField(auto_now=True)),
+                ('approved', models.NullBooleanField(default=None)),
+                ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='emmited_follow_requests', to='federation.Actor')),
+                ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='received_follow_requests', to='federation.Actor')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Library',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('modification_date', models.DateTimeField(auto_now=True)),
+                ('fetched_date', models.DateTimeField(blank=True, null=True)),
+                ('uuid', models.UUIDField(default=uuid.uuid4)),
+                ('url', models.URLField()),
+                ('federation_enabled', models.BooleanField()),
+                ('download_files', models.BooleanField()),
+                ('autoimport', models.BooleanField()),
+                ('tracks_count', models.PositiveIntegerField(blank=True, null=True)),
+                ('actor', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='library', to='federation.Actor')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='LibraryTrack',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('url', models.URLField(unique=True)),
+                ('audio_url', models.URLField()),
+                ('audio_mimetype', models.CharField(max_length=200)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('modification_date', models.DateTimeField(auto_now=True)),
+                ('fetched_date', models.DateTimeField(blank=True, null=True)),
+                ('published_date', models.DateTimeField(blank=True, null=True)),
+                ('artist_name', models.CharField(max_length=500)),
+                ('album_title', models.CharField(max_length=500)),
+                ('title', models.CharField(max_length=500)),
+                ('metadata', django.contrib.postgres.fields.jsonb.JSONField(default={}, max_length=10000)),
+                ('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tracks', to='federation.Library')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='actor',
+            name='followers',
+            field=models.ManyToManyField(related_name='following', through='federation.Follow', to='federation.Actor'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='follow',
+            unique_together={('actor', 'target')},
+        ),
+    ]

api/funkwhale_api/federation/models.py

@@ -1,4 +1,7 @@
+import uuid
+
 from django.conf import settings
+from django.contrib.postgres.fields import JSONField
 from django.db import models
 from django.utils import timezone
 
@@ -12,6 +15,8 @@ TYPE_CHOICES = [
 
 
 class Actor(models.Model):
+    ap_type = 'Actor'
+
     url = models.URLField(unique=True, max_length=500, db_index=True)
     outbox_url = models.URLField(max_length=500)
     inbox_url = models.URLField(max_length=500)
@@ -31,6 +36,16 @@ class Actor(models.Model):
     last_fetch_date = models.DateTimeField(
         default=timezone.now)
     manually_approves_followers = models.NullBooleanField(default=None)
+    followers = models.ManyToManyField(
+        to='self',
+        symmetrical=False,
+        through='Follow',
+        through_fields=('target', 'actor'),
+        related_name='following',
+    )
+
+    class Meta:
+        unique_together = ['domain', 'preferred_username']
 
     @property
     def webfinger_subject(self):
@@ -57,3 +72,127 @@ class Actor(models.Model):
                 setattr(self, field, v.lower())
 
         super().save(**kwargs)
+
+    @property
+    def is_local(self):
+        return self.domain == settings.FEDERATION_HOSTNAME
+
+    @property
+    def is_system(self):
+        from . import actors
+        return all([
+            settings.FEDERATION_HOSTNAME == self.domain,
+            self.preferred_username in actors.SYSTEM_ACTORS
+        ])
+
+    @property
+    def system_conf(self):
+        from . import actors
+        if self.is_system:
+            return actors.SYSTEM_ACTORS[self.preferred_username]
+
+
+class Follow(models.Model):
+    ap_type = 'Follow'
+
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    actor = models.ForeignKey(
+        Actor,
+        related_name='emitted_follows',
+        on_delete=models.CASCADE,
+    )
+    target = models.ForeignKey(
+        Actor,
+        related_name='received_follows',
+        on_delete=models.CASCADE,
+    )
+    creation_date = models.DateTimeField(default=timezone.now)
+    modification_date = models.DateTimeField(
+        auto_now=True)
+
+    class Meta:
+        unique_together = ['actor', 'target']
+
+    def get_federation_url(self):
+        return '{}#follows/{}'.format(self.actor.url, self.uuid)
+
+
+class FollowRequest(models.Model):
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    actor = models.ForeignKey(
+        Actor,
+        related_name='emmited_follow_requests',
+        on_delete=models.CASCADE,
+    )
+    target = models.ForeignKey(
+        Actor,
+        related_name='received_follow_requests',
+        on_delete=models.CASCADE,
+    )
+    creation_date = models.DateTimeField(default=timezone.now)
+    modification_date = models.DateTimeField(
+        auto_now=True)
+    approved = models.NullBooleanField(default=None)
+
+    def approve(self):
+        from . import activity
+        from . import serializers
+        self.approved = True
+        self.save(update_fields=['approved'])
+        Follow.objects.get_or_create(
+            target=self.target,
+            actor=self.actor
+        )
+        if self.target.is_local:
+            follow = {
+                '@context': serializers.AP_CONTEXT,
+                'actor': self.actor.url,
+                'id': self.actor.url + '#follows/{}'.format(uuid.uuid4()),
+                'object': self.target.url,
+                'type': 'Follow'
+            }
+            activity.accept_follow(
+                self.target, follow, self.actor
+            )
+
+    def refuse(self):
+        self.approved = False
+        self.save(update_fields=['approved'])
+
+
+class Library(models.Model):
+    creation_date = models.DateTimeField(default=timezone.now)
+    modification_date = models.DateTimeField(
+        auto_now=True)
+    fetched_date = models.DateTimeField(null=True, blank=True)
+    actor = models.OneToOneField(
+        Actor,
+        on_delete=models.CASCADE,
+        related_name='library')
+    uuid = models.UUIDField(default=uuid.uuid4)
+    url = models.URLField()
+
+    # use this flag to disable federation with a library
+    federation_enabled = models.BooleanField()
+    # should we mirror files locally or hotlink them?
+    download_files = models.BooleanField()
+    # should we automatically import new files from this library?
+    autoimport = models.BooleanField()
+    tracks_count = models.PositiveIntegerField(null=True, blank=True)
+
+
+class LibraryTrack(models.Model):
+    url = models.URLField(unique=True)
+    audio_url = models.URLField()
+    audio_mimetype = models.CharField(max_length=200)
+    creation_date = models.DateTimeField(default=timezone.now)
+    modification_date = models.DateTimeField(
+        auto_now=True)
+    fetched_date = models.DateTimeField(null=True, blank=True)
+    published_date = models.DateTimeField(null=True, blank=True)
+    library = models.ForeignKey(
+        Library, related_name='tracks', on_delete=models.CASCADE)
+    artist_name = models.CharField(max_length=500)
+    album_title = models.CharField(max_length=500)
+    title = models.CharField(max_length=500)
+    metadata = JSONField(default={}, max_length=10000)

api/funkwhale_api/federation/permissions.py

@@ -0,0 +1,19 @@
+from django.conf import settings
+
+from rest_framework.permissions import BasePermission
+
+from . import actors
+
+
+class LibraryFollower(BasePermission):
+
+    def has_permission(self, request, view):
+        if not settings.FEDERATION_MUSIC_NEEDS_APPROVAL:
+            return True
+
+        actor = getattr(request, 'actor', None)
+        if actor is None:
+            return False
+
+        library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+        return library.followers.filter(url=actor.url).exists()

api/funkwhale_api/federation/serializers.py

@@ -2,15 +2,25 @@ import urllib.parse
 
 from django.urls import reverse
 from django.conf import settings
+from django.core.paginator import Paginator
+from django.db import transaction
 
 from rest_framework import serializers
 from dynamic_preferences.registries import global_preferences_registry
 
+from funkwhale_api.common.utils import set_query_parameter
+
 from . import activity
 from . import models
 from . import utils
 
 
+AP_CONTEXT = [
+    'https://www.w3.org/ns/activitystreams',
+    'https://w3id.org/security/v1',
+    {},
+]
+
 class ActorSerializer(serializers.ModelSerializer):
     # left maps to activitypub fields, right to our internal models
     id = serializers.URLField(source='url')
@@ -43,11 +53,7 @@ class ActorSerializer(serializers.ModelSerializer):
 
     def to_representation(self, instance):
         ret = super().to_representation(instance)
-        ret['@context'] = [
+        ret['@context'] = AP_CONTEXT
-            'https://www.w3.org/ns/activitystreams',
-            'https://w3id.org/security/v1',
-            {},
-        ]
         if instance.public_key:
             ret['publicKey'] = {
                 'owner': instance.url,
@@ -87,6 +93,28 @@ class ActorSerializer(serializers.ModelSerializer):
             return value[:500]
 
 
+class FollowSerializer(serializers.ModelSerializer):
+    # left maps to activitypub fields, right to our internal models
+    id = serializers.URLField(source='get_federation_url')
+    object = serializers.URLField(source='target.url')
+    actor = serializers.URLField(source='actor.url')
+    type = serializers.CharField(source='ap_type')
+
+    class Meta:
+        model = models.Actor
+        fields = [
+            'id',
+            'object',
+            'actor',
+            'type'
+        ]
+
+    def to_representation(self, instance):
+        ret = super().to_representation(instance)
+        ret['@context'] = AP_CONTEXT
+        return ret
+
+
 class ActorWebfingerSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.Actor
@@ -120,7 +148,9 @@ class ActivitySerializer(serializers.Serializer):
             type = value['type']
         except KeyError:
             raise serializers.ValidationError('Missing object type')
-
+        except TypeError:
+            # probably a URL
+            return value
         try:
             object_serializer = OBJECT_SERIALIZERS[type]
         except KeyError:
@@ -173,3 +203,212 @@ OBJECT_SERIALIZERS = {
     t: ObjectSerializer
     for t in activity.OBJECT_TYPES
 }
+
+
+class PaginatedCollectionSerializer(serializers.Serializer):
+
+    def to_representation(self, conf):
+        paginator = Paginator(
+            conf['items'],
+            conf.get('page_size', 20)
+        )
+        first = set_query_parameter(conf['id'], page=1)
+        current = first
+        last = set_query_parameter(conf['id'], page=paginator.num_pages)
+        d = {
+            'id': conf['id'],
+            'actor': conf['actor'].url,
+            'totalItems': paginator.count,
+            'type': 'Collection',
+            'current': current,
+            'first': first,
+            'last': last,
+        }
+        if self.context.get('include_ap_context', True):
+            d['@context'] = AP_CONTEXT
+        return d
+
+
+class CollectionPageSerializer(serializers.Serializer):
+
+    def to_representation(self, conf):
+        page = conf['page']
+        first = set_query_parameter(conf['id'], page=1)
+        last = set_query_parameter(conf['id'], page=page.paginator.num_pages)
+        id = set_query_parameter(conf['id'], page=page.number)
+        d = {
+            'id': id,
+            'partOf': conf['id'],
+            'actor': conf['actor'].url,
+            'totalItems': page.paginator.count,
+            'type': 'CollectionPage',
+            'first': first,
+            'last': last,
+            'items': [
+                conf['item_serializer'](
+                    i,
+                    context={
+                        'actor': conf['actor'],
+                        'include_ap_context': False}
+                ).data
+                for i in page.object_list
+            ]
+        }
+
+        if page.has_previous():
+            d['prev'] = set_query_parameter(
+                conf['id'], page=page.previous_page_number())
+
+        if page.has_previous():
+            d['next'] = set_query_parameter(
+                conf['id'], page=page.next_page_number())
+
+        if self.context.get('include_ap_context', True):
+            d['@context'] = AP_CONTEXT
+        return d
+
+
+class ArtistMetadataSerializer(serializers.Serializer):
+    musicbrainz_id = serializers.UUIDField(required=False)
+    name = serializers.CharField()
+
+
+class ReleaseMetadataSerializer(serializers.Serializer):
+    musicbrainz_id = serializers.UUIDField(required=False)
+    title = serializers.CharField()
+
+
+class RecordingMetadataSerializer(serializers.Serializer):
+    musicbrainz_id = serializers.UUIDField(required=False)
+    title = serializers.CharField()
+
+
+class AudioMetadataSerializer(serializers.Serializer):
+    artist = ArtistMetadataSerializer()
+    release = ReleaseMetadataSerializer()
+    recording = RecordingMetadataSerializer()
+
+
+class AudioSerializer(serializers.Serializer):
+    type = serializers.CharField()
+    id = serializers.URLField()
+    url = serializers.JSONField()
+    published = serializers.DateTimeField()
+    updated = serializers.DateTimeField(required=False)
+    metadata = AudioMetadataSerializer()
+
+    def validate_type(self, v):
+        if v != 'Audio':
+            raise serializers.ValidationError('Invalid type for audio')
+        return v
+
+    def validate_url(self, v):
+        try:
+            url = v['href']
+        except (KeyError, TypeError):
+            raise serializers.ValidationError('Missing href')
+
+        try:
+            media_type = v['mediaType']
+        except (KeyError, TypeError):
+            raise serializers.ValidationError('Missing mediaType')
+
+        if not media_type.startswith('audio/'):
+            raise serializers.ValidationError('Invalid mediaType')
+
+        return url
+
+    def validate_url(self, v):
+        try:
+            url = v['href']
+        except (KeyError, TypeError):
+            raise serializers.ValidationError('Missing href')
+
+        try:
+            media_type = v['mediaType']
+        except (KeyError, TypeError):
+            raise serializers.ValidationError('Missing mediaType')
+
+        if not media_type.startswith('audio/'):
+            raise serializers.ValidationError('Invalid mediaType')
+
+        return v
+
+    def create(self, validated_data):
+        defaults = {
+            'audio_mimetype': validated_data['url']['mediaType'],
+            'audio_url': validated_data['url']['href'],
+            'metadata': validated_data['metadata'],
+            'artist_name': validated_data['metadata']['artist']['name'],
+            'album_title': validated_data['metadata']['release']['title'],
+            'title': validated_data['metadata']['recording']['title'],
+            'published_date': validated_data['published'],
+            'modification_date': validated_data.get('updated'),
+        }
+        return models.LibraryTrack.objects.get_or_create(
+            library=self.context['library'],
+            url=validated_data['id'],
+            defaults=defaults
+        )[0]
+
+    def to_representation(self, instance):
+        track = instance.track
+        album = instance.track.album
+        artist = instance.track.artist
+
+        d = {
+            'type': 'Audio',
+            'id': instance.get_federation_url(),
+            'name': instance.track.full_name,
+            'published': instance.creation_date.isoformat(),
+            'updated': instance.modification_date.isoformat(),
+            'metadata': {
+                'artist': {
+                    'musicbrainz_id': str(artist.mbid) if artist.mbid else None,
+                    'name': artist.name,
+                },
+                'release': {
+                    'musicbrainz_id': str(album.mbid) if album.mbid else None,
+                    'title': album.title,
+                },
+                'recording': {
+                    'musicbrainz_id': str(track.mbid) if track.mbid else None,
+                    'title': track.title,
+                },
+            },
+            'url': {
+                'href': utils.full_url(instance.path),
+                'type': 'Link',
+                'mediaType': instance.mimetype
+            },
+            'attributedTo': [
+                self.context['actor'].url
+            ]
+        }
+        if self.context.get('include_ap_context', True):
+            d['@context'] = AP_CONTEXT
+        return d
+
+
+class CollectionSerializer(serializers.Serializer):
+
+    def to_representation(self, conf):
+        d = {
+            'id': conf['id'],
+            'actor': conf['actor'].url,
+            'totalItems': len(conf['items']),
+            'type': 'Collection',
+            'items': [
+                conf['item_serializer'](
+                    i,
+                    context={
+                        'actor': conf['actor'],
+                        'include_ap_context': False}
+                ).data
+                for i in conf['items']
+            ]
+        }
+
+        if self.context.get('include_ap_context', True):
+            d['@context'] = AP_CONTEXT
+        return d

api/funkwhale_api/federation/signing.py

@@ -53,3 +53,18 @@ def verify_django(django_request, public_key):
             request.headers[h] = str(v)
     prepared_request = request.prepare()
     return verify(request, public_key)
+
+
+def get_auth(private_key, private_key_id):
+    return requests_http_signature.HTTPSignatureAuth(
+        use_auth_header=False,
+        headers=[
+            '(request-target)',
+            'user-agent',
+            'host',
+            'date',
+            'content-type'],
+        algorithm='rsa-sha256',
+        key=private_key.encode('utf-8'),
+        key_id=private_key_id,
+    )

api/funkwhale_api/federation/urls.py

@@ -1,8 +1,10 @@
-from rest_framework import routers
+from django.conf.urls import include, url
 
+from rest_framework import routers
 from . import views
 
 router = routers.SimpleRouter(trailing_slash=False)
+music_router = routers.SimpleRouter(trailing_slash=False)
 router.register(
     r'federation/instance/actors',
     views.InstanceActorViewSet,
@@ -12,4 +14,11 @@ router.register(
     views.WellKnownViewSet,
     'well-known')
 
-urlpatterns = router.urls
+music_router.register(
+    r'files',
+    views.MusicFilesViewSet,
+    'files',
+)
+urlpatterns = router.urls + [
+    url('federation/music/', include((music_router.urls, 'music'), namespace='music'))
+]

api/funkwhale_api/federation/views.py

@@ -1,16 +1,22 @@
 from django import forms
 from django.conf import settings
+from django.core import paginator
 from django.http import HttpResponse
+from django.urls import reverse
 
 from rest_framework import viewsets
 from rest_framework import views
 from rest_framework import response
 from rest_framework.decorators import list_route, detail_route
 
+from funkwhale_api.music.models import TrackFile
+
 from . import actors
 from . import authentication
+from . import permissions
 from . import renderers
 from . import serializers
+from . import utils
 from . import webfinger
 
 
@@ -38,8 +44,8 @@ class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet):
     def retrieve(self, request, *args, **kwargs):
         system_actor = self.get_object()
         actor = system_actor.get_actor_instance()
-        serializer = serializers.ActorSerializer(actor)
+        data = actor.system_conf.serialize()
-        return response.Response(serializer.data, status=200)
+        return response.Response(data, status=200)
 
     @detail_route(methods=['get', 'post'])
     def inbox(self, request, *args, **kwargs):
@@ -101,3 +107,50 @@ class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet):
         username, hostname = clean_result
         actor = actors.SYSTEM_ACTORS[username].get_actor_instance()
         return serializers.ActorWebfingerSerializer(actor).data
+
+
+class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet):
+    authentication_classes = [
+        authentication.SignatureAuthentication]
+    permission_classes = [permissions.LibraryFollower]
+    renderer_classes = [renderers.ActivityPubRenderer]
+
+    def list(self, request, *args, **kwargs):
+        page = request.GET.get('page')
+        library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+        qs = TrackFile.objects.order_by('-creation_date').select_related(
+            'track__artist',
+            'track__album__artist'
+        )
+        if page is None:
+            conf = {
+                'id': utils.full_url(reverse('federation:music:files-list')),
+                'page_size': settings.FEDERATION_COLLECTION_PAGE_SIZE,
+                'items': qs,
+                'item_serializer': serializers.AudioSerializer,
+                'actor': library,
+            }
+            serializer = serializers.PaginatedCollectionSerializer(conf)
+            data = serializer.data
+        else:
+            try:
+                page_number = int(page)
+            except:
+                return response.Response(
+                    {'page': ['Invalid page number']}, status=400)
+            p = paginator.Paginator(
+                qs, settings.FEDERATION_COLLECTION_PAGE_SIZE)
+            try:
+                page = p.page(page_number)
+                conf = {
+                    'id': utils.full_url(reverse('federation:music:files-list')),
+                    'page': page,
+                    'item_serializer': serializers.AudioSerializer,
+                    'actor': library,
+                }
+                serializer = serializers.CollectionPageSerializer(conf)
+                data = serializer.data
+            except paginator.EmptyPage:
+                return response.Response(status=404)
+
+        return response.Response(data)

api/funkwhale_api/music/factories.py

@@ -2,6 +2,9 @@ import factory
 import os
 
 from funkwhale_api.factories import registry, ManyToManyFromList
+from funkwhale_api.federation.factories import (
+    LibraryTrackFactory,
+)
 from funkwhale_api.users.factories import UserFactory
 
 SAMPLES_PATH = os.path.join(
@@ -53,6 +56,18 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = 'music.TrackFile'
 
+    class Params:
+        federation = factory.Trait(
+            audio_file=None,
+            library_track=factory.SubFactory(LibraryTrackFactory),
+            mimetype=factory.LazyAttribute(
+                lambda o: o.library_track.audio_mimetype
+            ),
+            source=factory.LazyAttribute(
+                lambda o: o.library_track.audio_url
+            ),
+        )
+
 
 @registry.register
 class ImportBatchFactory(factory.django.DjangoModelFactory):
@@ -61,6 +76,12 @@ class ImportBatchFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = 'music.ImportBatch'
 
+    class Params:
+        federation = factory.Trait(
+            submitted_by=None,
+            source='federation',
+        )
+
 
 @registry.register
 class ImportJobFactory(factory.django.DjangoModelFactory):
@@ -71,6 +92,13 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = 'music.ImportJob'
 
+    class Params:
+        federation = factory.Trait(
+            mbid=None,
+            library_track=factory.SubFactory(LibraryTrackFactory),
+            batch=factory.SubFactory(ImportBatchFactory, federation=True),
+        )
+
 
 @registry.register(name='music.FileImportJob')
 class FileImportJobFactory(ImportJobFactory):

api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py

@@ -0,0 +1,88 @@
+# Generated by Django 2.0.3 on 2018-04-07 10:10
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0003_auto_20180407_1010'),
+        ('music', '0022_importbatch_import_request'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='album',
+            name='uuid',
+            field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
+        ),
+        migrations.AddField(
+            model_name='artist',
+            name='uuid',
+            field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
+        ),
+        migrations.AddField(
+            model_name='importbatch',
+            name='uuid',
+            field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
+        ),
+        migrations.AddField(
+            model_name='importjob',
+            name='library_track',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='import_jobs', to='federation.LibraryTrack'),
+        ),
+        migrations.AddField(
+            model_name='importjob',
+            name='uuid',
+            field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
+        ),
+        migrations.AddField(
+            model_name='lyrics',
+            name='uuid',
+            field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
+        ),
+        migrations.AddField(
+            model_name='track',
+            name='uuid',
+            field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
+        ),
+        migrations.AddField(
+            model_name='trackfile',
+            name='creation_date',
+            field=models.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.AddField(
+            model_name='trackfile',
+            name='library_track',
+            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='local_track_file', to='federation.LibraryTrack'),
+        ),
+        migrations.AddField(
+            model_name='trackfile',
+            name='modification_date',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='trackfile',
+            name='uuid',
+            field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
+        ),
+        migrations.AddField(
+            model_name='work',
+            name='uuid',
+            field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='importbatch',
+            name='source',
+            field=models.CharField(choices=[('api', 'api'), ('shell', 'shell'), ('federation', 'federation')], default='api', max_length=30),
+        ),
+        migrations.AlterField(
+            model_name='importbatch',
+            name='submitted_by',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to=settings.AUTH_USER_MODEL),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -5,6 +5,7 @@ import datetime
 import tempfile
 import shutil
 import markdown
+import uuid
 
 from django.conf import settings
 from django.db import models
@@ -20,12 +21,15 @@ from versatileimagefield.fields import VersatileImageField
 
 from funkwhale_api import downloader
 from funkwhale_api import musicbrainz
+from funkwhale_api.federation import utils as federation_utils
 from . import importers
 from . import utils
 
 
 class APIModelMixin(models.Model):
     mbid = models.UUIDField(unique=True, db_index=True, null=True, blank=True)
+    uuid = models.UUIDField(
+        unique=True, db_index=True, default=uuid.uuid4)
     api_includes = []
     creation_date = models.DateTimeField(default=timezone.now)
     import_hooks = []
@@ -65,6 +69,13 @@ class APIModelMixin(models.Model):
                 pass
         return cleaned_data
 
+    @property
+    def musicbrainz_url(self):
+        if self.mbid:
+            return 'https://musicbrainz.org/{}/{}'.format(
+                self.musicbrainz_model, self.mbid)
+
+
 class Artist(APIModelMixin):
     name = models.CharField(max_length=255)
 
@@ -90,10 +101,19 @@ class Artist(APIModelMixin):
                 t.append(tag)
         return set(t)
 
+    @classmethod
+    def get_or_create_from_name(cls, name, **kwargs):
+        kwargs.update({'name': name})
+        return cls.objects.get_or_create(
+            name__iexact=name,
+            defaults=kwargs)[0]
+
+
 def import_artist(v):
     a = Artist.get_or_create_from_api(mbid=v[0]['artist']['id'])[0]
     return a
 
+
 def parse_date(v):
     if len(v) == 4:
         return datetime.date(int(v), 1, 1)
@@ -108,6 +128,7 @@ def import_tracks(instance, cleaned_data, raw_data):
         track_cleaned_data['position'] = int(track_data['position'])
         track = importers.load(Track, track_cleaned_data, track_data, Track.import_hooks)
 
+
 class Album(APIModelMixin):
     title = models.CharField(max_length=255)
     artist = models.ForeignKey(
@@ -170,6 +191,14 @@ class Album(APIModelMixin):
                 t.append(tag)
         return set(t)
 
+    @classmethod
+    def get_or_create_from_title(cls, title, **kwargs):
+        kwargs.update({'title': title})
+        return cls.objects.get_or_create(
+            title__iexact=title,
+            defaults=kwargs)[0]
+
+
 def import_tags(instance, cleaned_data, raw_data):
     MINIMUM_COUNT = 2
     tags_to_add = []
@@ -182,6 +211,7 @@ def import_tags(instance, cleaned_data, raw_data):
         tags_to_add.append(tag_data['name'])
     instance.tags.add(*tags_to_add)
 
+
 def import_album(v):
     a = Album.get_or_create_from_api(mbid=v[0]['id'])[0]
     return a
@@ -248,6 +278,8 @@ class Work(APIModelMixin):
 
 
 class Lyrics(models.Model):
+    uuid = models.UUIDField(
+        unique=True, db_index=True, default=uuid.uuid4)
     work = models.ForeignKey(
         Work,
         related_name='lyrics',
@@ -328,7 +360,7 @@ class Track(APIModelMixin):
     def save(self, **kwargs):
         try:
             self.artist
-        except  Artist.DoesNotExist:
+        except Artist.DoesNotExist:
             self.artist = self.album.artist
         super().save(**kwargs)
 
@@ -366,16 +398,35 @@ class Track(APIModelMixin):
                 self.mbid)
         return settings.FUNKWHALE_URL + '/tracks/{}'.format(self.pk)
 
+    @classmethod
+    def get_or_create_from_title(cls, title, **kwargs):
+        kwargs.update({'title': title})
+        return cls.objects.get_or_create(
+            title__iexact=title,
+            defaults=kwargs)[0]
+
 
 class TrackFile(models.Model):
+    uuid = models.UUIDField(
+        unique=True, db_index=True, default=uuid.uuid4)
     track = models.ForeignKey(
         Track, related_name='files', on_delete=models.CASCADE)
     audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255)
     source = models.URLField(null=True, blank=True)
+    creation_date = models.DateTimeField(default=timezone.now)
+    modification_date = models.DateTimeField(auto_now=True)
     duration = models.IntegerField(null=True, blank=True)
     acoustid_track_id = models.UUIDField(null=True, blank=True)
     mimetype = models.CharField(null=True, blank=True, max_length=200)
 
+    library_track = models.OneToOneField(
+        'federation.LibraryTrack',
+        related_name='local_track_file',
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+    )
+
     def download_file(self):
         # import the track file, since there is not any
         # we create a tmp dir for the download
@@ -391,12 +442,15 @@ class TrackFile(models.Model):
         shutil.rmtree(tmp_dir)
         return self.audio_file
 
+    def get_federation_url(self):
+        return federation_utils.full_url(
+            '/federation/music/file/{}'.format(self.uuid)
+        )
+
     @property
     def path(self):
-        if settings.PROTECT_AUDIO_FILES:
+        return reverse(
-            return reverse(
+            'api:v1:trackfiles-serve', kwargs={'pk': self.pk})
-                'api:v1:trackfiles-serve', kwargs={'pk': self.pk})
-        return self.audio_file.url
 
     @property
     def filename(self):
@@ -417,10 +471,14 @@ IMPORT_STATUS_CHOICES = (
     ('skipped', 'Skipped'),
 )
 
+
 class ImportBatch(models.Model):
+    uuid = models.UUIDField(
+        unique=True, db_index=True, default=uuid.uuid4)
     IMPORT_BATCH_SOURCES = [
         ('api', 'api'),
-        ('shell', 'shell')
+        ('shell', 'shell'),
+        ('federation', 'federation'),
     ]
     source = models.CharField(
         max_length=30, default='api', choices=IMPORT_BATCH_SOURCES)
@@ -428,6 +486,8 @@ class ImportBatch(models.Model):
     submitted_by = models.ForeignKey(
         'users.User',
         related_name='imports',
+        null=True,
+        blank=True,
         on_delete=models.CASCADE)
     status = models.CharField(
         choices=IMPORT_STATUS_CHOICES, default='pending', max_length=30)
@@ -437,6 +497,7 @@ class ImportBatch(models.Model):
         null=True,
         blank=True,
         on_delete=models.CASCADE)
+
     class Meta:
         ordering = ['-creation_date']
 
@@ -449,6 +510,8 @@ class ImportBatch(models.Model):
 
 
 class ImportJob(models.Model):
+    uuid = models.UUIDField(
+        unique=True, db_index=True, default=uuid.uuid4)
     batch = models.ForeignKey(
         ImportBatch, related_name='jobs', on_delete=models.CASCADE)
     track_file = models.ForeignKey(
@@ -465,6 +528,14 @@ class ImportJob(models.Model):
     audio_file = models.FileField(
         upload_to='imports/%Y/%m/%d', max_length=255, null=True, blank=True)
 
+    library_track = models.ForeignKey(
+        'federation.LibraryTrack',
+        related_name='import_jobs',
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True
+    )
+
     class Meta:
         ordering = ('id', )
 

api/funkwhale_api/music/permissions.py

@@ -0,0 +1,23 @@
+from django.conf import settings
+
+from rest_framework.permissions import BasePermission
+
+from funkwhale_api.federation import actors
+
+
+class Listen(BasePermission):
+
+    def has_permission(self, request, view):
+        if not settings.PROTECT_AUDIO_FILES:
+            return True
+
+        user = getattr(request, 'user', None)
+        if user and user.is_authenticated:
+            return True
+
+        actor = getattr(request, 'actor', None)
+        if actor is None:
+            return False
+
+        library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+        return library.followers.filter(url=actor.url).exists()

api/funkwhale_api/music/serializers.py

@@ -1,7 +1,10 @@
+from django.db import transaction
 from rest_framework import serializers
 from taggit.models import Tag
 
 from funkwhale_api.activity import serializers as activity_serializers
+from funkwhale_api.federation.serializers import AP_CONTEXT
+from funkwhale_api.federation import utils as federation_utils
 
 from . import models
 

api/funkwhale_api/music/tasks.py

@@ -25,6 +25,48 @@ def set_acoustid_on_track_file(track_file):
         return update(result['id'])
 
 
+def import_track_from_remote(library_track):
+    metadata = library_track.metadata
+    try:
+        track_mbid = metadata['recording']['musicbrainz_id']
+        assert track_mbid  # for null/empty values
+    except (KeyError, AssertionError):
+        pass
+    else:
+        return models.Track.get_or_create_from_api(mbid=track_mbid)
+
+    try:
+        album_mbid = metadata['release']['musicbrainz_id']
+        assert album_mbid  # for null/empty values
+    except (KeyError, AssertionError):
+        pass
+    else:
+        album = models.Album.get_or_create_from_api(mbid=album_mbid)
+        return models.Track.get_or_create_from_title(
+            library_track.title, artist=album.artist, album=album)
+
+    try:
+        artist_mbid = metadata['artist']['musicbrainz_id']
+        assert artist_mbid  # for null/empty values
+    except (KeyError, AssertionError):
+        pass
+    else:
+        artist = models.Artist.get_or_create_from_api(mbid=artist_mbid)
+        album = models.Album.get_or_create_from_title(
+            library_track.album_title, artist=artist)
+        return models.Track.get_or_create_from_title(
+            library_track.title, artist=artist, album=album)
+
+    # worst case scenario, we have absolutely no way to link to a
+    # musicbrainz resource, we rely on the name/titles
+    artist = models.Artist.get_or_create_from_name(
+        library_track.artist_name)
+    album = models.Album.get_or_create_from_title(
+        library_track.album_title, artist=artist)
+    return models.Track.get_or_create_from_title(
+        library_track.title, artist=artist, album=album)
+
+
 def _do_import(import_job, replace, use_acoustid=True):
     from_file = bool(import_job.audio_file)
     mbid = import_job.mbid
@@ -43,8 +85,14 @@ def _do_import(import_job, replace, use_acoustid=True):
             acoustid_track_id = match['id']
     if mbid:
         track, _ = models.Track.get_or_create_from_api(mbid=mbid)
-    else:
+    elif import_job.audio_file:
         track = import_track_data_from_path(import_job.audio_file.path)
+    elif import_job.library_track:
+        track = import_track_from_remote(import_job.library_track)
+    else:
+        raise ValueError(
+            'Not enough data to process import, '
+            'add a mbid, an audio file or a library track')
 
     track_file = None
     if replace:
@@ -63,6 +111,14 @@ def _do_import(import_job, replace, use_acoustid=True):
         track_file.audio_file = ContentFile(import_job.audio_file.read())
         track_file.audio_file.name = import_job.audio_file.name
         track_file.duration = duration
+    elif import_job.library_track:
+        track_file.library_track = import_job.library_track
+        track_file.mimetype = import_job.library_track.audio_mimetype
+        if import_job.library_track.library.download_files:
+            raise NotImplementedError()
+        else:
+            # no downloading, we hotlink
+            pass
     else:
         track_file.download_file()
     track_file.save()

api/funkwhale_api/music/utils.py

@@ -60,3 +60,10 @@ def compute_status(jobs):
     if pending:
         return 'pending'
     return 'finished'
+
+
+def get_ext_from_type(mimetype):
+    mapping = {
+        'audio/ogg': 'ogg',
+        'audio/mpeg': 'mp3',
+    }

api/funkwhale_api/music/views.py

@@ -1,36 +1,42 @@
 import ffmpeg
 import os
 import json
+import requests
 import subprocess
 import unicodedata
 import urllib
 
-from django.urls import reverse
+from django.contrib.auth.decorators import login_required
+from django.core.exceptions import ObjectDoesNotExist
+from django.conf import settings
 from django.db import models, transaction
 from django.db.models.functions import Length
-from django.conf import settings
 from django.http import StreamingHttpResponse
+from django.urls import reverse
+from django.utils.decorators import method_decorator
 
 from rest_framework import viewsets, views, mixins
 from rest_framework.decorators import detail_route, list_route
 from rest_framework.response import Response
+from rest_framework import settings as rest_settings
 from rest_framework import permissions
 from musicbrainzngs import ResponseError
-from django.contrib.auth.decorators import login_required
-from django.utils.decorators import method_decorator
 
 from funkwhale_api.common import utils as funkwhale_utils
+from funkwhale_api.federation import actors
 from funkwhale_api.requests.models import ImportRequest
 from funkwhale_api.musicbrainz import api
 from funkwhale_api.common.permissions import (
     ConditionalAuthentication, HasModelPermission)
 from taggit.models import Tag
+from funkwhale_api.federation.authentication import SignatureAuthentication
 
-from . import forms
-from . import models
-from . import serializers
-from . import importers
 from . import filters
+from . import forms
+from . import importers
+from . import models
+from . import permissions as music_permissions
+from . import serializers
 from . import tasks
 from . import utils
 
@@ -45,6 +51,7 @@ class SearchMixin(object):
         serializer = self.serializer_class(queryset, many=True)
         return Response(serializer.data)
 
+
 class TagViewSetMixin(object):
 
     def get_queryset(self):
@@ -179,22 +186,54 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
 class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = (models.TrackFile.objects.all().order_by('-id'))
     serializer_class = serializers.TrackFileSerializer
-    permission_classes = [ConditionalAuthentication]
+    authentication_classes = rest_settings.api_settings.DEFAULT_AUTHENTICATION_CLASSES + [
+        SignatureAuthentication
+    ]
+    permission_classes = [music_permissions.Listen]
 
     @detail_route(methods=['get'])
     def serve(self, request, *args, **kwargs):
         try:
-            f = models.TrackFile.objects.get(pk=kwargs['pk'])
+            f = models.TrackFile.objects.select_related(
+                'library_track',
+                'track__album__artist',
+                'track__artist',
+            ).get(pk=kwargs['pk'])
         except models.TrackFile.DoesNotExist:
             return Response(status=404)
 
-        response = Response()
+        mt = f.mimetype
+        try:
+            library_track = f.library_track
+        except ObjectDoesNotExist:
+            library_track = None
+        if library_track and not f.audio_file:
+            # we proxy the response to the remote library
+            # since we did not mirror the file locally
+            mt = library_track.audio_mimetype
+            file_extension = utils.get_ext_from_type(mt)
+            filename = '{}.{}'.format(f.track.full_name, file_extension)
+            auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
+            remote_response = requests.get(
+                library_track.audio_url,
+                auth=auth,
+                stream=True,
+                headers={
+                    'Content-Type': 'application/activity+json'
+                })
+            response = StreamingHttpResponse(remote_response.iter_content())
+        else:
+            response = Response()
+            filename = f.filename
+            response['X-Accel-Redirect'] = "{}{}".format(
+                settings.PROTECT_FILES_PATH,
+                f.audio_file.url)
         filename = "filename*=UTF-8''{}".format(
-            urllib.parse.quote(f.filename))
+            urllib.parse.quote(filename))
         response["Content-Disposition"] = "attachment; {}".format(filename)
-        response['X-Accel-Redirect'] = "{}{}".format(
+        if mt:
-            settings.PROTECT_FILES_PATH,
+            response["Content-Type"] = mt
-            f.audio_file.url)
+
         return response
 
     @list_route(methods=['get'])

api/tests/conftest.py

@@ -162,3 +162,12 @@ def media_root(settings):
 def r_mock():
     with requests_mock.mock() as m:
         yield m
+
+
+@pytest.fixture
+def authenticated_actor(factories, mocker):
+    actor = factories['federation.Actor']()
+    mocker.patch(
+        'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor',
+        return_value=actor)
+    yield actor

api/tests/federation/conftest.py

@@ -1,10 +0,0 @@
-import pytest
-
-
-@pytest.fixture
-def authenticated_actor(nodb_factories, mocker):
-    actor = nodb_factories['federation.Actor']()
-    mocker.patch(
-        'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor',
-        return_value=actor)
-    yield actor

api/tests/federation/test_activity.py

@@ -1,5 +1,8 @@
+import uuid
+
 from funkwhale_api.federation import activity
 
+
 def test_deliver(nodb_factories, r_mock, mocker):
     to = nodb_factories['federation.Actor']()
     mocker.patch(
@@ -30,3 +33,42 @@ def test_deliver(nodb_factories, r_mock, mocker):
     assert r_mock.call_count == 1
     assert request.url == to.inbox_url
     assert request.headers['content-type'] == 'application/activity+json'
+
+
+def test_accept_follow(mocker, factories):
+    deliver = mocker.patch(
+        'funkwhale_api.federation.activity.deliver')
+    actor = factories['federation.Actor']()
+    target = factories['federation.Actor'](local=True)
+    follow = {
+        'actor': actor.url,
+        'type': 'Follow',
+        'id': 'http://test.federation/user#follows/267',
+        'object': target.url,
+    }
+    uid = uuid.uuid4()
+    mocker.patch('uuid.uuid4', return_value=uid)
+    expected_accept = {
+        "@context": [
+            "https://www.w3.org/ns/activitystreams",
+            "https://w3id.org/security/v1",
+            {}
+        ],
+        "id": target.url + '#accepts/follows/{}'.format(uid),
+        "type": "Accept",
+        "actor": target.url,
+        "object": {
+            "id": follow['id'],
+            "type": "Follow",
+            "actor": actor.url,
+            "object": target.url
+        },
+    }
+    activity.accept_follow(
+        target, follow, actor
+    )
+    deliver.assert_called_once_with(
+        expected_accept, to=[actor.url], on_behalf_of=target
+    )
+    follow_instance = actor.emitted_follows.first()
+    assert follow_instance.target == target

api/tests/federation/test_actors.py

@@ -1,4 +1,6 @@
+import arrow
 import pytest
+import uuid
 
 from django.urls import reverse
 from django.utils import timezone
@@ -6,6 +8,7 @@ from django.utils import timezone
 from rest_framework import exceptions
 
 from funkwhale_api.federation import actors
+from funkwhale_api.federation import models
 from funkwhale_api.federation import serializers
 from funkwhale_api.federation import utils
 
@@ -23,14 +26,17 @@ def test_actor_fetching(r_mock):
     assert r == payload
 
 
-def test_get_library(settings, preferences):
+def test_get_library(db, settings, mocker):
-    preferences['federation__public_key'] = 'public_key'
+    get_key_pair = mocker.patch(
+        'funkwhale_api.federation.keys.get_key_pair',
+        return_value=(b'private', b'public'))
     expected = {
         'preferred_username': 'library',
         'domain': settings.FEDERATION_HOSTNAME,
         'type': 'Person',
         'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME),
         'manually_approves_followers': True,
+        'public_key': 'public',
         'url': utils.full_url(
             reverse(
                 'federation:instance-actors-detail',
@@ -47,7 +53,6 @@ def test_get_library(settings, preferences):
             reverse(
                 'federation:instance-actors-outbox',
                 kwargs={'actor': 'library'})),
-        'public_key': 'public_key',
         'summary': 'Bot account to federate with {}\'s library'.format(
         settings.FEDERATION_HOSTNAME),
     }
@@ -56,14 +61,17 @@ def test_get_library(settings, preferences):
         assert getattr(actor, key) == value
 
 
-def test_get_test(settings, preferences):
+def test_get_test(db, mocker, settings):
-    preferences['federation__public_key'] = 'public_key'
+    get_key_pair = mocker.patch(
+        'funkwhale_api.federation.keys.get_key_pair',
+        return_value=(b'private', b'public'))
     expected = {
         'preferred_username': 'test',
         'domain': settings.FEDERATION_HOSTNAME,
         'type': 'Person',
         'name': '{}\'s test account'.format(settings.FEDERATION_HOSTNAME),
         'manually_approves_followers': False,
+        'public_key': 'public',
         'url': utils.full_url(
             reverse(
                 'federation:instance-actors-detail',
@@ -80,7 +88,6 @@ def test_get_test(settings, preferences):
             reverse(
                 'federation:instance-actors-outbox',
                 kwargs={'actor': 'test'})),
-        'public_key': 'public_key',
         'summary': 'Bot account to test federation with {}. Send me /ping and I\'ll answer you.'.format(
         settings.FEDERATION_HOSTNAME),
     }
@@ -91,18 +98,18 @@ def test_get_test(settings, preferences):
 
 def test_test_get_outbox():
     expected = {
-    	"@context": [
+        "@context": [
-    		"https://www.w3.org/ns/activitystreams",
+            "https://www.w3.org/ns/activitystreams",
-    		"https://w3id.org/security/v1",
+            "https://w3id.org/security/v1",
-    		{}
+            {}
-    	],
+        ],
-    	"id": utils.full_url(
+        "id": utils.full_url(
             reverse(
                 'federation:instance-actors-outbox',
                 kwargs={'actor': 'test'})),
-    	"type": "OrderedCollection",
+        "type": "OrderedCollection",
-    	"totalItems": 0,
+        "totalItems": 0,
-    	"orderedItems": []
+        "orderedItems": []
     }
 
     data = actors.SYSTEM_ACTORS['test'].get_outbox({}, actor=None)
@@ -126,7 +133,7 @@ def test_test_post_outbox_validates_actor(nodb_factories):
         assert msg in exc_info.value
 
 
-def test_test_post_outbox_handles_create_note(
+def test_test_post_inbox_handles_create_note(
         settings, mocker, factories):
     deliver = mocker.patch(
         'funkwhale_api.federation.activity.deliver')
@@ -167,11 +174,7 @@ def test_test_post_outbox_handles_create_note(
         }]
     )
     expected_activity = {
-        '@context': [
+        '@context': serializers.AP_CONTEXT,
-            'https://www.w3.org/ns/activitystreams',
-            'https://w3id.org/security/v1',
-            {}
-        ],
         'actor': test_actor.url,
         'id': 'https://{}/activities/note/{}/activity'.format(
             settings.FEDERATION_HOSTNAME, now.timestamp()
@@ -188,3 +191,276 @@ def test_test_post_outbox_handles_create_note(
         to=[actor.url],
         on_behalf_of=actors.SYSTEM_ACTORS['test'].get_actor_instance()
     )
+
+
+def test_getting_actor_instance_persists_in_db(db):
+    test = actors.SYSTEM_ACTORS['test'].get_actor_instance()
+    from_db = models.Actor.objects.get(url=test.url)
+
+    for f in test._meta.fields:
+        assert getattr(from_db, f.name) == getattr(test, f.name)
+
+
+@pytest.mark.parametrize('username,domain,expected', [
+    ('test', 'wrongdomain.com', False),
+    ('notsystem', '', False),
+    ('test', '', True),
+])
+def test_actor_is_system(
+        username, domain, expected, nodb_factories, settings):
+    if not domain:
+        domain = settings.FEDERATION_HOSTNAME
+
+    actor = nodb_factories['federation.Actor'](
+        preferred_username=username,
+        domain=domain,
+    )
+    assert actor.is_system is expected
+
+
+@pytest.mark.parametrize('username,domain,expected', [
+    ('test', 'wrongdomain.com', None),
+    ('notsystem', '', None),
+    ('test', '', actors.SYSTEM_ACTORS['test']),
+])
+def test_actor_is_system(
+        username, domain, expected, nodb_factories, settings):
+    if not domain:
+        domain = settings.FEDERATION_HOSTNAME
+    actor = nodb_factories['federation.Actor'](
+        preferred_username=username,
+        domain=domain,
+    )
+    assert actor.system_conf == expected
+
+
+@pytest.mark.parametrize('value', [False, True])
+def test_library_actor_manually_approves_based_on_setting(
+        value, settings):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = value
+    library_conf = actors.SYSTEM_ACTORS['library']
+    assert library_conf.manually_approves_followers is value
+
+
+def test_system_actor_handle(mocker, nodb_factories):
+    handler = mocker.patch(
+        'funkwhale_api.federation.actors.TestActor.handle_create')
+    actor = nodb_factories['federation.Actor']()
+    activity = nodb_factories['federation.Activity'](
+        type='Create', actor=actor.url)
+    serializer = serializers.ActivitySerializer(
+        data=activity
+    )
+    assert serializer.is_valid()
+    actors.SYSTEM_ACTORS['test'].handle(activity, actor)
+    handler.assert_called_once_with(activity, actor)
+
+
+def test_test_actor_handles_follow(
+        settings, mocker, factories):
+    deliver = mocker.patch(
+        'funkwhale_api.federation.activity.deliver')
+    actor = factories['federation.Actor']()
+    now = timezone.now()
+    mocker.patch('django.utils.timezone.now', return_value=now)
+    accept_follow = mocker.patch(
+        'funkwhale_api.federation.activity.accept_follow')
+    test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance()
+    data = {
+        'actor': actor.url,
+        'type': 'Follow',
+        'id': 'http://test.federation/user#follows/267',
+        'object': test_actor.url,
+    }
+    uid = uuid.uuid4()
+    mocker.patch('uuid.uuid4', return_value=uid)
+    expected_follow = {
+        '@context': serializers.AP_CONTEXT,
+        'actor': test_actor.url,
+        'id': test_actor.url + '#follows/{}'.format(uid),
+        'object': actor.url,
+        'type': 'Follow'
+    }
+
+    actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor)
+    accept_follow.assert_called_once_with(
+        test_actor, data, actor
+    )
+    expected_calls = [
+        mocker.call(
+            expected_follow,
+            to=[actor.url],
+            on_behalf_of=test_actor,
+        )
+    ]
+    deliver.assert_has_calls(expected_calls)
+
+
+def test_test_actor_handles_undo_follow(
+        settings, mocker, factories):
+    deliver = mocker.patch(
+        'funkwhale_api.federation.activity.deliver')
+    test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance()
+    follow = factories['federation.Follow'](target=test_actor)
+    reverse_follow = factories['federation.Follow'](
+        actor=test_actor, target=follow.actor)
+    follow_serializer = serializers.FollowSerializer(follow)
+    reverse_follow_serializer = serializers.FollowSerializer(
+        reverse_follow)
+    undo = {
+        '@context': serializers.AP_CONTEXT,
+        'type': 'Undo',
+        'id': follow_serializer.data['id'] + '/undo',
+        'actor': follow.actor.url,
+        'object': follow_serializer.data,
+    }
+    expected_undo = {
+        '@context': serializers.AP_CONTEXT,
+        'type': 'Undo',
+        'id': reverse_follow_serializer.data['id'] + '/undo',
+        'actor': reverse_follow.actor.url,
+        'object': reverse_follow_serializer.data,
+    }
+
+    actors.SYSTEM_ACTORS['test'].post_inbox(undo, actor=follow.actor)
+    deliver.assert_called_once_with(
+        expected_undo,
+        to=[follow.actor.url],
+        on_behalf_of=test_actor,)
+
+    assert models.Follow.objects.count() == 0
+
+
+def test_library_actor_handles_follow_manual_approval(
+        settings, mocker, factories):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True
+    actor = factories['federation.Actor']()
+    now = timezone.now()
+    mocker.patch('django.utils.timezone.now', return_value=now)
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    data = {
+        'actor': actor.url,
+        'type': 'Follow',
+        'id': 'http://test.federation/user#follows/267',
+        'object': library_actor.url,
+    }
+
+    library_actor.system_conf.post_inbox(data, actor=actor)
+    fr = library_actor.received_follow_requests.first()
+
+    assert library_actor.received_follow_requests.count() == 1
+    assert fr.target == library_actor
+    assert fr.actor == actor
+    assert fr.approved is None
+
+
+def test_library_actor_handles_follow_auto_approval(
+        settings, mocker, factories):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False
+    actor = factories['federation.Actor']()
+    accept_follow = mocker.patch(
+        'funkwhale_api.federation.activity.accept_follow')
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    data = {
+        'actor': actor.url,
+        'type': 'Follow',
+        'id': 'http://test.federation/user#follows/267',
+        'object': library_actor.url,
+    }
+    library_actor.system_conf.post_inbox(data, actor=actor)
+
+    assert library_actor.received_follow_requests.count() == 0
+    accept_follow.assert_called_once_with(
+        library_actor, data, actor
+    )
+
+
+def test_library_actor_handle_create_audio_no_library(mocker, factories):
+    # when we receive inbox create audio, we should not do anything
+    # if we don't have a configured library matching the sender
+    mocked_create = mocker.patch(
+        'funkwhale_api.federation.serializers.AudioSerializer.create'
+    )
+    actor = factories['federation.Actor']()
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    data = {
+        'actor': actor.url,
+        'type': 'Create',
+        'id': 'http://test.federation/audio/create',
+        'object': {
+            'id': 'https://batch.import',
+            'type': 'Collection',
+            'totalItems': 2,
+            'items': factories['federation.Audio'].create_batch(size=2)
+        },
+    }
+    library_actor.system_conf.post_inbox(data, actor=actor)
+
+    mocked_create.assert_not_called()
+    models.LibraryTrack.objects.count() == 0
+
+
+def test_library_actor_handle_create_audio_no_library_enabled(
+        mocker, factories):
+    # when we receive inbox create audio, we should not do anything
+    # if we don't have an enabled library
+    mocked_create = mocker.patch(
+        'funkwhale_api.federation.serializers.AudioSerializer.create'
+    )
+    disabled_library = factories['federation.Library'](
+        federation_enabled=False)
+    actor = disabled_library.actor
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    data = {
+        'actor': actor.url,
+        'type': 'Create',
+        'id': 'http://test.federation/audio/create',
+        'object': {
+            'id': 'https://batch.import',
+            'type': 'Collection',
+            'totalItems': 2,
+            'items': factories['federation.Audio'].create_batch(size=2)
+        },
+    }
+    library_actor.system_conf.post_inbox(data, actor=actor)
+
+    mocked_create.assert_not_called()
+    models.LibraryTrack.objects.count() == 0
+
+
+def test_library_actor_handle_create_audio(mocker, factories):
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    remote_library = factories['federation.Library'](
+        federation_enabled=True
+    )
+
+    data = {
+        'actor': remote_library.actor.url,
+        'type': 'Create',
+        'id': 'http://test.federation/audio/create',
+        'object': {
+            'id': 'https://batch.import',
+            'type': 'Collection',
+            'totalItems': 2,
+            'items': factories['federation.Audio'].create_batch(size=2)
+        },
+    }
+
+    library_actor.system_conf.post_inbox(data, actor=remote_library.actor)
+
+    lts = list(remote_library.tracks.order_by('id'))
+
+    assert len(lts) == 2
+
+    for i, a in enumerate(data['object']['items']):
+        lt = lts[i]
+        assert lt.pk is not None
+        assert lt.url == a['id']
+        assert lt.library == remote_library
+        assert lt.audio_url == a['url']['href']
+        assert lt.audio_mimetype == a['url']['mediaType']
+        assert lt.metadata == a['metadata']
+        assert lt.title == a['metadata']['recording']['title']
+        assert lt.artist_name == a['metadata']['artist']['name']
+        assert lt.album_title == a['metadata']['release']['title']
+        assert lt.published_date == arrow.get(a['published'])

api/tests/federation/test_authentication.py

@@ -3,7 +3,7 @@ from funkwhale_api.federation import keys
 from funkwhale_api.federation import signing
 
 
-def test_authenticate(nodb_factories, mocker, api_request):
+def test_authenticate(factories, mocker, api_request):
     private, public = keys.get_key_pair()
     actor_url = 'https://test.federation/actor'
     mocker.patch(
@@ -18,7 +18,7 @@ def test_authenticate(nodb_factories, mocker, api_request):
                 'id': actor_url + '#main-key',
             }
         })
-    signed_request = nodb_factories['federation.SignedRequest'](
+    signed_request = factories['federation.SignedRequest'](
         auth__key=private,
         auth__key_id=actor_url + '#main-key',
         auth__headers=[

api/tests/federation/test_commands.py

@@ -1,14 +0,0 @@
-from django.core.management import call_command
-
-
-def test_generate_instance_key_pair(preferences, mocker):
-    mocker.patch(
-        'funkwhale_api.federation.keys.get_key_pair',
-        return_value=(b'private', b'public'))
-    assert preferences['federation__public_key'] == ''
-    assert preferences['federation__private_key'] == ''
-
-    call_command('generate_keys', interactive=False)
-
-    assert preferences['federation__private_key'] == 'private'
-    assert preferences['federation__public_key'] == 'public'

api/tests/federation/test_models.py

@@ -0,0 +1,85 @@
+import pytest
+import uuid
+
+from django import db
+
+from funkwhale_api.federation import models
+from funkwhale_api.federation import serializers
+
+
+def test_cannot_duplicate_actor(factories):
+    actor = factories['federation.Actor']()
+
+    with pytest.raises(db.IntegrityError):
+        factories['federation.Actor'](
+            domain=actor.domain,
+            preferred_username=actor.preferred_username,
+        )
+
+
+def test_cannot_duplicate_follow(factories):
+    follow = factories['federation.Follow']()
+
+    with pytest.raises(db.IntegrityError):
+        factories['federation.Follow'](
+            target=follow.target,
+            actor=follow.actor,
+        )
+
+
+def test_follow_federation_url(factories):
+    follow = factories['federation.Follow'](local=True)
+    expected = '{}#follows/{}'.format(
+        follow.actor.url, follow.uuid)
+
+    assert follow.get_federation_url() == expected
+
+
+def test_follow_request_approve(mocker, factories):
+    uid = uuid.uuid4()
+    mocker.patch('uuid.uuid4', return_value=uid)
+    accept_follow = mocker.patch(
+        'funkwhale_api.federation.activity.accept_follow')
+    fr = factories['federation.FollowRequest'](target__local=True)
+    fr.approve()
+
+    follow = {
+        '@context': serializers.AP_CONTEXT,
+        'actor': fr.actor.url,
+        'id': fr.actor.url + '#follows/{}'.format(uid),
+        'object': fr.target.url,
+        'type': 'Follow'
+    }
+
+    assert fr.approved is True
+    assert list(fr.target.followers.all()) == [fr.actor]
+    accept_follow.assert_called_once_with(
+        fr.target, follow, fr.actor
+    )
+
+
+def test_follow_request_approve_non_local(mocker, factories):
+    uid = uuid.uuid4()
+    mocker.patch('uuid.uuid4', return_value=uid)
+    accept_follow = mocker.patch(
+        'funkwhale_api.federation.activity.accept_follow')
+    fr = factories['federation.FollowRequest']()
+    fr.approve()
+
+    assert fr.approved is True
+    assert list(fr.target.followers.all()) == [fr.actor]
+    accept_follow.assert_not_called()
+
+
+def test_follow_request_refused(mocker, factories):
+    fr = factories['federation.FollowRequest']()
+    fr.refuse()
+
+    assert fr.approved is False
+    assert fr.target.followers.count() == 0
+
+
+def test_library_model_unique_per_actor(factories):
+    library = factories['federation.Library']()
+    with pytest.raises(db.IntegrityError):
+        factories['federation.Library'](actor=library.actor)

api/tests/federation/test_permissions.py

@@ -0,0 +1,45 @@
+from rest_framework.views import APIView
+
+from funkwhale_api.federation import actors
+from funkwhale_api.federation import permissions
+
+
+def test_library_follower(
+        factories, api_request, anonymous_user, settings):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True
+    view = APIView.as_view()
+    permission = permissions.LibraryFollower()
+    request = api_request.get('/')
+    setattr(request, 'user', anonymous_user)
+    check = permission.has_permission(request, view)
+
+    assert check is False
+
+
+def test_library_follower_actor_non_follower(
+        factories, api_request, anonymous_user, settings):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True
+    actor = factories['federation.Actor']()
+    view = APIView.as_view()
+    permission = permissions.LibraryFollower()
+    request = api_request.get('/')
+    setattr(request, 'user', anonymous_user)
+    setattr(request, 'actor', actor)
+    check = permission.has_permission(request, view)
+
+    assert check is False
+
+
+def test_library_follower_actor_follower(
+        factories, api_request, anonymous_user, settings):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True
+    library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    follow = factories['federation.Follow'](target=library)
+    view = APIView.as_view()
+    permission = permissions.LibraryFollower()
+    request = api_request.get('/')
+    setattr(request, 'user', anonymous_user)
+    setattr(request, 'actor', follow.actor)
+    check = permission.has_permission(request, view)
+
+    assert check is True

api/tests/federation/test_serializers.py

@@ -1,31 +1,36 @@
-from django.urls import reverse
+import arrow
 
+from django.urls import reverse
+from django.core.paginator import Paginator
+
+from funkwhale_api.federation import actors
 from funkwhale_api.federation import keys
 from funkwhale_api.federation import models
 from funkwhale_api.federation import serializers
+from funkwhale_api.federation import utils
 
 
 def test_actor_serializer_from_ap(db):
     payload = {
-    	'id': 'https://test.federation/user',
+        'id': 'https://test.federation/user',
-    	'type': 'Person',
+        'type': 'Person',
-    	'following': 'https://test.federation/user/following',
+        'following': 'https://test.federation/user/following',
-    	'followers': 'https://test.federation/user/followers',
+        'followers': 'https://test.federation/user/followers',
-    	'inbox': 'https://test.federation/user/inbox',
+        'inbox': 'https://test.federation/user/inbox',
-    	'outbox': 'https://test.federation/user/outbox',
+        'outbox': 'https://test.federation/user/outbox',
-    	'preferredUsername': 'user',
+        'preferredUsername': 'user',
-    	'name': 'Real User',
+        'name': 'Real User',
-    	'summary': 'Hello world',
+        'summary': 'Hello world',
-    	'url': 'https://test.federation/@user',
+        'url': 'https://test.federation/@user',
-    	'manuallyApprovesFollowers': False,
+        'manuallyApprovesFollowers': False,
-    	'publicKey': {
+        'publicKey': {
-    		'id': 'https://test.federation/user#main-key',
+            'id': 'https://test.federation/user#main-key',
-    		'owner': 'https://test.federation/user',
+            'owner': 'https://test.federation/user',
-    		'publicKeyPem': 'yolo'
+            'publicKeyPem': 'yolo'
-    	},
+        },
-    	'endpoints': {
+        'endpoints': {
-    		'sharedInbox': 'https://test.federation/inbox'
+            'sharedInbox': 'https://test.federation/inbox'
-    	},
+        },
     }
 
     serializer = serializers.ActorSerializer(data=payload)
@@ -50,13 +55,13 @@ def test_actor_serializer_from_ap(db):
 
 def test_actor_serializer_only_mandatory_field_from_ap(db):
     payload = {
-    	'id': 'https://test.federation/user',
+        'id': 'https://test.federation/user',
-    	'type': 'Person',
+        'type': 'Person',
-    	'following': 'https://test.federation/user/following',
+        'following': 'https://test.federation/user/following',
-    	'followers': 'https://test.federation/user/followers',
+        'followers': 'https://test.federation/user/followers',
-    	'inbox': 'https://test.federation/user/inbox',
+        'inbox': 'https://test.federation/user/inbox',
-    	'outbox': 'https://test.federation/user/outbox',
+        'outbox': 'https://test.federation/user/outbox',
-    	'preferredUsername': 'user',
+        'preferredUsername': 'user',
     }
 
     serializer = serializers.ActorSerializer(data=payload)
@@ -82,24 +87,24 @@ def test_actor_serializer_to_ap():
             'https://w3id.org/security/v1',
             {},
         ],
-    	'id': 'https://test.federation/user',
+        'id': 'https://test.federation/user',
-    	'type': 'Person',
+        'type': 'Person',
-    	'following': 'https://test.federation/user/following',
+        'following': 'https://test.federation/user/following',
-    	'followers': 'https://test.federation/user/followers',
+        'followers': 'https://test.federation/user/followers',
-    	'inbox': 'https://test.federation/user/inbox',
+        'inbox': 'https://test.federation/user/inbox',
-    	'outbox': 'https://test.federation/user/outbox',
+        'outbox': 'https://test.federation/user/outbox',
-    	'preferredUsername': 'user',
+        'preferredUsername': 'user',
-    	'name': 'Real User',
+        'name': 'Real User',
-    	'summary': 'Hello world',
+        'summary': 'Hello world',
-    	'manuallyApprovesFollowers': False,
+        'manuallyApprovesFollowers': False,
-    	'publicKey': {
+        'publicKey': {
-    		'id': 'https://test.federation/user#main-key',
+            'id': 'https://test.federation/user#main-key',
-    		'owner': 'https://test.federation/user',
+            'owner': 'https://test.federation/user',
-    		'publicKeyPem': 'yolo'
+            'publicKeyPem': 'yolo'
-    	},
+        },
-    	'endpoints': {
+        'endpoints': {
-    		'sharedInbox': 'https://test.federation/inbox'
+            'sharedInbox': 'https://test.federation/inbox'
-    	},
+        },
     }
     ac = models.Actor(
         url=expected['id'],
@@ -144,3 +149,229 @@ def test_webfinger_serializer():
     serializer = serializers.ActorWebfingerSerializer(actor)
 
     assert serializer.data == expected
+
+
+def test_follow_serializer_to_ap(factories):
+    follow = factories['federation.Follow'](local=True)
+    serializer = serializers.FollowSerializer(follow)
+
+    expected = {
+        '@context': [
+            'https://www.w3.org/ns/activitystreams',
+            'https://w3id.org/security/v1',
+            {},
+        ],
+        'id': follow.get_federation_url(),
+        'type': 'Follow',
+        'actor': follow.actor.url,
+        'object': follow.target.url,
+    }
+
+    assert serializer.data == expected
+
+
+def test_paginated_collection_serializer(factories):
+    tfs = factories['music.TrackFile'].create_batch(size=5)
+    actor = factories['federation.Actor'](local=True)
+
+    conf = {
+        'id': 'https://test.federation/test',
+        'items': tfs,
+        'item_serializer': serializers.AudioSerializer,
+        'actor': actor,
+        'page_size': 2,
+    }
+    expected = {
+        '@context': [
+            'https://www.w3.org/ns/activitystreams',
+            'https://w3id.org/security/v1',
+            {},
+        ],
+        'type': 'Collection',
+        'id': conf['id'],
+        'actor': actor.url,
+        'totalItems': len(tfs),
+        'current': conf['id'] + '?page=1',
+        'last': conf['id'] + '?page=3',
+        'first': conf['id'] + '?page=1',
+    }
+
+    serializer = serializers.PaginatedCollectionSerializer(conf)
+
+    assert serializer.data == expected
+
+
+def test_collection_page_serializer(factories):
+    tfs = factories['music.TrackFile'].create_batch(size=5)
+    actor = factories['federation.Actor'](local=True)
+
+    conf = {
+        'id': 'https://test.federation/test',
+        'item_serializer': serializers.AudioSerializer,
+        'actor': actor,
+        'page': Paginator(tfs, 2).page(2),
+    }
+    expected = {
+        '@context': [
+            'https://www.w3.org/ns/activitystreams',
+            'https://w3id.org/security/v1',
+            {},
+        ],
+        'type': 'CollectionPage',
+        'id': conf['id'] + '?page=2',
+        'actor': actor.url,
+        'totalItems': len(tfs),
+        'partOf': conf['id'],
+        'prev': conf['id'] + '?page=1',
+        'next': conf['id'] + '?page=3',
+        'first': conf['id'] + '?page=1',
+        'last': conf['id'] + '?page=3',
+        'items': [
+            conf['item_serializer'](
+                i,
+                context={'actor': actor, 'include_ap_context': False}
+            ).data
+            for i in conf['page'].object_list
+        ]
+    }
+
+    serializer = serializers.CollectionPageSerializer(conf)
+
+    assert serializer.data == expected
+
+
+def test_activity_pub_audio_serializer_to_library_track(factories):
+    remote_library = factories['federation.Library']()
+    audio = factories['federation.Audio']()
+    serializer = serializers.AudioSerializer(
+        data=audio, context={'library': remote_library})
+
+    assert serializer.is_valid(raise_exception=True)
+
+    lt = serializer.save()
+
+    assert lt.pk is not None
+    assert lt.url == audio['id']
+    assert lt.library == remote_library
+    assert lt.audio_url == audio['url']['href']
+    assert lt.audio_mimetype == audio['url']['mediaType']
+    assert lt.metadata == audio['metadata']
+    assert lt.title == audio['metadata']['recording']['title']
+    assert lt.artist_name == audio['metadata']['artist']['name']
+    assert lt.album_title == audio['metadata']['release']['title']
+    assert lt.published_date == arrow.get(audio['published'])
+
+
+def test_activity_pub_audio_serializer_to_ap(factories):
+    tf = factories['music.TrackFile'](mimetype='audio/mp3')
+    library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    expected = {
+        '@context': serializers.AP_CONTEXT,
+        'type': 'Audio',
+        'id': tf.get_federation_url(),
+        'name': tf.track.full_name,
+        'published': tf.creation_date.isoformat(),
+        'updated': tf.modification_date.isoformat(),
+        'metadata': {
+            'artist': {
+                'musicbrainz_id': tf.track.artist.mbid,
+                'name': tf.track.artist.name,
+            },
+            'release': {
+                'musicbrainz_id': tf.track.album.mbid,
+                'title': tf.track.album.title,
+            },
+            'recording': {
+                'musicbrainz_id': tf.track.mbid,
+                'title': tf.track.title,
+            },
+        },
+        'url': {
+            'href': utils.full_url(tf.path),
+            'type': 'Link',
+            'mediaType': 'audio/mp3'
+        },
+        'attributedTo': [
+            library.url
+        ]
+    }
+
+    serializer = serializers.AudioSerializer(tf, context={'actor': library})
+
+    assert serializer.data == expected
+
+
+def test_activity_pub_audio_serializer_to_ap_no_mbid(factories):
+    tf = factories['music.TrackFile'](
+        mimetype='audio/mp3',
+        track__mbid=None,
+        track__album__mbid=None,
+        track__album__artist__mbid=None,
+    )
+    library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    expected = {
+        '@context': serializers.AP_CONTEXT,
+        'type': 'Audio',
+        'id': tf.get_federation_url(),
+        'name': tf.track.full_name,
+        'published': tf.creation_date.isoformat(),
+        'updated': tf.modification_date.isoformat(),
+        'metadata': {
+            'artist': {
+                'name': tf.track.artist.name,
+                'musicbrainz_id': None,
+            },
+            'release': {
+                'title': tf.track.album.title,
+                'musicbrainz_id': None,
+            },
+            'recording': {
+                'title': tf.track.title,
+                'musicbrainz_id': None,
+            },
+        },
+        'url': {
+            'href': utils.full_url(tf.path),
+            'type': 'Link',
+            'mediaType': 'audio/mp3'
+        },
+        'attributedTo': [
+            library.url
+        ]
+    }
+
+    serializer = serializers.AudioSerializer(tf, context={'actor': library})
+
+    assert serializer.data == expected
+
+
+def test_collection_serializer_to_ap(factories):
+    tf1 = factories['music.TrackFile'](mimetype='audio/mp3')
+    tf2 = factories['music.TrackFile'](mimetype='audio/ogg')
+    library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    expected = {
+        '@context': serializers.AP_CONTEXT,
+        'id': 'https://test.id',
+        'actor': library.url,
+        'totalItems': 2,
+        'type': 'Collection',
+        'items': [
+            serializers.AudioSerializer(
+                tf1, context={'actor': library, 'include_ap_context': False}
+            ).data,
+            serializers.AudioSerializer(
+                tf2, context={'actor': library, 'include_ap_context': False}
+            ).data,
+        ]
+    }
+
+    collection = {
+        'id': expected['id'],
+        'actor': library,
+        'items': [tf1, tf2],
+        'item_serializer': serializers.AudioSerializer
+    }
+    serializer = serializers.CollectionSerializer(
+        collection, context={'actor': library, 'id': 'https://test.id'})
+
+    assert serializer.data == expected

api/tests/federation/test_views.py

@@ -1,13 +1,14 @@
 from django.urls import reverse
+from django.core.paginator import Paginator
 
 import pytest
 
 from funkwhale_api.federation import actors
 from funkwhale_api.federation import serializers
+from funkwhale_api.federation import utils
 from funkwhale_api.federation import webfinger
 
 
-
 @pytest.mark.parametrize('system_actor', actors.SYSTEM_ACTORS.keys())
 def test_instance_actors(system_actor, db, settings, api_client):
     actor = actors.SYSTEM_ACTORS[system_actor].get_actor_instance()
@@ -17,6 +18,8 @@ def test_instance_actors(system_actor, db, settings, api_client):
     response = api_client.get(url)
     serializer = serializers.ActorSerializer(actor)
 
+    if system_actor == 'library':
+        response.data.pop('url')
     assert response.status_code == 200
     assert response.data == serializer.data
 
@@ -62,3 +65,89 @@ def test_wellknown_webfinger_system(
     assert response.status_code == 200
     assert response['Content-Type'] == 'application/jrd+json'
     assert response.data == serializer.data
+
+
+def test_audio_file_list_requires_authenticated_actor(
+        db, settings, api_client):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True
+    url = reverse('federation:music:files-list')
+    response = api_client.get(url)
+
+    assert response.status_code == 403
+
+
+def test_audio_file_list_actor_no_page(
+        db, settings, api_client, factories):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False
+    settings.FEDERATION_COLLECTION_PAGE_SIZE = 2
+    library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    tfs = factories['music.TrackFile'].create_batch(size=5)
+    conf = {
+        'id': utils.full_url(reverse('federation:music:files-list')),
+        'page_size': 2,
+        'items': list(reversed(tfs)),  # we order by -creation_date
+        'item_serializer': serializers.AudioSerializer,
+        'actor': library
+    }
+    expected = serializers.PaginatedCollectionSerializer(conf).data
+    url = reverse('federation:music:files-list')
+    response = api_client.get(url)
+
+    assert response.status_code == 200
+    assert response.data == expected
+
+
+def test_audio_file_list_actor_page(
+        db, settings, api_client, factories):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False
+    settings.FEDERATION_COLLECTION_PAGE_SIZE = 2
+    library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    tfs = factories['music.TrackFile'].create_batch(size=5)
+    conf = {
+        'id': utils.full_url(reverse('federation:music:files-list')),
+        'page': Paginator(list(reversed(tfs)), 2).page(2),
+        'item_serializer': serializers.AudioSerializer,
+        'actor': library
+    }
+    expected = serializers.CollectionPageSerializer(conf).data
+    url = reverse('federation:music:files-list')
+    response = api_client.get(url, data={'page': 2})
+
+    assert response.status_code == 200
+    assert response.data == expected
+
+
+def test_audio_file_list_actor_page_error(
+        db, settings, api_client, factories):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False
+    url = reverse('federation:music:files-list')
+    response = api_client.get(url, data={'page': 'nope'})
+
+    assert response.status_code == 400
+
+
+def test_audio_file_list_actor_page_error_too_far(
+        db, settings, api_client, factories):
+    settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False
+    url = reverse('federation:music:files-list')
+    response = api_client.get(url, data={'page': 5000})
+
+    assert response.status_code == 404
+
+
+def test_library_actor_includes_library_link(db, settings, api_client):
+    actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    url = reverse(
+        'federation:instance-actors-detail',
+        kwargs={'actor': 'library'})
+    response = api_client.get(url)
+    expected_links = [
+        {
+            'type': 'Link',
+            'name': 'library',
+            'mediaType': 'application/activity+json',
+            'href': utils.full_url(reverse('federation:music:files-list'))
+        }
+    ]
+    assert response.status_code == 200
+    assert response.data['url'] == expected_links

api/tests/music/test_import.py

@@ -1,7 +1,10 @@
 import json
+import pytest
 
 from django.urls import reverse
 
+from funkwhale_api.music import tasks
+
 
 def test_create_import_can_bind_to_request(
         artists, albums, mocker, factories, superuser_api_client):
@@ -33,3 +36,111 @@ def test_create_import_can_bind_to_request(
     batch = request.import_batches.latest('id')
 
     assert batch.import_request == request
+
+
+def test_import_job_from_federation_no_musicbrainz(factories):
+    lt = factories['federation.LibraryTrack'](
+        artist_name='Hello',
+        album_title='World',
+        title='Ping',
+    )
+    job = factories['music.ImportJob'](
+        federation=True,
+        library_track=lt,
+    )
+
+    tasks.import_job_run(import_job_id=job.pk)
+    job.refresh_from_db()
+
+    tf = job.track_file
+    assert tf.mimetype == lt.audio_mimetype
+    assert tf.library_track == job.library_track
+    assert tf.track.title == 'Ping'
+    assert tf.track.artist.name == 'Hello'
+    assert tf.track.album.title == 'World'
+
+
+def test_import_job_from_federation_musicbrainz_recording(factories, mocker):
+    t = factories['music.Track']()
+    track_from_api = mocker.patch(
+        'funkwhale_api.music.models.Track.get_or_create_from_api',
+        return_value=t)
+    lt = factories['federation.LibraryTrack'](
+        metadata__recording__musicbrainz=True,
+        artist_name='Hello',
+        album_title='World',
+    )
+    job = factories['music.ImportJob'](
+        federation=True,
+        library_track=lt,
+    )
+
+    tasks.import_job_run(import_job_id=job.pk)
+    job.refresh_from_db()
+
+    tf = job.track_file
+    assert tf.mimetype == lt.audio_mimetype
+    assert tf.library_track == job.library_track
+    assert tf.track == t
+    track_from_api.assert_called_once_with(
+        mbid=lt.metadata['recording']['musicbrainz_id'])
+
+
+def test_import_job_from_federation_musicbrainz_release(factories, mocker):
+    a = factories['music.Album']()
+    album_from_api = mocker.patch(
+        'funkwhale_api.music.models.Album.get_or_create_from_api',
+        return_value=a)
+    lt = factories['federation.LibraryTrack'](
+        metadata__release__musicbrainz=True,
+        artist_name='Hello',
+        title='Ping',
+    )
+    job = factories['music.ImportJob'](
+        federation=True,
+        library_track=lt,
+    )
+
+    tasks.import_job_run(import_job_id=job.pk)
+    job.refresh_from_db()
+
+    tf = job.track_file
+    assert tf.mimetype == lt.audio_mimetype
+    assert tf.library_track == job.library_track
+    assert tf.track.title == 'Ping'
+    assert tf.track.artist == a.artist
+    assert tf.track.album == a
+
+    album_from_api.assert_called_once_with(
+        mbid=lt.metadata['release']['musicbrainz_id'])
+
+
+def test_import_job_from_federation_musicbrainz_artist(factories, mocker):
+    a = factories['music.Artist']()
+    artist_from_api = mocker.patch(
+        'funkwhale_api.music.models.Artist.get_or_create_from_api',
+        return_value=a)
+    lt = factories['federation.LibraryTrack'](
+        metadata__artist__musicbrainz=True,
+        album_title='World',
+        title='Ping',
+    )
+    job = factories['music.ImportJob'](
+        federation=True,
+        library_track=lt,
+    )
+
+    tasks.import_job_run(import_job_id=job.pk)
+    job.refresh_from_db()
+
+    tf = job.track_file
+    assert tf.mimetype == lt.audio_mimetype
+    assert tf.library_track == job.library_track
+
+    assert tf.track.title == 'Ping'
+    assert tf.track.artist == a
+    assert tf.track.album.artist == a
+    assert tf.track.album.title == 'World'
+
+    artist_from_api.assert_called_once_with(
+        mbid=lt.metadata['artist']['musicbrainz_id'])

api/tests/music/test_permissions.py

@@ -0,0 +1,56 @@
+from rest_framework.views import APIView
+
+from funkwhale_api.federation import actors
+from funkwhale_api.music import permissions
+
+
+def test_list_permission_no_protect(anonymous_user, api_request, settings):
+    settings.PROTECT_AUDIO_FILES = False
+    view = APIView.as_view()
+    permission = permissions.Listen()
+    request = api_request.get('/')
+    assert permission.has_permission(request, view) is True
+
+
+def test_list_permission_protect_anonymous(
+        anonymous_user, api_request, settings):
+    settings.PROTECT_AUDIO_FILES = True
+    view = APIView.as_view()
+    permission = permissions.Listen()
+    request = api_request.get('/')
+    assert permission.has_permission(request, view) is False
+
+
+def test_list_permission_protect_authenticated(
+        factories, api_request, settings):
+    settings.PROTECT_AUDIO_FILES = True
+    user = factories['users.User']()
+    view = APIView.as_view()
+    permission = permissions.Listen()
+    request = api_request.get('/')
+    setattr(request, 'user', user)
+    assert permission.has_permission(request, view) is True
+
+
+def test_list_permission_protect_not_following_actor(
+        factories, api_request, settings):
+    settings.PROTECT_AUDIO_FILES = True
+    actor = factories['federation.Actor']()
+    view = APIView.as_view()
+    permission = permissions.Listen()
+    request = api_request.get('/')
+    setattr(request, 'actor', actor)
+    assert permission.has_permission(request, view) is False
+
+
+def test_list_permission_protect_following_actor(
+        factories, api_request, settings):
+    settings.PROTECT_AUDIO_FILES = True
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    follow = factories['federation.Follow'](target=library_actor)
+    view = APIView.as_view()
+    permission = permissions.Listen()
+    request = api_request.get('/')
+    setattr(request, 'actor', follow.actor)
+
+    assert permission.has_permission(request, view) is True

api/tests/music/test_views.py

@@ -1,6 +1,8 @@
+import io
 import pytest
 
 from funkwhale_api.music import views
+from funkwhale_api.federation import actors
 
 
 @pytest.mark.parametrize('param,expected', [
@@ -43,3 +45,41 @@ def test_album_view_filter_listenable(
     queryset = view.filter_queryset(view.get_queryset())
 
     assert list(queryset) == expected
+
+
+def test_can_serve_track_file_as_remote_library(
+        factories, authenticated_actor, settings, api_client):
+    settings.PROTECT_AUDIO_FILES = True
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    follow = factories['federation.Follow'](
+        actor=authenticated_actor, target=library_actor)
+
+    track_file = factories['music.TrackFile']()
+    response = api_client.get(track_file.path)
+
+    assert response.status_code == 200
+    assert response['X-Accel-Redirect'] == "{}{}".format(
+        settings.PROTECT_FILES_PATH,
+        track_file.audio_file.url)
+
+
+def test_can_serve_track_file_as_remote_library_deny_not_following(
+        factories, authenticated_actor, settings, api_client):
+    settings.PROTECT_AUDIO_FILES = True
+    track_file = factories['music.TrackFile']()
+    response = api_client.get(track_file.path)
+
+    assert response.status_code == 403
+
+
+def test_can_proxy_remote_track(
+        factories, settings, api_client, r_mock):
+    settings.PROTECT_AUDIO_FILES = False
+    track_file = factories['music.TrackFile'](federation=True)
+
+    r_mock.get(track_file.library_track.audio_url, body=io.StringIO('test'))
+    response = api_client.get(track_file.path)
+
+    assert response.status_code == 200
+    assert list(response.streaming_content) == [b't', b'e', b's', b't']
+    assert response['Content-Type'] == track_file.library_track.audio_mimetype

deploy/env.prod.sample

@@ -85,3 +85,12 @@ API_AUTHENTICATION_REQUIRED=True
 # This will help us detect and correct bugs
 RAVEN_ENABLED=false
 RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5
+
+# This settings enable/disable federation on the instance level
+FEDERATION_ENABLED=True
+# This setting decide wether music library is shared automatically
+# to followers or if it requires manual approval before.
+# FEDERATION_MUSIC_NEEDS_APPROVAL=False
+# means anyone can subscribe to your library and import your file,
+# use with caution.
+FEDERATION_MUSIC_NEEDS_APPROVAL=True