diff --git a/api/config/settings/common.py b/api/config/settings/common.py index fbe3b7045..e45f6c256 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -30,7 +30,12 @@ FUNKWHALE_HOSTNAME = urlsplit(FUNKWHALE_URL).netloc FEDERATION_ENABLED = env.bool('FEDERATION_ENABLED', default=True) FEDERATION_HOSTNAME = env('FEDERATION_HOSTNAME', default=FUNKWHALE_HOSTNAME) - +FEDERATION_COLLECTION_PAGE_SIZE = env.int( + 'FEDERATION_COLLECTION_PAGE_SIZE', default=50 +) +FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool( + 'FEDERATION_MUSIC_NEEDS_APPROVAL', default=True +) ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS') # APP CONFIGURATION diff --git a/api/funkwhale_api/common/utils.py b/api/funkwhale_api/common/utils.py index c9d450e6a..2d7641bf5 100644 --- a/api/funkwhale_api/common/utils.py +++ b/api/funkwhale_api/common/utils.py @@ -1,3 +1,4 @@ +from urllib.parse import urlencode, parse_qs, urlsplit, urlunsplit import os import shutil @@ -25,3 +26,20 @@ def on_commit(f, *args, **kwargs): return transaction.on_commit( lambda: f(*args, **kwargs) ) + + +def set_query_parameter(url, **kwargs): + """Given a URL, set or replace a query parameter and return the + modified URL. + + >>> set_query_parameter('http://example.com?foo=bar&biz=baz', 'foo', 'stuff') + 'http://example.com?foo=stuff&biz=baz' + """ + scheme, netloc, path, query_string, fragment = urlsplit(url) + query_params = parse_qs(query_string) + + for param_name, param_value in kwargs.items(): + query_params[param_name] = [param_value] + new_query_string = urlencode(query_params, doseq=True) + + return urlunsplit((scheme, netloc, path, new_query_string, fragment)) diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index 4eeb193b1..7502bd739 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -2,7 +2,9 @@ import logging import json import requests import requests_http_signature +import uuid +from . import models from . import signing logger = logging.getLogger(__name__) @@ -42,33 +44,26 @@ ACTIVITY_TYPES = [ OBJECT_TYPES = [ 'Article', 'Audio', + 'Collection', 'Document', 'Event', 'Image', 'Note', + 'OrderedCollection', 'Page', 'Place', 'Profile', 'Relationship', 'Tombstone', 'Video', -] +] + ACTIVITY_TYPES + def deliver(activity, on_behalf_of, to=[]): from . import actors logger.info('Preparing activity delivery to %s', to) - auth = requests_http_signature.HTTPSignatureAuth( - use_auth_header=False, - headers=[ - '(request-target)', - 'user-agent', - 'host', - 'date', - 'content-type',], - algorithm='rsa-sha256', - key=on_behalf_of.private_key.encode('utf-8'), - key_id=on_behalf_of.private_key_id, - ) + auth = signing.get_auth( + on_behalf_of.private_key, on_behalf_of.private_key_id) for url in to: recipient_actor = actors.get_actor(url) logger.debug('delivering to %s', recipient_actor.inbox_url) @@ -83,3 +78,68 @@ def deliver(activity, on_behalf_of, to=[]): ) response.raise_for_status() logger.debug('Remote answered with %s', response.status_code) + + +def get_follow(follow_id, follower, followed): + return { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {} + ], + 'actor': follower.url, + 'id': follower.url + '#follows/{}'.format(follow_id), + 'object': followed.url, + 'type': 'Follow' + } + + +def get_undo(id, actor, object): + return { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {} + ], + 'type': 'Undo', + 'id': id + '/undo', + 'actor': actor.url, + 'object': object, + } + + +def get_accept_follow(accept_id, accept_actor, follow, follow_actor): + return { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + "id": accept_actor.url + '#accepts/follows/{}'.format( + accept_id), + "type": "Accept", + "actor": accept_actor.url, + "object": { + "id": follow['id'], + "type": "Follow", + "actor": follow_actor.url, + "object": accept_actor.url + }, + } + + +def accept_follow(target, follow, actor): + accept_uuid = uuid.uuid4() + accept = get_accept_follow( + accept_id=accept_uuid, + accept_actor=target, + follow=follow, + follow_actor=actor) + deliver( + accept, + to=[actor.url], + on_behalf_of=target) + return models.Follow.objects.get_or_create( + actor=actor, + target=target, + ) diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 69033f5ca..54d78d9ff 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -1,8 +1,10 @@ import logging import requests +import uuid import xml from django.conf import settings +from django.db import transaction from django.urls import reverse from django.utils import timezone @@ -11,8 +13,10 @@ from rest_framework.exceptions import PermissionDenied from dynamic_preferences.registries import global_preferences_registry from . import activity +from . import keys from . import models from . import serializers +from . import signing from . import utils logger = logging.getLogger(__name__) @@ -47,31 +51,48 @@ def get_actor(actor_url): class SystemActor(object): additional_attributes = {} + manually_approves_followers = False + + def get_request_auth(self): + actor = self.get_actor_instance() + return signing.get_auth( + actor.private_key, actor.private_key_id) + + def serialize(self): + actor = self.get_actor_instance() + serializer = serializers.ActorSerializer(actor) + return serializer.data def get_actor_instance(self): - a = models.Actor( - **self.get_instance_argument( - self.id, - name=self.name, - summary=self.summary, - **self.additional_attributes - ) + try: + return models.Actor.objects.get(url=self.get_actor_url()) + except models.Actor.DoesNotExist: + pass + private, public = keys.get_key_pair() + args = self.get_instance_argument( + self.id, + name=self.name, + summary=self.summary, + **self.additional_attributes ) - a.pk = self.id - return a + args['private_key'] = private.decode('utf-8') + args['public_key'] = public.decode('utf-8') + return models.Actor.objects.create(**args) + + def get_actor_url(self): + return utils.full_url( + reverse( + 'federation:instance-actors-detail', + kwargs={'actor': self.id})) def get_instance_argument(self, id, name, summary, **kwargs): - preferences = global_preferences_registry.manager() p = { 'preferred_username': id, 'domain': settings.FEDERATION_HOSTNAME, 'type': 'Person', 'name': name.format(host=settings.FEDERATION_HOSTNAME), 'manually_approves_followers': True, - 'url': utils.full_url( - reverse( - 'federation:instance-actors-detail', - kwargs={'actor': id})), + 'url': self.get_actor_url(), 'shared_inbox_url': utils.full_url( reverse( 'federation:instance-actors-inbox', @@ -84,8 +105,6 @@ class SystemActor(object): reverse( 'federation:instance-actors-outbox', kwargs={'actor': id})), - 'public_key': preferences['federation__public_key'], - 'private_key': preferences['federation__private_key'], 'summary': summary.format(host=settings.FEDERATION_HOSTNAME) } p.update(kwargs) @@ -95,7 +114,7 @@ class SystemActor(object): raise NotImplementedError def post_inbox(self, data, actor=None): - raise NotImplementedError + return self.handle(data, actor=actor) def get_outbox(self, data, actor=None): raise NotImplementedError @@ -103,6 +122,62 @@ class SystemActor(object): def post_outbox(self, data, actor=None): raise NotImplementedError + def handle(self, data, actor=None): + """ + Main entrypoint for handling activities posted to the + actor's inbox + """ + logger.info('Received activity on %s inbox', self.id) + + if actor is None: + raise PermissionDenied('Actor not authenticated') + + serializer = serializers.ActivitySerializer( + data=data, context={'actor': actor}) + serializer.is_valid(raise_exception=True) + + ac = serializer.data + try: + handler = getattr( + self, 'handle_{}'.format(ac['type'].lower())) + except (KeyError, AttributeError): + logger.debug( + 'No handler for activity %s', ac['type']) + return + + return handler(data, actor) + + def handle_follow(self, ac, sender): + system_actor = self.get_actor_instance() + if self.manually_approves_followers: + fr, created = models.FollowRequest.objects.get_or_create( + actor=sender, + target=system_actor, + approved=None, + ) + return fr + + return activity.accept_follow( + system_actor, ac, sender + ) + + def handle_undo_follow(self, ac, sender): + actor = self.get_actor_instance() + models.Follow.objects.filter( + actor=sender, + target=actor, + ).delete() + + def handle_undo(self, ac, sender): + if ac['object']['type'] != 'Follow': + return + + if ac['object']['actor'] != sender.url: + # not the same actor, permission issue + return + + self.handle_undo_follow(ac, sender) + class LibraryActor(SystemActor): id = 'library' @@ -112,6 +187,62 @@ class LibraryActor(SystemActor): 'manually_approves_followers': True } + def serialize(self): + data = super().serialize() + urls = data.setdefault('url', []) + urls.append({ + 'type': 'Link', + 'mediaType': 'application/activity+json', + 'name': 'library', + 'href': utils.full_url(reverse('federation:music:files-list')) + }) + return data + + @property + def manually_approves_followers(self): + return settings.FEDERATION_MUSIC_NEEDS_APPROVAL + + @transaction.atomic + def handle_create(self, ac, sender): + try: + remote_library = models.Library.objects.get( + actor=sender, + federation_enabled=True, + ) + except models.Library.DoesNotExist: + logger.info( + 'Skipping import, we\'re not following %s', sender.url) + return + + if ac['object']['type'] != 'Collection': + return + + if ac['object']['totalItems'] <= 0: + return + + try: + items = ac['object']['items'] + except KeyError: + logger.warning('No items in collection!') + return + + item_serializers = [ + serializers.AudioSerializer( + data=i, context={'library': remote_library}) + for i in items + ] + + valid_serializers = [] + for s in item_serializers: + if s.is_valid(): + valid_serializers.append(s) + else: + logger.debug( + 'Skipping invalid item %s, %s', s.initial_data, s.errors) + + for s in valid_serializers: + s.save() + class TestActor(SystemActor): id = 'test' @@ -123,40 +254,24 @@ class TestActor(SystemActor): additional_attributes = { 'manually_approves_followers': False } + manually_approves_followers = False def get_outbox(self, data, actor=None): return { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {} - ], - "id": utils.full_url( + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + "id": utils.full_url( reverse( 'federation:instance-actors-outbox', kwargs={'actor': self.id})), - "type": "OrderedCollection", - "totalItems": 0, - "orderedItems": [] + "type": "OrderedCollection", + "totalItems": 0, + "orderedItems": [] } - def post_inbox(self, data, actor=None): - if actor is None: - raise PermissionDenied('Actor not authenticated') - - serializer = serializers.ActivitySerializer( - data=data, context={'actor': actor}) - serializer.is_valid(raise_exception=True) - - ac = serializer.validated_data - logger.info('Received activity on %s inbox', self.id) - if ac['type'] == 'Create' and ac['object']['type'] == 'Note': - # we received a toot \o/ - command = self.parse_command(ac['object']['content']) - logger.debug('Parsed command: %s', command) - if command == 'ping': - self.handle_ping(ac, actor) - def parse_command(self, message): """ Remove any links or fancy markup to extract /command from @@ -168,7 +283,16 @@ class TestActor(SystemActor): except IndexError: return - def handle_ping(self, ac, sender): + def handle_create(self, ac, sender): + if ac['object']['type'] != 'Note': + return + + # we received a toot \o/ + command = self.parse_command(ac['object']['content']) + logger.debug('Parsed command: %s', command) + if command != 'ping': + return + now = timezone.now() test_actor = self.get_actor_instance() reply_url = 'https://{}/activities/note/{}'.format( @@ -179,10 +303,10 @@ class TestActor(SystemActor): ) reply_activity = { "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {} - ], + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], 'type': 'Create', 'actor': test_actor.url, 'id': '{}/activity'.format(reply_url), @@ -214,6 +338,43 @@ class TestActor(SystemActor): to=[ac['actor']], on_behalf_of=test_actor) + def handle_follow(self, ac, sender): + super().handle_follow(ac, sender) + # also, we follow back + test_actor = self.get_actor_instance() + follow_uuid = uuid.uuid4() + follow = activity.get_follow( + follow_id=follow_uuid, + follower=test_actor, + followed=sender) + activity.deliver( + follow, + to=[ac['actor']], + on_behalf_of=test_actor) + + def handle_undo_follow(self, ac, sender): + super().handle_undo_follow(ac, sender) + actor = self.get_actor_instance() + # we also unfollow the sender, if possible + try: + follow = models.Follow.objects.get( + target=sender, + actor=actor, + ) + except models.Follow.DoesNotExist: + return + undo = activity.get_undo( + id=follow.get_federation_url(), + actor=actor, + object=serializers.FollowSerializer(follow).data, + ) + follow.delete() + activity.deliver( + undo, + to=[sender.url], + on_behalf_of=actor) + + SYSTEM_ACTORS = { 'library': LibraryActor(), 'test': TestActor(), diff --git a/api/funkwhale_api/federation/authentication.py b/api/funkwhale_api/federation/authentication.py index e199ef134..7f8ad6653 100644 --- a/api/funkwhale_api/federation/authentication.py +++ b/api/funkwhale_api/federation/authentication.py @@ -7,6 +7,7 @@ from rest_framework import exceptions from . import actors from . import keys +from . import models from . import serializers from . import signing from . import utils @@ -42,11 +43,16 @@ class SignatureAuthentication(authentication.BaseAuthentication): except cryptography.exceptions.InvalidSignature: raise exceptions.AuthenticationFailed('Invalid signature') - return serializer.build() + try: + return models.Actor.objects.get(url=actor_data['id']) + except models.Actor.DoesNotExist: + return serializer.save() def authenticate(self, request): setattr(request, 'actor', None) actor = self.authenticate_actor(request) + if not actor: + return user = AnonymousUser() setattr(request, 'actor', actor) return (user, None) diff --git a/api/funkwhale_api/federation/dynamic_preferences_registry.py b/api/funkwhale_api/federation/dynamic_preferences_registry.py index 83d0285be..c7cb015a8 100644 --- a/api/funkwhale_api/federation/dynamic_preferences_registry.py +++ b/api/funkwhale_api/federation/dynamic_preferences_registry.py @@ -4,31 +4,3 @@ from dynamic_preferences import types from dynamic_preferences.registries import global_preferences_registry federation = types.Section('federation') - - -@global_preferences_registry.register -class FederationPrivateKey(types.StringPreference): - show_in_api = False - section = federation - name = 'private_key' - default = '' - help_text = ( - 'Instance private key, used for signing federation HTTP requests' - ) - verbose_name = ( - 'Instance private key (keep it secret, do not change it)' - ) - - -@global_preferences_registry.register -class FederationPublicKey(types.StringPreference): - show_in_api = False - section = federation - name = 'public_key' - default = '' - help_text = ( - 'Instance public key, used for signing federation HTTP requests' - ) - verbose_name = ( - 'Instance public key (do not change it)' - ) diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index 88c86f791..b3ac72039 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -1,8 +1,10 @@ import factory import requests import requests_http_signature +import uuid from django.utils import timezone +from django.conf import settings from funkwhale_api.factories import registry @@ -51,9 +53,23 @@ class SignedRequestFactory(factory.Factory): self.headers.update(default_headers) +@registry.register(name='federation.Link') +class LinkFactory(factory.Factory): + type = 'Link' + href = factory.Faker('url') + mediaType = 'text/html' + + class Meta: + model = dict + + class Params: + audio = factory.Trait( + mediaType=factory.Iterator(['audio/mp3', 'audio/ogg']) + ) + + @registry.register class ActorFactory(factory.DjangoModelFactory): - public_key = None private_key = None preferred_username = factory.Faker('user_name') @@ -66,6 +82,12 @@ class ActorFactory(factory.DjangoModelFactory): class Meta: model = models.Actor + class Params: + local = factory.Trait( + domain=factory.LazyAttribute( + lambda o: settings.FEDERATION_HOSTNAME) + ) + @classmethod def _generate(cls, create, attrs): has_public = attrs.get('public_key') is not None @@ -77,6 +99,102 @@ class ActorFactory(factory.DjangoModelFactory): return super()._generate(create, attrs) +@registry.register +class FollowFactory(factory.DjangoModelFactory): + target = factory.SubFactory(ActorFactory) + actor = factory.SubFactory(ActorFactory) + + class Meta: + model = models.Follow + + class Params: + local = factory.Trait( + actor=factory.SubFactory(ActorFactory, local=True) + ) + + +@registry.register +class FollowRequestFactory(factory.DjangoModelFactory): + target = factory.SubFactory(ActorFactory) + actor = factory.SubFactory(ActorFactory) + + class Meta: + model = models.FollowRequest + + +@registry.register +class LibraryFactory(factory.DjangoModelFactory): + actor = factory.SubFactory(ActorFactory) + url = factory.Faker('url') + federation_enabled = True + download_files = False + autoimport = False + + class Meta: + model = models.Library + + +class ArtistMetadataFactory(factory.Factory): + name = factory.Faker('name') + + class Meta: + model = dict + + class Params: + musicbrainz = factory.Trait( + musicbrainz_id=factory.Faker('uuid4') + ) + + +class ReleaseMetadataFactory(factory.Factory): + title = factory.Faker('sentence') + + class Meta: + model = dict + + class Params: + musicbrainz = factory.Trait( + musicbrainz_id=factory.Faker('uuid4') + ) + + +class RecordingMetadataFactory(factory.Factory): + title = factory.Faker('sentence') + + class Meta: + model = dict + + class Params: + musicbrainz = factory.Trait( + musicbrainz_id=factory.Faker('uuid4') + ) + + +@registry.register(name='federation.LibraryTrackMetadata') +class LibraryTrackMetadataFactory(factory.Factory): + artist = factory.SubFactory(ArtistMetadataFactory) + recording = factory.SubFactory(RecordingMetadataFactory) + release = factory.SubFactory(ReleaseMetadataFactory) + + class Meta: + model = dict + + +@registry.register +class LibraryTrackFactory(factory.DjangoModelFactory): + library = factory.SubFactory(LibraryFactory) + url = factory.Faker('url') + title = factory.Faker('sentence') + artist_name = factory.Faker('sentence') + album_title = factory.Faker('sentence') + audio_url = factory.Faker('url') + audio_mimetype = 'audio/ogg' + metadata = factory.SubFactory(LibraryTrackMetadataFactory) + + class Meta: + model = models.LibraryTrack + + @registry.register(name='federation.Note') class NoteFactory(factory.Factory): type = 'Note' @@ -89,3 +207,51 @@ class NoteFactory(factory.Factory): class Meta: model = dict + + +@registry.register(name='federation.Activity') +class ActivityFactory(factory.Factory): + type = 'Create' + id = factory.Faker('url') + published = factory.LazyFunction( + lambda: timezone.now().isoformat() + ) + actor = factory.Faker('url') + object = factory.SubFactory( + NoteFactory, + actor=factory.SelfAttribute('..actor'), + published=factory.SelfAttribute('..published')) + + class Meta: + model = dict + + +@registry.register(name='federation.AudioMetadata') +class AudioMetadataFactory(factory.Factory): + recording = factory.LazyAttribute( + lambda o: 'https://musicbrainz.org/recording/{}'.format(uuid.uuid4()) + ) + artist = factory.LazyAttribute( + lambda o: 'https://musicbrainz.org/artist/{}'.format(uuid.uuid4()) + ) + release = factory.LazyAttribute( + lambda o: 'https://musicbrainz.org/release/{}'.format(uuid.uuid4()) + ) + + class Meta: + model = dict + + +@registry.register(name='federation.Audio') +class AudioFactory(factory.Factory): + type = 'Audio' + id = factory.Faker('url') + published = factory.LazyFunction( + lambda: timezone.now().isoformat() + ) + actor = factory.Faker('url') + url = factory.SubFactory(LinkFactory, audio=True) + metadata = factory.SubFactory(LibraryTrackMetadataFactory) + + class Meta: + model = dict diff --git a/api/funkwhale_api/federation/management/__init__.py b/api/funkwhale_api/federation/management/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/api/funkwhale_api/federation/management/commands/__init__.py b/api/funkwhale_api/federation/management/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/api/funkwhale_api/federation/management/commands/generate_keys.py b/api/funkwhale_api/federation/management/commands/generate_keys.py deleted file mode 100644 index eafe9aae3..000000000 --- a/api/funkwhale_api/federation/management/commands/generate_keys.py +++ /dev/null @@ -1,53 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction - -from dynamic_preferences.registries import global_preferences_registry - -from funkwhale_api.federation import keys - - -class Command(BaseCommand): - help = ( - 'Generate a public/private key pair for your instance,' - ' for federation purposes. If a key pair already exists, does nothing.' - ) - - def add_arguments(self, parser): - parser.add_argument( - '--replace', - action='store_true', - dest='replace', - default=False, - help='Replace existing key pair, if any', - ) - parser.add_argument( - '--noinput', '--no-input', action='store_false', dest='interactive', - help="Do NOT prompt the user for input of any kind.", - ) - - @transaction.atomic - def handle(self, *args, **options): - preferences = global_preferences_registry.manager() - existing_public = preferences['federation__public_key'] - existing_private = preferences['federation__public_key'] - - if existing_public or existing_private and not options['replace']: - raise CommandError( - 'Keys are already present! ' - 'Replace them with --replace if you know what you are doing.') - - if options['interactive']: - message = ( - 'Are you sure you want to do this?\n\n' - "Type 'yes' to continue, or 'no' to cancel: " - ) - if input(''.join(message)) != 'yes': - raise CommandError("Operation cancelled.") - private, public = keys.get_key_pair() - preferences['federation__public_key'] = public.decode('utf-8') - preferences['federation__private_key'] = private.decode('utf-8') - - self.stdout.write( - 'Your new key pair was generated.' - 'Your public key is now:\n\n{}'.format(public.decode('utf-8')) - ) diff --git a/api/funkwhale_api/federation/migrations/0002_auto_20180403_1620.py b/api/funkwhale_api/federation/migrations/0002_auto_20180403_1620.py new file mode 100644 index 000000000..2200424d8 --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0002_auto_20180403_1620.py @@ -0,0 +1,17 @@ +# Generated by Django 2.0.3 on 2018-04-03 16:20 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0001_initial'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='actor', + unique_together={('domain', 'preferred_username')}, + ), + ] diff --git a/api/funkwhale_api/federation/migrations/0003_auto_20180407_1010.py b/api/funkwhale_api/federation/migrations/0003_auto_20180407_1010.py new file mode 100644 index 000000000..12e3d73fe --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0003_auto_20180407_1010.py @@ -0,0 +1,94 @@ +# Generated by Django 2.0.3 on 2018-04-07 10:10 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import uuid + + +def delete_system_actors(apps, schema_editor): + """Revert site domain and name to default.""" + Actor = apps.get_model("federation", "Actor") + Actor.objects.filter(preferred_username__in=['test', 'library']).delete() + + +def backward(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0002_auto_20180403_1620'), + ] + + operations = [ + migrations.RunPython(delete_system_actors, backward), + migrations.CreateModel( + name='Follow', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('modification_date', models.DateTimeField(auto_now=True)), + ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='emitted_follows', to='federation.Actor')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='received_follows', to='federation.Actor')), + ], + ), + migrations.CreateModel( + name='FollowRequest', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('modification_date', models.DateTimeField(auto_now=True)), + ('approved', models.NullBooleanField(default=None)), + ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='emmited_follow_requests', to='federation.Actor')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='received_follow_requests', to='federation.Actor')), + ], + ), + migrations.CreateModel( + name='Library', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('modification_date', models.DateTimeField(auto_now=True)), + ('fetched_date', models.DateTimeField(blank=True, null=True)), + ('uuid', models.UUIDField(default=uuid.uuid4)), + ('url', models.URLField()), + ('federation_enabled', models.BooleanField()), + ('download_files', models.BooleanField()), + ('autoimport', models.BooleanField()), + ('tracks_count', models.PositiveIntegerField(blank=True, null=True)), + ('actor', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='library', to='federation.Actor')), + ], + ), + migrations.CreateModel( + name='LibraryTrack', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.URLField(unique=True)), + ('audio_url', models.URLField()), + ('audio_mimetype', models.CharField(max_length=200)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('modification_date', models.DateTimeField(auto_now=True)), + ('fetched_date', models.DateTimeField(blank=True, null=True)), + ('published_date', models.DateTimeField(blank=True, null=True)), + ('artist_name', models.CharField(max_length=500)), + ('album_title', models.CharField(max_length=500)), + ('title', models.CharField(max_length=500)), + ('metadata', django.contrib.postgres.fields.jsonb.JSONField(default={}, max_length=10000)), + ('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tracks', to='federation.Library')), + ], + ), + migrations.AddField( + model_name='actor', + name='followers', + field=models.ManyToManyField(related_name='following', through='federation.Follow', to='federation.Actor'), + ), + migrations.AlterUniqueTogether( + name='follow', + unique_together={('actor', 'target')}, + ), + ] diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index d76ad173b..bf1e5d830 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -1,4 +1,7 @@ +import uuid + from django.conf import settings +from django.contrib.postgres.fields import JSONField from django.db import models from django.utils import timezone @@ -12,6 +15,8 @@ TYPE_CHOICES = [ class Actor(models.Model): + ap_type = 'Actor' + url = models.URLField(unique=True, max_length=500, db_index=True) outbox_url = models.URLField(max_length=500) inbox_url = models.URLField(max_length=500) @@ -31,6 +36,16 @@ class Actor(models.Model): last_fetch_date = models.DateTimeField( default=timezone.now) manually_approves_followers = models.NullBooleanField(default=None) + followers = models.ManyToManyField( + to='self', + symmetrical=False, + through='Follow', + through_fields=('target', 'actor'), + related_name='following', + ) + + class Meta: + unique_together = ['domain', 'preferred_username'] @property def webfinger_subject(self): @@ -57,3 +72,127 @@ class Actor(models.Model): setattr(self, field, v.lower()) super().save(**kwargs) + + @property + def is_local(self): + return self.domain == settings.FEDERATION_HOSTNAME + + @property + def is_system(self): + from . import actors + return all([ + settings.FEDERATION_HOSTNAME == self.domain, + self.preferred_username in actors.SYSTEM_ACTORS + ]) + + @property + def system_conf(self): + from . import actors + if self.is_system: + return actors.SYSTEM_ACTORS[self.preferred_username] + + +class Follow(models.Model): + ap_type = 'Follow' + + uuid = models.UUIDField(default=uuid.uuid4, unique=True) + actor = models.ForeignKey( + Actor, + related_name='emitted_follows', + on_delete=models.CASCADE, + ) + target = models.ForeignKey( + Actor, + related_name='received_follows', + on_delete=models.CASCADE, + ) + creation_date = models.DateTimeField(default=timezone.now) + modification_date = models.DateTimeField( + auto_now=True) + + class Meta: + unique_together = ['actor', 'target'] + + def get_federation_url(self): + return '{}#follows/{}'.format(self.actor.url, self.uuid) + + +class FollowRequest(models.Model): + uuid = models.UUIDField(default=uuid.uuid4, unique=True) + actor = models.ForeignKey( + Actor, + related_name='emmited_follow_requests', + on_delete=models.CASCADE, + ) + target = models.ForeignKey( + Actor, + related_name='received_follow_requests', + on_delete=models.CASCADE, + ) + creation_date = models.DateTimeField(default=timezone.now) + modification_date = models.DateTimeField( + auto_now=True) + approved = models.NullBooleanField(default=None) + + def approve(self): + from . import activity + from . import serializers + self.approved = True + self.save(update_fields=['approved']) + Follow.objects.get_or_create( + target=self.target, + actor=self.actor + ) + if self.target.is_local: + follow = { + '@context': serializers.AP_CONTEXT, + 'actor': self.actor.url, + 'id': self.actor.url + '#follows/{}'.format(uuid.uuid4()), + 'object': self.target.url, + 'type': 'Follow' + } + activity.accept_follow( + self.target, follow, self.actor + ) + + def refuse(self): + self.approved = False + self.save(update_fields=['approved']) + + +class Library(models.Model): + creation_date = models.DateTimeField(default=timezone.now) + modification_date = models.DateTimeField( + auto_now=True) + fetched_date = models.DateTimeField(null=True, blank=True) + actor = models.OneToOneField( + Actor, + on_delete=models.CASCADE, + related_name='library') + uuid = models.UUIDField(default=uuid.uuid4) + url = models.URLField() + + # use this flag to disable federation with a library + federation_enabled = models.BooleanField() + # should we mirror files locally or hotlink them? + download_files = models.BooleanField() + # should we automatically import new files from this library? + autoimport = models.BooleanField() + tracks_count = models.PositiveIntegerField(null=True, blank=True) + + +class LibraryTrack(models.Model): + url = models.URLField(unique=True) + audio_url = models.URLField() + audio_mimetype = models.CharField(max_length=200) + creation_date = models.DateTimeField(default=timezone.now) + modification_date = models.DateTimeField( + auto_now=True) + fetched_date = models.DateTimeField(null=True, blank=True) + published_date = models.DateTimeField(null=True, blank=True) + library = models.ForeignKey( + Library, related_name='tracks', on_delete=models.CASCADE) + artist_name = models.CharField(max_length=500) + album_title = models.CharField(max_length=500) + title = models.CharField(max_length=500) + metadata = JSONField(default={}, max_length=10000) diff --git a/api/funkwhale_api/federation/permissions.py b/api/funkwhale_api/federation/permissions.py new file mode 100644 index 000000000..370328eaa --- /dev/null +++ b/api/funkwhale_api/federation/permissions.py @@ -0,0 +1,19 @@ +from django.conf import settings + +from rest_framework.permissions import BasePermission + +from . import actors + + +class LibraryFollower(BasePermission): + + def has_permission(self, request, view): + if not settings.FEDERATION_MUSIC_NEEDS_APPROVAL: + return True + + actor = getattr(request, 'actor', None) + if actor is None: + return False + + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + return library.followers.filter(url=actor.url).exists() diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 2137e8d91..17541c50f 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -2,15 +2,25 @@ import urllib.parse from django.urls import reverse from django.conf import settings +from django.core.paginator import Paginator +from django.db import transaction from rest_framework import serializers from dynamic_preferences.registries import global_preferences_registry +from funkwhale_api.common.utils import set_query_parameter + from . import activity from . import models from . import utils +AP_CONTEXT = [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, +] + class ActorSerializer(serializers.ModelSerializer): # left maps to activitypub fields, right to our internal models id = serializers.URLField(source='url') @@ -43,11 +53,7 @@ class ActorSerializer(serializers.ModelSerializer): def to_representation(self, instance): ret = super().to_representation(instance) - ret['@context'] = [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - {}, - ] + ret['@context'] = AP_CONTEXT if instance.public_key: ret['publicKey'] = { 'owner': instance.url, @@ -87,6 +93,28 @@ class ActorSerializer(serializers.ModelSerializer): return value[:500] +class FollowSerializer(serializers.ModelSerializer): + # left maps to activitypub fields, right to our internal models + id = serializers.URLField(source='get_federation_url') + object = serializers.URLField(source='target.url') + actor = serializers.URLField(source='actor.url') + type = serializers.CharField(source='ap_type') + + class Meta: + model = models.Actor + fields = [ + 'id', + 'object', + 'actor', + 'type' + ] + + def to_representation(self, instance): + ret = super().to_representation(instance) + ret['@context'] = AP_CONTEXT + return ret + + class ActorWebfingerSerializer(serializers.ModelSerializer): class Meta: model = models.Actor @@ -120,7 +148,9 @@ class ActivitySerializer(serializers.Serializer): type = value['type'] except KeyError: raise serializers.ValidationError('Missing object type') - + except TypeError: + # probably a URL + return value try: object_serializer = OBJECT_SERIALIZERS[type] except KeyError: @@ -173,3 +203,212 @@ OBJECT_SERIALIZERS = { t: ObjectSerializer for t in activity.OBJECT_TYPES } + + +class PaginatedCollectionSerializer(serializers.Serializer): + + def to_representation(self, conf): + paginator = Paginator( + conf['items'], + conf.get('page_size', 20) + ) + first = set_query_parameter(conf['id'], page=1) + current = first + last = set_query_parameter(conf['id'], page=paginator.num_pages) + d = { + 'id': conf['id'], + 'actor': conf['actor'].url, + 'totalItems': paginator.count, + 'type': 'Collection', + 'current': current, + 'first': first, + 'last': last, + } + if self.context.get('include_ap_context', True): + d['@context'] = AP_CONTEXT + return d + + +class CollectionPageSerializer(serializers.Serializer): + + def to_representation(self, conf): + page = conf['page'] + first = set_query_parameter(conf['id'], page=1) + last = set_query_parameter(conf['id'], page=page.paginator.num_pages) + id = set_query_parameter(conf['id'], page=page.number) + d = { + 'id': id, + 'partOf': conf['id'], + 'actor': conf['actor'].url, + 'totalItems': page.paginator.count, + 'type': 'CollectionPage', + 'first': first, + 'last': last, + 'items': [ + conf['item_serializer']( + i, + context={ + 'actor': conf['actor'], + 'include_ap_context': False} + ).data + for i in page.object_list + ] + } + + if page.has_previous(): + d['prev'] = set_query_parameter( + conf['id'], page=page.previous_page_number()) + + if page.has_previous(): + d['next'] = set_query_parameter( + conf['id'], page=page.next_page_number()) + + if self.context.get('include_ap_context', True): + d['@context'] = AP_CONTEXT + return d + + +class ArtistMetadataSerializer(serializers.Serializer): + musicbrainz_id = serializers.UUIDField(required=False) + name = serializers.CharField() + + +class ReleaseMetadataSerializer(serializers.Serializer): + musicbrainz_id = serializers.UUIDField(required=False) + title = serializers.CharField() + + +class RecordingMetadataSerializer(serializers.Serializer): + musicbrainz_id = serializers.UUIDField(required=False) + title = serializers.CharField() + + +class AudioMetadataSerializer(serializers.Serializer): + artist = ArtistMetadataSerializer() + release = ReleaseMetadataSerializer() + recording = RecordingMetadataSerializer() + + +class AudioSerializer(serializers.Serializer): + type = serializers.CharField() + id = serializers.URLField() + url = serializers.JSONField() + published = serializers.DateTimeField() + updated = serializers.DateTimeField(required=False) + metadata = AudioMetadataSerializer() + + def validate_type(self, v): + if v != 'Audio': + raise serializers.ValidationError('Invalid type for audio') + return v + + def validate_url(self, v): + try: + url = v['href'] + except (KeyError, TypeError): + raise serializers.ValidationError('Missing href') + + try: + media_type = v['mediaType'] + except (KeyError, TypeError): + raise serializers.ValidationError('Missing mediaType') + + if not media_type.startswith('audio/'): + raise serializers.ValidationError('Invalid mediaType') + + return url + + def validate_url(self, v): + try: + url = v['href'] + except (KeyError, TypeError): + raise serializers.ValidationError('Missing href') + + try: + media_type = v['mediaType'] + except (KeyError, TypeError): + raise serializers.ValidationError('Missing mediaType') + + if not media_type.startswith('audio/'): + raise serializers.ValidationError('Invalid mediaType') + + return v + + def create(self, validated_data): + defaults = { + 'audio_mimetype': validated_data['url']['mediaType'], + 'audio_url': validated_data['url']['href'], + 'metadata': validated_data['metadata'], + 'artist_name': validated_data['metadata']['artist']['name'], + 'album_title': validated_data['metadata']['release']['title'], + 'title': validated_data['metadata']['recording']['title'], + 'published_date': validated_data['published'], + 'modification_date': validated_data.get('updated'), + } + return models.LibraryTrack.objects.get_or_create( + library=self.context['library'], + url=validated_data['id'], + defaults=defaults + )[0] + + def to_representation(self, instance): + track = instance.track + album = instance.track.album + artist = instance.track.artist + + d = { + 'type': 'Audio', + 'id': instance.get_federation_url(), + 'name': instance.track.full_name, + 'published': instance.creation_date.isoformat(), + 'updated': instance.modification_date.isoformat(), + 'metadata': { + 'artist': { + 'musicbrainz_id': str(artist.mbid) if artist.mbid else None, + 'name': artist.name, + }, + 'release': { + 'musicbrainz_id': str(album.mbid) if album.mbid else None, + 'title': album.title, + }, + 'recording': { + 'musicbrainz_id': str(track.mbid) if track.mbid else None, + 'title': track.title, + }, + }, + 'url': { + 'href': utils.full_url(instance.path), + 'type': 'Link', + 'mediaType': instance.mimetype + }, + 'attributedTo': [ + self.context['actor'].url + ] + } + if self.context.get('include_ap_context', True): + d['@context'] = AP_CONTEXT + return d + + +class CollectionSerializer(serializers.Serializer): + + def to_representation(self, conf): + d = { + 'id': conf['id'], + 'actor': conf['actor'].url, + 'totalItems': len(conf['items']), + 'type': 'Collection', + 'items': [ + conf['item_serializer']( + i, + context={ + 'actor': conf['actor'], + 'include_ap_context': False} + ).data + for i in conf['items'] + ] + } + + if self.context.get('include_ap_context', True): + d['@context'] = AP_CONTEXT + return d diff --git a/api/funkwhale_api/federation/signing.py b/api/funkwhale_api/federation/signing.py index 7e4d2aa5a..8d984d3ff 100644 --- a/api/funkwhale_api/federation/signing.py +++ b/api/funkwhale_api/federation/signing.py @@ -53,3 +53,18 @@ def verify_django(django_request, public_key): request.headers[h] = str(v) prepared_request = request.prepare() return verify(request, public_key) + + +def get_auth(private_key, private_key_id): + return requests_http_signature.HTTPSignatureAuth( + use_auth_header=False, + headers=[ + '(request-target)', + 'user-agent', + 'host', + 'date', + 'content-type'], + algorithm='rsa-sha256', + key=private_key.encode('utf-8'), + key_id=private_key_id, + ) diff --git a/api/funkwhale_api/federation/urls.py b/api/funkwhale_api/federation/urls.py index f2c6f4c78..2c24b5257 100644 --- a/api/funkwhale_api/federation/urls.py +++ b/api/funkwhale_api/federation/urls.py @@ -1,8 +1,10 @@ -from rest_framework import routers +from django.conf.urls import include, url +from rest_framework import routers from . import views router = routers.SimpleRouter(trailing_slash=False) +music_router = routers.SimpleRouter(trailing_slash=False) router.register( r'federation/instance/actors', views.InstanceActorViewSet, @@ -12,4 +14,11 @@ router.register( views.WellKnownViewSet, 'well-known') -urlpatterns = router.urls +music_router.register( + r'files', + views.MusicFilesViewSet, + 'files', +) +urlpatterns = router.urls + [ + url('federation/music/', include((music_router.urls, 'music'), namespace='music')) +] diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 2e3feb8d0..35d8a75a5 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -1,16 +1,22 @@ from django import forms from django.conf import settings +from django.core import paginator from django.http import HttpResponse +from django.urls import reverse from rest_framework import viewsets from rest_framework import views from rest_framework import response from rest_framework.decorators import list_route, detail_route +from funkwhale_api.music.models import TrackFile + from . import actors from . import authentication +from . import permissions from . import renderers from . import serializers +from . import utils from . import webfinger @@ -38,8 +44,8 @@ class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet): def retrieve(self, request, *args, **kwargs): system_actor = self.get_object() actor = system_actor.get_actor_instance() - serializer = serializers.ActorSerializer(actor) - return response.Response(serializer.data, status=200) + data = actor.system_conf.serialize() + return response.Response(data, status=200) @detail_route(methods=['get', 'post']) def inbox(self, request, *args, **kwargs): @@ -101,3 +107,50 @@ class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet): username, hostname = clean_result actor = actors.SYSTEM_ACTORS[username].get_actor_instance() return serializers.ActorWebfingerSerializer(actor).data + + +class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet): + authentication_classes = [ + authentication.SignatureAuthentication] + permission_classes = [permissions.LibraryFollower] + renderer_classes = [renderers.ActivityPubRenderer] + + def list(self, request, *args, **kwargs): + page = request.GET.get('page') + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + qs = TrackFile.objects.order_by('-creation_date').select_related( + 'track__artist', + 'track__album__artist' + ) + if page is None: + conf = { + 'id': utils.full_url(reverse('federation:music:files-list')), + 'page_size': settings.FEDERATION_COLLECTION_PAGE_SIZE, + 'items': qs, + 'item_serializer': serializers.AudioSerializer, + 'actor': library, + } + serializer = serializers.PaginatedCollectionSerializer(conf) + data = serializer.data + else: + try: + page_number = int(page) + except: + return response.Response( + {'page': ['Invalid page number']}, status=400) + p = paginator.Paginator( + qs, settings.FEDERATION_COLLECTION_PAGE_SIZE) + try: + page = p.page(page_number) + conf = { + 'id': utils.full_url(reverse('federation:music:files-list')), + 'page': page, + 'item_serializer': serializers.AudioSerializer, + 'actor': library, + } + serializer = serializers.CollectionPageSerializer(conf) + data = serializer.data + except paginator.EmptyPage: + return response.Response(status=404) + + return response.Response(data) diff --git a/api/funkwhale_api/music/factories.py b/api/funkwhale_api/music/factories.py index 303e45228..2bf1960ca 100644 --- a/api/funkwhale_api/music/factories.py +++ b/api/funkwhale_api/music/factories.py @@ -2,6 +2,9 @@ import factory import os from funkwhale_api.factories import registry, ManyToManyFromList +from funkwhale_api.federation.factories import ( + LibraryTrackFactory, +) from funkwhale_api.users.factories import UserFactory SAMPLES_PATH = os.path.join( @@ -53,6 +56,18 @@ class TrackFileFactory(factory.django.DjangoModelFactory): class Meta: model = 'music.TrackFile' + class Params: + federation = factory.Trait( + audio_file=None, + library_track=factory.SubFactory(LibraryTrackFactory), + mimetype=factory.LazyAttribute( + lambda o: o.library_track.audio_mimetype + ), + source=factory.LazyAttribute( + lambda o: o.library_track.audio_url + ), + ) + @registry.register class ImportBatchFactory(factory.django.DjangoModelFactory): @@ -61,6 +76,12 @@ class ImportBatchFactory(factory.django.DjangoModelFactory): class Meta: model = 'music.ImportBatch' + class Params: + federation = factory.Trait( + submitted_by=None, + source='federation', + ) + @registry.register class ImportJobFactory(factory.django.DjangoModelFactory): @@ -71,6 +92,13 @@ class ImportJobFactory(factory.django.DjangoModelFactory): class Meta: model = 'music.ImportJob' + class Params: + federation = factory.Trait( + mbid=None, + library_track=factory.SubFactory(LibraryTrackFactory), + batch=factory.SubFactory(ImportBatchFactory, federation=True), + ) + @registry.register(name='music.FileImportJob') class FileImportJobFactory(ImportJobFactory): diff --git a/api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py b/api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py new file mode 100644 index 000000000..0539d90f6 --- /dev/null +++ b/api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py @@ -0,0 +1,88 @@ +# Generated by Django 2.0.3 on 2018-04-07 10:10 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0003_auto_20180407_1010'), + ('music', '0022_importbatch_import_request'), + ] + + operations = [ + migrations.AddField( + model_name='album', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AddField( + model_name='artist', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AddField( + model_name='importbatch', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AddField( + model_name='importjob', + name='library_track', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='import_jobs', to='federation.LibraryTrack'), + ), + migrations.AddField( + model_name='importjob', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AddField( + model_name='lyrics', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AddField( + model_name='track', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AddField( + model_name='trackfile', + name='creation_date', + field=models.DateTimeField(default=django.utils.timezone.now), + ), + migrations.AddField( + model_name='trackfile', + name='library_track', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='local_track_file', to='federation.LibraryTrack'), + ), + migrations.AddField( + model_name='trackfile', + name='modification_date', + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name='trackfile', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AddField( + model_name='work', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='importbatch', + name='source', + field=models.CharField(choices=[('api', 'api'), ('shell', 'shell'), ('federation', 'federation')], default='api', max_length=30), + ), + migrations.AlterField( + model_name='importbatch', + name='submitted_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index 7138dcdd6..beec551a5 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -5,6 +5,7 @@ import datetime import tempfile import shutil import markdown +import uuid from django.conf import settings from django.db import models @@ -20,12 +21,15 @@ from versatileimagefield.fields import VersatileImageField from funkwhale_api import downloader from funkwhale_api import musicbrainz +from funkwhale_api.federation import utils as federation_utils from . import importers from . import utils class APIModelMixin(models.Model): mbid = models.UUIDField(unique=True, db_index=True, null=True, blank=True) + uuid = models.UUIDField( + unique=True, db_index=True, default=uuid.uuid4) api_includes = [] creation_date = models.DateTimeField(default=timezone.now) import_hooks = [] @@ -65,6 +69,13 @@ class APIModelMixin(models.Model): pass return cleaned_data + @property + def musicbrainz_url(self): + if self.mbid: + return 'https://musicbrainz.org/{}/{}'.format( + self.musicbrainz_model, self.mbid) + + class Artist(APIModelMixin): name = models.CharField(max_length=255) @@ -90,10 +101,19 @@ class Artist(APIModelMixin): t.append(tag) return set(t) + @classmethod + def get_or_create_from_name(cls, name, **kwargs): + kwargs.update({'name': name}) + return cls.objects.get_or_create( + name__iexact=name, + defaults=kwargs)[0] + + def import_artist(v): a = Artist.get_or_create_from_api(mbid=v[0]['artist']['id'])[0] return a + def parse_date(v): if len(v) == 4: return datetime.date(int(v), 1, 1) @@ -108,6 +128,7 @@ def import_tracks(instance, cleaned_data, raw_data): track_cleaned_data['position'] = int(track_data['position']) track = importers.load(Track, track_cleaned_data, track_data, Track.import_hooks) + class Album(APIModelMixin): title = models.CharField(max_length=255) artist = models.ForeignKey( @@ -170,6 +191,14 @@ class Album(APIModelMixin): t.append(tag) return set(t) + @classmethod + def get_or_create_from_title(cls, title, **kwargs): + kwargs.update({'title': title}) + return cls.objects.get_or_create( + title__iexact=title, + defaults=kwargs)[0] + + def import_tags(instance, cleaned_data, raw_data): MINIMUM_COUNT = 2 tags_to_add = [] @@ -182,6 +211,7 @@ def import_tags(instance, cleaned_data, raw_data): tags_to_add.append(tag_data['name']) instance.tags.add(*tags_to_add) + def import_album(v): a = Album.get_or_create_from_api(mbid=v[0]['id'])[0] return a @@ -248,6 +278,8 @@ class Work(APIModelMixin): class Lyrics(models.Model): + uuid = models.UUIDField( + unique=True, db_index=True, default=uuid.uuid4) work = models.ForeignKey( Work, related_name='lyrics', @@ -328,7 +360,7 @@ class Track(APIModelMixin): def save(self, **kwargs): try: self.artist - except Artist.DoesNotExist: + except Artist.DoesNotExist: self.artist = self.album.artist super().save(**kwargs) @@ -366,16 +398,35 @@ class Track(APIModelMixin): self.mbid) return settings.FUNKWHALE_URL + '/tracks/{}'.format(self.pk) + @classmethod + def get_or_create_from_title(cls, title, **kwargs): + kwargs.update({'title': title}) + return cls.objects.get_or_create( + title__iexact=title, + defaults=kwargs)[0] + class TrackFile(models.Model): + uuid = models.UUIDField( + unique=True, db_index=True, default=uuid.uuid4) track = models.ForeignKey( Track, related_name='files', on_delete=models.CASCADE) audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255) source = models.URLField(null=True, blank=True) + creation_date = models.DateTimeField(default=timezone.now) + modification_date = models.DateTimeField(auto_now=True) duration = models.IntegerField(null=True, blank=True) acoustid_track_id = models.UUIDField(null=True, blank=True) mimetype = models.CharField(null=True, blank=True, max_length=200) + library_track = models.OneToOneField( + 'federation.LibraryTrack', + related_name='local_track_file', + on_delete=models.CASCADE, + null=True, + blank=True, + ) + def download_file(self): # import the track file, since there is not any # we create a tmp dir for the download @@ -391,12 +442,15 @@ class TrackFile(models.Model): shutil.rmtree(tmp_dir) return self.audio_file + def get_federation_url(self): + return federation_utils.full_url( + '/federation/music/file/{}'.format(self.uuid) + ) + @property def path(self): - if settings.PROTECT_AUDIO_FILES: - return reverse( - 'api:v1:trackfiles-serve', kwargs={'pk': self.pk}) - return self.audio_file.url + return reverse( + 'api:v1:trackfiles-serve', kwargs={'pk': self.pk}) @property def filename(self): @@ -417,10 +471,14 @@ IMPORT_STATUS_CHOICES = ( ('skipped', 'Skipped'), ) + class ImportBatch(models.Model): + uuid = models.UUIDField( + unique=True, db_index=True, default=uuid.uuid4) IMPORT_BATCH_SOURCES = [ ('api', 'api'), - ('shell', 'shell') + ('shell', 'shell'), + ('federation', 'federation'), ] source = models.CharField( max_length=30, default='api', choices=IMPORT_BATCH_SOURCES) @@ -428,6 +486,8 @@ class ImportBatch(models.Model): submitted_by = models.ForeignKey( 'users.User', related_name='imports', + null=True, + blank=True, on_delete=models.CASCADE) status = models.CharField( choices=IMPORT_STATUS_CHOICES, default='pending', max_length=30) @@ -437,6 +497,7 @@ class ImportBatch(models.Model): null=True, blank=True, on_delete=models.CASCADE) + class Meta: ordering = ['-creation_date'] @@ -449,6 +510,8 @@ class ImportBatch(models.Model): class ImportJob(models.Model): + uuid = models.UUIDField( + unique=True, db_index=True, default=uuid.uuid4) batch = models.ForeignKey( ImportBatch, related_name='jobs', on_delete=models.CASCADE) track_file = models.ForeignKey( @@ -465,6 +528,14 @@ class ImportJob(models.Model): audio_file = models.FileField( upload_to='imports/%Y/%m/%d', max_length=255, null=True, blank=True) + library_track = models.ForeignKey( + 'federation.LibraryTrack', + related_name='import_jobs', + on_delete=models.SET_NULL, + null=True, + blank=True + ) + class Meta: ordering = ('id', ) diff --git a/api/funkwhale_api/music/permissions.py b/api/funkwhale_api/music/permissions.py new file mode 100644 index 000000000..a8e62f1e7 --- /dev/null +++ b/api/funkwhale_api/music/permissions.py @@ -0,0 +1,23 @@ +from django.conf import settings + +from rest_framework.permissions import BasePermission + +from funkwhale_api.federation import actors + + +class Listen(BasePermission): + + def has_permission(self, request, view): + if not settings.PROTECT_AUDIO_FILES: + return True + + user = getattr(request, 'user', None) + if user and user.is_authenticated: + return True + + actor = getattr(request, 'actor', None) + if actor is None: + return False + + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + return library.followers.filter(url=actor.url).exists() diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index 48419bbe4..42795dbea 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -1,7 +1,10 @@ +from django.db import transaction from rest_framework import serializers from taggit.models import Tag from funkwhale_api.activity import serializers as activity_serializers +from funkwhale_api.federation.serializers import AP_CONTEXT +from funkwhale_api.federation import utils as federation_utils from . import models diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py index bf7a847d0..012b72cd2 100644 --- a/api/funkwhale_api/music/tasks.py +++ b/api/funkwhale_api/music/tasks.py @@ -25,6 +25,48 @@ def set_acoustid_on_track_file(track_file): return update(result['id']) +def import_track_from_remote(library_track): + metadata = library_track.metadata + try: + track_mbid = metadata['recording']['musicbrainz_id'] + assert track_mbid # for null/empty values + except (KeyError, AssertionError): + pass + else: + return models.Track.get_or_create_from_api(mbid=track_mbid) + + try: + album_mbid = metadata['release']['musicbrainz_id'] + assert album_mbid # for null/empty values + except (KeyError, AssertionError): + pass + else: + album = models.Album.get_or_create_from_api(mbid=album_mbid) + return models.Track.get_or_create_from_title( + library_track.title, artist=album.artist, album=album) + + try: + artist_mbid = metadata['artist']['musicbrainz_id'] + assert artist_mbid # for null/empty values + except (KeyError, AssertionError): + pass + else: + artist = models.Artist.get_or_create_from_api(mbid=artist_mbid) + album = models.Album.get_or_create_from_title( + library_track.album_title, artist=artist) + return models.Track.get_or_create_from_title( + library_track.title, artist=artist, album=album) + + # worst case scenario, we have absolutely no way to link to a + # musicbrainz resource, we rely on the name/titles + artist = models.Artist.get_or_create_from_name( + library_track.artist_name) + album = models.Album.get_or_create_from_title( + library_track.album_title, artist=artist) + return models.Track.get_or_create_from_title( + library_track.title, artist=artist, album=album) + + def _do_import(import_job, replace, use_acoustid=True): from_file = bool(import_job.audio_file) mbid = import_job.mbid @@ -43,8 +85,14 @@ def _do_import(import_job, replace, use_acoustid=True): acoustid_track_id = match['id'] if mbid: track, _ = models.Track.get_or_create_from_api(mbid=mbid) - else: + elif import_job.audio_file: track = import_track_data_from_path(import_job.audio_file.path) + elif import_job.library_track: + track = import_track_from_remote(import_job.library_track) + else: + raise ValueError( + 'Not enough data to process import, ' + 'add a mbid, an audio file or a library track') track_file = None if replace: @@ -63,6 +111,14 @@ def _do_import(import_job, replace, use_acoustid=True): track_file.audio_file = ContentFile(import_job.audio_file.read()) track_file.audio_file.name = import_job.audio_file.name track_file.duration = duration + elif import_job.library_track: + track_file.library_track = import_job.library_track + track_file.mimetype = import_job.library_track.audio_mimetype + if import_job.library_track.library.download_files: + raise NotImplementedError() + else: + # no downloading, we hotlink + pass else: track_file.download_file() track_file.save() diff --git a/api/funkwhale_api/music/utils.py b/api/funkwhale_api/music/utils.py index df659cb80..af0e59ab4 100644 --- a/api/funkwhale_api/music/utils.py +++ b/api/funkwhale_api/music/utils.py @@ -60,3 +60,10 @@ def compute_status(jobs): if pending: return 'pending' return 'finished' + + +def get_ext_from_type(mimetype): + mapping = { + 'audio/ogg': 'ogg', + 'audio/mpeg': 'mp3', + } diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index 5ac3143f9..5f8fc1736 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -1,36 +1,42 @@ import ffmpeg import os import json +import requests import subprocess import unicodedata import urllib -from django.urls import reverse +from django.contrib.auth.decorators import login_required +from django.core.exceptions import ObjectDoesNotExist +from django.conf import settings from django.db import models, transaction from django.db.models.functions import Length -from django.conf import settings from django.http import StreamingHttpResponse +from django.urls import reverse +from django.utils.decorators import method_decorator from rest_framework import viewsets, views, mixins from rest_framework.decorators import detail_route, list_route from rest_framework.response import Response +from rest_framework import settings as rest_settings from rest_framework import permissions from musicbrainzngs import ResponseError -from django.contrib.auth.decorators import login_required -from django.utils.decorators import method_decorator from funkwhale_api.common import utils as funkwhale_utils +from funkwhale_api.federation import actors from funkwhale_api.requests.models import ImportRequest from funkwhale_api.musicbrainz import api from funkwhale_api.common.permissions import ( ConditionalAuthentication, HasModelPermission) from taggit.models import Tag +from funkwhale_api.federation.authentication import SignatureAuthentication -from . import forms -from . import models -from . import serializers -from . import importers from . import filters +from . import forms +from . import importers +from . import models +from . import permissions as music_permissions +from . import serializers from . import tasks from . import utils @@ -45,6 +51,7 @@ class SearchMixin(object): serializer = self.serializer_class(queryset, many=True) return Response(serializer.data) + class TagViewSetMixin(object): def get_queryset(self): @@ -179,22 +186,54 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet): class TrackFileViewSet(viewsets.ReadOnlyModelViewSet): queryset = (models.TrackFile.objects.all().order_by('-id')) serializer_class = serializers.TrackFileSerializer - permission_classes = [ConditionalAuthentication] + authentication_classes = rest_settings.api_settings.DEFAULT_AUTHENTICATION_CLASSES + [ + SignatureAuthentication + ] + permission_classes = [music_permissions.Listen] @detail_route(methods=['get']) def serve(self, request, *args, **kwargs): try: - f = models.TrackFile.objects.get(pk=kwargs['pk']) + f = models.TrackFile.objects.select_related( + 'library_track', + 'track__album__artist', + 'track__artist', + ).get(pk=kwargs['pk']) except models.TrackFile.DoesNotExist: return Response(status=404) - response = Response() + mt = f.mimetype + try: + library_track = f.library_track + except ObjectDoesNotExist: + library_track = None + if library_track and not f.audio_file: + # we proxy the response to the remote library + # since we did not mirror the file locally + mt = library_track.audio_mimetype + file_extension = utils.get_ext_from_type(mt) + filename = '{}.{}'.format(f.track.full_name, file_extension) + auth = actors.SYSTEM_ACTORS['library'].get_request_auth() + remote_response = requests.get( + library_track.audio_url, + auth=auth, + stream=True, + headers={ + 'Content-Type': 'application/activity+json' + }) + response = StreamingHttpResponse(remote_response.iter_content()) + else: + response = Response() + filename = f.filename + response['X-Accel-Redirect'] = "{}{}".format( + settings.PROTECT_FILES_PATH, + f.audio_file.url) filename = "filename*=UTF-8''{}".format( - urllib.parse.quote(f.filename)) + urllib.parse.quote(filename)) response["Content-Disposition"] = "attachment; {}".format(filename) - response['X-Accel-Redirect'] = "{}{}".format( - settings.PROTECT_FILES_PATH, - f.audio_file.url) + if mt: + response["Content-Type"] = mt + return response @list_route(methods=['get']) diff --git a/api/tests/conftest.py b/api/tests/conftest.py index d5bb56565..4f1ee8962 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -162,3 +162,12 @@ def media_root(settings): def r_mock(): with requests_mock.mock() as m: yield m + + +@pytest.fixture +def authenticated_actor(factories, mocker): + actor = factories['federation.Actor']() + mocker.patch( + 'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor', + return_value=actor) + yield actor diff --git a/api/tests/federation/conftest.py b/api/tests/federation/conftest.py deleted file mode 100644 index c5831914b..000000000 --- a/api/tests/federation/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -import pytest - - -@pytest.fixture -def authenticated_actor(nodb_factories, mocker): - actor = nodb_factories['federation.Actor']() - mocker.patch( - 'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor', - return_value=actor) - yield actor diff --git a/api/tests/federation/test_activity.py b/api/tests/federation/test_activity.py index a6e1d28aa..09c5e3bf7 100644 --- a/api/tests/federation/test_activity.py +++ b/api/tests/federation/test_activity.py @@ -1,5 +1,8 @@ +import uuid + from funkwhale_api.federation import activity + def test_deliver(nodb_factories, r_mock, mocker): to = nodb_factories['federation.Actor']() mocker.patch( @@ -30,3 +33,42 @@ def test_deliver(nodb_factories, r_mock, mocker): assert r_mock.call_count == 1 assert request.url == to.inbox_url assert request.headers['content-type'] == 'application/activity+json' + + +def test_accept_follow(mocker, factories): + deliver = mocker.patch( + 'funkwhale_api.federation.activity.deliver') + actor = factories['federation.Actor']() + target = factories['federation.Actor'](local=True) + follow = { + 'actor': actor.url, + 'type': 'Follow', + 'id': 'http://test.federation/user#follows/267', + 'object': target.url, + } + uid = uuid.uuid4() + mocker.patch('uuid.uuid4', return_value=uid) + expected_accept = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + "id": target.url + '#accepts/follows/{}'.format(uid), + "type": "Accept", + "actor": target.url, + "object": { + "id": follow['id'], + "type": "Follow", + "actor": actor.url, + "object": target.url + }, + } + activity.accept_follow( + target, follow, actor + ) + deliver.assert_called_once_with( + expected_accept, to=[actor.url], on_behalf_of=target + ) + follow_instance = actor.emitted_follows.first() + assert follow_instance.target == target diff --git a/api/tests/federation/test_actors.py b/api/tests/federation/test_actors.py index b3b0f8df0..090d9b03f 100644 --- a/api/tests/federation/test_actors.py +++ b/api/tests/federation/test_actors.py @@ -1,4 +1,6 @@ +import arrow import pytest +import uuid from django.urls import reverse from django.utils import timezone @@ -6,6 +8,7 @@ from django.utils import timezone from rest_framework import exceptions from funkwhale_api.federation import actors +from funkwhale_api.federation import models from funkwhale_api.federation import serializers from funkwhale_api.federation import utils @@ -23,14 +26,17 @@ def test_actor_fetching(r_mock): assert r == payload -def test_get_library(settings, preferences): - preferences['federation__public_key'] = 'public_key' +def test_get_library(db, settings, mocker): + get_key_pair = mocker.patch( + 'funkwhale_api.federation.keys.get_key_pair', + return_value=(b'private', b'public')) expected = { 'preferred_username': 'library', 'domain': settings.FEDERATION_HOSTNAME, 'type': 'Person', 'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME), 'manually_approves_followers': True, + 'public_key': 'public', 'url': utils.full_url( reverse( 'federation:instance-actors-detail', @@ -47,7 +53,6 @@ def test_get_library(settings, preferences): reverse( 'federation:instance-actors-outbox', kwargs={'actor': 'library'})), - 'public_key': 'public_key', 'summary': 'Bot account to federate with {}\'s library'.format( settings.FEDERATION_HOSTNAME), } @@ -56,14 +61,17 @@ def test_get_library(settings, preferences): assert getattr(actor, key) == value -def test_get_test(settings, preferences): - preferences['federation__public_key'] = 'public_key' +def test_get_test(db, mocker, settings): + get_key_pair = mocker.patch( + 'funkwhale_api.federation.keys.get_key_pair', + return_value=(b'private', b'public')) expected = { 'preferred_username': 'test', 'domain': settings.FEDERATION_HOSTNAME, 'type': 'Person', 'name': '{}\'s test account'.format(settings.FEDERATION_HOSTNAME), 'manually_approves_followers': False, + 'public_key': 'public', 'url': utils.full_url( reverse( 'federation:instance-actors-detail', @@ -80,7 +88,6 @@ def test_get_test(settings, preferences): reverse( 'federation:instance-actors-outbox', kwargs={'actor': 'test'})), - 'public_key': 'public_key', 'summary': 'Bot account to test federation with {}. Send me /ping and I\'ll answer you.'.format( settings.FEDERATION_HOSTNAME), } @@ -91,18 +98,18 @@ def test_get_test(settings, preferences): def test_test_get_outbox(): expected = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {} - ], - "id": utils.full_url( + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + "id": utils.full_url( reverse( 'federation:instance-actors-outbox', kwargs={'actor': 'test'})), - "type": "OrderedCollection", - "totalItems": 0, - "orderedItems": [] + "type": "OrderedCollection", + "totalItems": 0, + "orderedItems": [] } data = actors.SYSTEM_ACTORS['test'].get_outbox({}, actor=None) @@ -126,7 +133,7 @@ def test_test_post_outbox_validates_actor(nodb_factories): assert msg in exc_info.value -def test_test_post_outbox_handles_create_note( +def test_test_post_inbox_handles_create_note( settings, mocker, factories): deliver = mocker.patch( 'funkwhale_api.federation.activity.deliver') @@ -167,11 +174,7 @@ def test_test_post_outbox_handles_create_note( }] ) expected_activity = { - '@context': [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - {} - ], + '@context': serializers.AP_CONTEXT, 'actor': test_actor.url, 'id': 'https://{}/activities/note/{}/activity'.format( settings.FEDERATION_HOSTNAME, now.timestamp() @@ -188,3 +191,276 @@ def test_test_post_outbox_handles_create_note( to=[actor.url], on_behalf_of=actors.SYSTEM_ACTORS['test'].get_actor_instance() ) + + +def test_getting_actor_instance_persists_in_db(db): + test = actors.SYSTEM_ACTORS['test'].get_actor_instance() + from_db = models.Actor.objects.get(url=test.url) + + for f in test._meta.fields: + assert getattr(from_db, f.name) == getattr(test, f.name) + + +@pytest.mark.parametrize('username,domain,expected', [ + ('test', 'wrongdomain.com', False), + ('notsystem', '', False), + ('test', '', True), +]) +def test_actor_is_system( + username, domain, expected, nodb_factories, settings): + if not domain: + domain = settings.FEDERATION_HOSTNAME + + actor = nodb_factories['federation.Actor']( + preferred_username=username, + domain=domain, + ) + assert actor.is_system is expected + + +@pytest.mark.parametrize('username,domain,expected', [ + ('test', 'wrongdomain.com', None), + ('notsystem', '', None), + ('test', '', actors.SYSTEM_ACTORS['test']), +]) +def test_actor_is_system( + username, domain, expected, nodb_factories, settings): + if not domain: + domain = settings.FEDERATION_HOSTNAME + actor = nodb_factories['federation.Actor']( + preferred_username=username, + domain=domain, + ) + assert actor.system_conf == expected + + +@pytest.mark.parametrize('value', [False, True]) +def test_library_actor_manually_approves_based_on_setting( + value, settings): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = value + library_conf = actors.SYSTEM_ACTORS['library'] + assert library_conf.manually_approves_followers is value + + +def test_system_actor_handle(mocker, nodb_factories): + handler = mocker.patch( + 'funkwhale_api.federation.actors.TestActor.handle_create') + actor = nodb_factories['federation.Actor']() + activity = nodb_factories['federation.Activity']( + type='Create', actor=actor.url) + serializer = serializers.ActivitySerializer( + data=activity + ) + assert serializer.is_valid() + actors.SYSTEM_ACTORS['test'].handle(activity, actor) + handler.assert_called_once_with(activity, actor) + + +def test_test_actor_handles_follow( + settings, mocker, factories): + deliver = mocker.patch( + 'funkwhale_api.federation.activity.deliver') + actor = factories['federation.Actor']() + now = timezone.now() + mocker.patch('django.utils.timezone.now', return_value=now) + accept_follow = mocker.patch( + 'funkwhale_api.federation.activity.accept_follow') + test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() + data = { + 'actor': actor.url, + 'type': 'Follow', + 'id': 'http://test.federation/user#follows/267', + 'object': test_actor.url, + } + uid = uuid.uuid4() + mocker.patch('uuid.uuid4', return_value=uid) + expected_follow = { + '@context': serializers.AP_CONTEXT, + 'actor': test_actor.url, + 'id': test_actor.url + '#follows/{}'.format(uid), + 'object': actor.url, + 'type': 'Follow' + } + + actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor) + accept_follow.assert_called_once_with( + test_actor, data, actor + ) + expected_calls = [ + mocker.call( + expected_follow, + to=[actor.url], + on_behalf_of=test_actor, + ) + ] + deliver.assert_has_calls(expected_calls) + + +def test_test_actor_handles_undo_follow( + settings, mocker, factories): + deliver = mocker.patch( + 'funkwhale_api.federation.activity.deliver') + test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() + follow = factories['federation.Follow'](target=test_actor) + reverse_follow = factories['federation.Follow']( + actor=test_actor, target=follow.actor) + follow_serializer = serializers.FollowSerializer(follow) + reverse_follow_serializer = serializers.FollowSerializer( + reverse_follow) + undo = { + '@context': serializers.AP_CONTEXT, + 'type': 'Undo', + 'id': follow_serializer.data['id'] + '/undo', + 'actor': follow.actor.url, + 'object': follow_serializer.data, + } + expected_undo = { + '@context': serializers.AP_CONTEXT, + 'type': 'Undo', + 'id': reverse_follow_serializer.data['id'] + '/undo', + 'actor': reverse_follow.actor.url, + 'object': reverse_follow_serializer.data, + } + + actors.SYSTEM_ACTORS['test'].post_inbox(undo, actor=follow.actor) + deliver.assert_called_once_with( + expected_undo, + to=[follow.actor.url], + on_behalf_of=test_actor,) + + assert models.Follow.objects.count() == 0 + + +def test_library_actor_handles_follow_manual_approval( + settings, mocker, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True + actor = factories['federation.Actor']() + now = timezone.now() + mocker.patch('django.utils.timezone.now', return_value=now) + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + data = { + 'actor': actor.url, + 'type': 'Follow', + 'id': 'http://test.federation/user#follows/267', + 'object': library_actor.url, + } + + library_actor.system_conf.post_inbox(data, actor=actor) + fr = library_actor.received_follow_requests.first() + + assert library_actor.received_follow_requests.count() == 1 + assert fr.target == library_actor + assert fr.actor == actor + assert fr.approved is None + + +def test_library_actor_handles_follow_auto_approval( + settings, mocker, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + actor = factories['federation.Actor']() + accept_follow = mocker.patch( + 'funkwhale_api.federation.activity.accept_follow') + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + data = { + 'actor': actor.url, + 'type': 'Follow', + 'id': 'http://test.federation/user#follows/267', + 'object': library_actor.url, + } + library_actor.system_conf.post_inbox(data, actor=actor) + + assert library_actor.received_follow_requests.count() == 0 + accept_follow.assert_called_once_with( + library_actor, data, actor + ) + + +def test_library_actor_handle_create_audio_no_library(mocker, factories): + # when we receive inbox create audio, we should not do anything + # if we don't have a configured library matching the sender + mocked_create = mocker.patch( + 'funkwhale_api.federation.serializers.AudioSerializer.create' + ) + actor = factories['federation.Actor']() + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + data = { + 'actor': actor.url, + 'type': 'Create', + 'id': 'http://test.federation/audio/create', + 'object': { + 'id': 'https://batch.import', + 'type': 'Collection', + 'totalItems': 2, + 'items': factories['federation.Audio'].create_batch(size=2) + }, + } + library_actor.system_conf.post_inbox(data, actor=actor) + + mocked_create.assert_not_called() + models.LibraryTrack.objects.count() == 0 + + +def test_library_actor_handle_create_audio_no_library_enabled( + mocker, factories): + # when we receive inbox create audio, we should not do anything + # if we don't have an enabled library + mocked_create = mocker.patch( + 'funkwhale_api.federation.serializers.AudioSerializer.create' + ) + disabled_library = factories['federation.Library']( + federation_enabled=False) + actor = disabled_library.actor + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + data = { + 'actor': actor.url, + 'type': 'Create', + 'id': 'http://test.federation/audio/create', + 'object': { + 'id': 'https://batch.import', + 'type': 'Collection', + 'totalItems': 2, + 'items': factories['federation.Audio'].create_batch(size=2) + }, + } + library_actor.system_conf.post_inbox(data, actor=actor) + + mocked_create.assert_not_called() + models.LibraryTrack.objects.count() == 0 + + +def test_library_actor_handle_create_audio(mocker, factories): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + remote_library = factories['federation.Library']( + federation_enabled=True + ) + + data = { + 'actor': remote_library.actor.url, + 'type': 'Create', + 'id': 'http://test.federation/audio/create', + 'object': { + 'id': 'https://batch.import', + 'type': 'Collection', + 'totalItems': 2, + 'items': factories['federation.Audio'].create_batch(size=2) + }, + } + + library_actor.system_conf.post_inbox(data, actor=remote_library.actor) + + lts = list(remote_library.tracks.order_by('id')) + + assert len(lts) == 2 + + for i, a in enumerate(data['object']['items']): + lt = lts[i] + assert lt.pk is not None + assert lt.url == a['id'] + assert lt.library == remote_library + assert lt.audio_url == a['url']['href'] + assert lt.audio_mimetype == a['url']['mediaType'] + assert lt.metadata == a['metadata'] + assert lt.title == a['metadata']['recording']['title'] + assert lt.artist_name == a['metadata']['artist']['name'] + assert lt.album_title == a['metadata']['release']['title'] + assert lt.published_date == arrow.get(a['published']) diff --git a/api/tests/federation/test_authentication.py b/api/tests/federation/test_authentication.py index 1837b3950..c6a97a07a 100644 --- a/api/tests/federation/test_authentication.py +++ b/api/tests/federation/test_authentication.py @@ -3,7 +3,7 @@ from funkwhale_api.federation import keys from funkwhale_api.federation import signing -def test_authenticate(nodb_factories, mocker, api_request): +def test_authenticate(factories, mocker, api_request): private, public = keys.get_key_pair() actor_url = 'https://test.federation/actor' mocker.patch( @@ -18,7 +18,7 @@ def test_authenticate(nodb_factories, mocker, api_request): 'id': actor_url + '#main-key', } }) - signed_request = nodb_factories['federation.SignedRequest']( + signed_request = factories['federation.SignedRequest']( auth__key=private, auth__key_id=actor_url + '#main-key', auth__headers=[ diff --git a/api/tests/federation/test_commands.py b/api/tests/federation/test_commands.py deleted file mode 100644 index 7c5333068..000000000 --- a/api/tests/federation/test_commands.py +++ /dev/null @@ -1,14 +0,0 @@ -from django.core.management import call_command - - -def test_generate_instance_key_pair(preferences, mocker): - mocker.patch( - 'funkwhale_api.federation.keys.get_key_pair', - return_value=(b'private', b'public')) - assert preferences['federation__public_key'] == '' - assert preferences['federation__private_key'] == '' - - call_command('generate_keys', interactive=False) - - assert preferences['federation__private_key'] == 'private' - assert preferences['federation__public_key'] == 'public' diff --git a/api/tests/federation/test_models.py b/api/tests/federation/test_models.py new file mode 100644 index 000000000..b17b6eb65 --- /dev/null +++ b/api/tests/federation/test_models.py @@ -0,0 +1,85 @@ +import pytest +import uuid + +from django import db + +from funkwhale_api.federation import models +from funkwhale_api.federation import serializers + + +def test_cannot_duplicate_actor(factories): + actor = factories['federation.Actor']() + + with pytest.raises(db.IntegrityError): + factories['federation.Actor']( + domain=actor.domain, + preferred_username=actor.preferred_username, + ) + + +def test_cannot_duplicate_follow(factories): + follow = factories['federation.Follow']() + + with pytest.raises(db.IntegrityError): + factories['federation.Follow']( + target=follow.target, + actor=follow.actor, + ) + + +def test_follow_federation_url(factories): + follow = factories['federation.Follow'](local=True) + expected = '{}#follows/{}'.format( + follow.actor.url, follow.uuid) + + assert follow.get_federation_url() == expected + + +def test_follow_request_approve(mocker, factories): + uid = uuid.uuid4() + mocker.patch('uuid.uuid4', return_value=uid) + accept_follow = mocker.patch( + 'funkwhale_api.federation.activity.accept_follow') + fr = factories['federation.FollowRequest'](target__local=True) + fr.approve() + + follow = { + '@context': serializers.AP_CONTEXT, + 'actor': fr.actor.url, + 'id': fr.actor.url + '#follows/{}'.format(uid), + 'object': fr.target.url, + 'type': 'Follow' + } + + assert fr.approved is True + assert list(fr.target.followers.all()) == [fr.actor] + accept_follow.assert_called_once_with( + fr.target, follow, fr.actor + ) + + +def test_follow_request_approve_non_local(mocker, factories): + uid = uuid.uuid4() + mocker.patch('uuid.uuid4', return_value=uid) + accept_follow = mocker.patch( + 'funkwhale_api.federation.activity.accept_follow') + fr = factories['federation.FollowRequest']() + fr.approve() + + assert fr.approved is True + assert list(fr.target.followers.all()) == [fr.actor] + accept_follow.assert_not_called() + + +def test_follow_request_refused(mocker, factories): + fr = factories['federation.FollowRequest']() + fr.refuse() + + assert fr.approved is False + assert fr.target.followers.count() == 0 + + +def test_library_model_unique_per_actor(factories): + library = factories['federation.Library']() + with pytest.raises(db.IntegrityError): + factories['federation.Library'](actor=library.actor) diff --git a/api/tests/federation/test_permissions.py b/api/tests/federation/test_permissions.py new file mode 100644 index 000000000..1a6977542 --- /dev/null +++ b/api/tests/federation/test_permissions.py @@ -0,0 +1,45 @@ +from rest_framework.views import APIView + +from funkwhale_api.federation import actors +from funkwhale_api.federation import permissions + + +def test_library_follower( + factories, api_request, anonymous_user, settings): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True + view = APIView.as_view() + permission = permissions.LibraryFollower() + request = api_request.get('/') + setattr(request, 'user', anonymous_user) + check = permission.has_permission(request, view) + + assert check is False + + +def test_library_follower_actor_non_follower( + factories, api_request, anonymous_user, settings): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True + actor = factories['federation.Actor']() + view = APIView.as_view() + permission = permissions.LibraryFollower() + request = api_request.get('/') + setattr(request, 'user', anonymous_user) + setattr(request, 'actor', actor) + check = permission.has_permission(request, view) + + assert check is False + + +def test_library_follower_actor_follower( + factories, api_request, anonymous_user, settings): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow = factories['federation.Follow'](target=library) + view = APIView.as_view() + permission = permissions.LibraryFollower() + request = api_request.get('/') + setattr(request, 'user', anonymous_user) + setattr(request, 'actor', follow.actor) + check = permission.has_permission(request, view) + + assert check is True diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index efa92b16a..45778ed48 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -1,31 +1,36 @@ -from django.urls import reverse +import arrow +from django.urls import reverse +from django.core.paginator import Paginator + +from funkwhale_api.federation import actors from funkwhale_api.federation import keys from funkwhale_api.federation import models from funkwhale_api.federation import serializers +from funkwhale_api.federation import utils def test_actor_serializer_from_ap(db): payload = { - 'id': 'https://test.federation/user', - 'type': 'Person', - 'following': 'https://test.federation/user/following', - 'followers': 'https://test.federation/user/followers', - 'inbox': 'https://test.federation/user/inbox', - 'outbox': 'https://test.federation/user/outbox', - 'preferredUsername': 'user', - 'name': 'Real User', - 'summary': 'Hello world', - 'url': 'https://test.federation/@user', - 'manuallyApprovesFollowers': False, - 'publicKey': { - 'id': 'https://test.federation/user#main-key', - 'owner': 'https://test.federation/user', - 'publicKeyPem': 'yolo' - }, - 'endpoints': { - 'sharedInbox': 'https://test.federation/inbox' - }, + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', + 'name': 'Real User', + 'summary': 'Hello world', + 'url': 'https://test.federation/@user', + 'manuallyApprovesFollowers': False, + 'publicKey': { + 'id': 'https://test.federation/user#main-key', + 'owner': 'https://test.federation/user', + 'publicKeyPem': 'yolo' + }, + 'endpoints': { + 'sharedInbox': 'https://test.federation/inbox' + }, } serializer = serializers.ActorSerializer(data=payload) @@ -50,13 +55,13 @@ def test_actor_serializer_from_ap(db): def test_actor_serializer_only_mandatory_field_from_ap(db): payload = { - 'id': 'https://test.federation/user', - 'type': 'Person', - 'following': 'https://test.federation/user/following', - 'followers': 'https://test.federation/user/followers', - 'inbox': 'https://test.federation/user/inbox', - 'outbox': 'https://test.federation/user/outbox', - 'preferredUsername': 'user', + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', } serializer = serializers.ActorSerializer(data=payload) @@ -82,24 +87,24 @@ def test_actor_serializer_to_ap(): 'https://w3id.org/security/v1', {}, ], - 'id': 'https://test.federation/user', - 'type': 'Person', - 'following': 'https://test.federation/user/following', - 'followers': 'https://test.federation/user/followers', - 'inbox': 'https://test.federation/user/inbox', - 'outbox': 'https://test.federation/user/outbox', - 'preferredUsername': 'user', - 'name': 'Real User', - 'summary': 'Hello world', - 'manuallyApprovesFollowers': False, - 'publicKey': { - 'id': 'https://test.federation/user#main-key', - 'owner': 'https://test.federation/user', - 'publicKeyPem': 'yolo' - }, - 'endpoints': { - 'sharedInbox': 'https://test.federation/inbox' - }, + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', + 'name': 'Real User', + 'summary': 'Hello world', + 'manuallyApprovesFollowers': False, + 'publicKey': { + 'id': 'https://test.federation/user#main-key', + 'owner': 'https://test.federation/user', + 'publicKeyPem': 'yolo' + }, + 'endpoints': { + 'sharedInbox': 'https://test.federation/inbox' + }, } ac = models.Actor( url=expected['id'], @@ -144,3 +149,229 @@ def test_webfinger_serializer(): serializer = serializers.ActorWebfingerSerializer(actor) assert serializer.data == expected + + +def test_follow_serializer_to_ap(factories): + follow = factories['federation.Follow'](local=True) + serializer = serializers.FollowSerializer(follow) + + expected = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url(), + 'type': 'Follow', + 'actor': follow.actor.url, + 'object': follow.target.url, + } + + assert serializer.data == expected + + +def test_paginated_collection_serializer(factories): + tfs = factories['music.TrackFile'].create_batch(size=5) + actor = factories['federation.Actor'](local=True) + + conf = { + 'id': 'https://test.federation/test', + 'items': tfs, + 'item_serializer': serializers.AudioSerializer, + 'actor': actor, + 'page_size': 2, + } + expected = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'type': 'Collection', + 'id': conf['id'], + 'actor': actor.url, + 'totalItems': len(tfs), + 'current': conf['id'] + '?page=1', + 'last': conf['id'] + '?page=3', + 'first': conf['id'] + '?page=1', + } + + serializer = serializers.PaginatedCollectionSerializer(conf) + + assert serializer.data == expected + + +def test_collection_page_serializer(factories): + tfs = factories['music.TrackFile'].create_batch(size=5) + actor = factories['federation.Actor'](local=True) + + conf = { + 'id': 'https://test.federation/test', + 'item_serializer': serializers.AudioSerializer, + 'actor': actor, + 'page': Paginator(tfs, 2).page(2), + } + expected = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'type': 'CollectionPage', + 'id': conf['id'] + '?page=2', + 'actor': actor.url, + 'totalItems': len(tfs), + 'partOf': conf['id'], + 'prev': conf['id'] + '?page=1', + 'next': conf['id'] + '?page=3', + 'first': conf['id'] + '?page=1', + 'last': conf['id'] + '?page=3', + 'items': [ + conf['item_serializer']( + i, + context={'actor': actor, 'include_ap_context': False} + ).data + for i in conf['page'].object_list + ] + } + + serializer = serializers.CollectionPageSerializer(conf) + + assert serializer.data == expected + + +def test_activity_pub_audio_serializer_to_library_track(factories): + remote_library = factories['federation.Library']() + audio = factories['federation.Audio']() + serializer = serializers.AudioSerializer( + data=audio, context={'library': remote_library}) + + assert serializer.is_valid(raise_exception=True) + + lt = serializer.save() + + assert lt.pk is not None + assert lt.url == audio['id'] + assert lt.library == remote_library + assert lt.audio_url == audio['url']['href'] + assert lt.audio_mimetype == audio['url']['mediaType'] + assert lt.metadata == audio['metadata'] + assert lt.title == audio['metadata']['recording']['title'] + assert lt.artist_name == audio['metadata']['artist']['name'] + assert lt.album_title == audio['metadata']['release']['title'] + assert lt.published_date == arrow.get(audio['published']) + + +def test_activity_pub_audio_serializer_to_ap(factories): + tf = factories['music.TrackFile'](mimetype='audio/mp3') + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + expected = { + '@context': serializers.AP_CONTEXT, + 'type': 'Audio', + 'id': tf.get_federation_url(), + 'name': tf.track.full_name, + 'published': tf.creation_date.isoformat(), + 'updated': tf.modification_date.isoformat(), + 'metadata': { + 'artist': { + 'musicbrainz_id': tf.track.artist.mbid, + 'name': tf.track.artist.name, + }, + 'release': { + 'musicbrainz_id': tf.track.album.mbid, + 'title': tf.track.album.title, + }, + 'recording': { + 'musicbrainz_id': tf.track.mbid, + 'title': tf.track.title, + }, + }, + 'url': { + 'href': utils.full_url(tf.path), + 'type': 'Link', + 'mediaType': 'audio/mp3' + }, + 'attributedTo': [ + library.url + ] + } + + serializer = serializers.AudioSerializer(tf, context={'actor': library}) + + assert serializer.data == expected + + +def test_activity_pub_audio_serializer_to_ap_no_mbid(factories): + tf = factories['music.TrackFile']( + mimetype='audio/mp3', + track__mbid=None, + track__album__mbid=None, + track__album__artist__mbid=None, + ) + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + expected = { + '@context': serializers.AP_CONTEXT, + 'type': 'Audio', + 'id': tf.get_federation_url(), + 'name': tf.track.full_name, + 'published': tf.creation_date.isoformat(), + 'updated': tf.modification_date.isoformat(), + 'metadata': { + 'artist': { + 'name': tf.track.artist.name, + 'musicbrainz_id': None, + }, + 'release': { + 'title': tf.track.album.title, + 'musicbrainz_id': None, + }, + 'recording': { + 'title': tf.track.title, + 'musicbrainz_id': None, + }, + }, + 'url': { + 'href': utils.full_url(tf.path), + 'type': 'Link', + 'mediaType': 'audio/mp3' + }, + 'attributedTo': [ + library.url + ] + } + + serializer = serializers.AudioSerializer(tf, context={'actor': library}) + + assert serializer.data == expected + + +def test_collection_serializer_to_ap(factories): + tf1 = factories['music.TrackFile'](mimetype='audio/mp3') + tf2 = factories['music.TrackFile'](mimetype='audio/ogg') + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + expected = { + '@context': serializers.AP_CONTEXT, + 'id': 'https://test.id', + 'actor': library.url, + 'totalItems': 2, + 'type': 'Collection', + 'items': [ + serializers.AudioSerializer( + tf1, context={'actor': library, 'include_ap_context': False} + ).data, + serializers.AudioSerializer( + tf2, context={'actor': library, 'include_ap_context': False} + ).data, + ] + } + + collection = { + 'id': expected['id'], + 'actor': library, + 'items': [tf1, tf2], + 'item_serializer': serializers.AudioSerializer + } + serializer = serializers.CollectionSerializer( + collection, context={'actor': library, 'id': 'https://test.id'}) + + assert serializer.data == expected diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index 0d2ac882f..c26810dad 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -1,13 +1,14 @@ from django.urls import reverse +from django.core.paginator import Paginator import pytest from funkwhale_api.federation import actors from funkwhale_api.federation import serializers +from funkwhale_api.federation import utils from funkwhale_api.federation import webfinger - @pytest.mark.parametrize('system_actor', actors.SYSTEM_ACTORS.keys()) def test_instance_actors(system_actor, db, settings, api_client): actor = actors.SYSTEM_ACTORS[system_actor].get_actor_instance() @@ -17,6 +18,8 @@ def test_instance_actors(system_actor, db, settings, api_client): response = api_client.get(url) serializer = serializers.ActorSerializer(actor) + if system_actor == 'library': + response.data.pop('url') assert response.status_code == 200 assert response.data == serializer.data @@ -62,3 +65,89 @@ def test_wellknown_webfinger_system( assert response.status_code == 200 assert response['Content-Type'] == 'application/jrd+json' assert response.data == serializer.data + + +def test_audio_file_list_requires_authenticated_actor( + db, settings, api_client): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True + url = reverse('federation:music:files-list') + response = api_client.get(url) + + assert response.status_code == 403 + + +def test_audio_file_list_actor_no_page( + db, settings, api_client, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + settings.FEDERATION_COLLECTION_PAGE_SIZE = 2 + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + tfs = factories['music.TrackFile'].create_batch(size=5) + conf = { + 'id': utils.full_url(reverse('federation:music:files-list')), + 'page_size': 2, + 'items': list(reversed(tfs)), # we order by -creation_date + 'item_serializer': serializers.AudioSerializer, + 'actor': library + } + expected = serializers.PaginatedCollectionSerializer(conf).data + url = reverse('federation:music:files-list') + response = api_client.get(url) + + assert response.status_code == 200 + assert response.data == expected + + +def test_audio_file_list_actor_page( + db, settings, api_client, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + settings.FEDERATION_COLLECTION_PAGE_SIZE = 2 + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + tfs = factories['music.TrackFile'].create_batch(size=5) + conf = { + 'id': utils.full_url(reverse('federation:music:files-list')), + 'page': Paginator(list(reversed(tfs)), 2).page(2), + 'item_serializer': serializers.AudioSerializer, + 'actor': library + } + expected = serializers.CollectionPageSerializer(conf).data + url = reverse('federation:music:files-list') + response = api_client.get(url, data={'page': 2}) + + assert response.status_code == 200 + assert response.data == expected + + +def test_audio_file_list_actor_page_error( + db, settings, api_client, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + url = reverse('federation:music:files-list') + response = api_client.get(url, data={'page': 'nope'}) + + assert response.status_code == 400 + + +def test_audio_file_list_actor_page_error_too_far( + db, settings, api_client, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + url = reverse('federation:music:files-list') + response = api_client.get(url, data={'page': 5000}) + + assert response.status_code == 404 + + +def test_library_actor_includes_library_link(db, settings, api_client): + actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + url = reverse( + 'federation:instance-actors-detail', + kwargs={'actor': 'library'}) + response = api_client.get(url) + expected_links = [ + { + 'type': 'Link', + 'name': 'library', + 'mediaType': 'application/activity+json', + 'href': utils.full_url(reverse('federation:music:files-list')) + } + ] + assert response.status_code == 200 + assert response.data['url'] == expected_links diff --git a/api/tests/music/test_import.py b/api/tests/music/test_import.py index 0f709e81f..a15f027ba 100644 --- a/api/tests/music/test_import.py +++ b/api/tests/music/test_import.py @@ -1,7 +1,10 @@ import json +import pytest from django.urls import reverse +from funkwhale_api.music import tasks + def test_create_import_can_bind_to_request( artists, albums, mocker, factories, superuser_api_client): @@ -33,3 +36,111 @@ def test_create_import_can_bind_to_request( batch = request.import_batches.latest('id') assert batch.import_request == request + + +def test_import_job_from_federation_no_musicbrainz(factories): + lt = factories['federation.LibraryTrack']( + artist_name='Hello', + album_title='World', + title='Ping', + ) + job = factories['music.ImportJob']( + federation=True, + library_track=lt, + ) + + tasks.import_job_run(import_job_id=job.pk) + job.refresh_from_db() + + tf = job.track_file + assert tf.mimetype == lt.audio_mimetype + assert tf.library_track == job.library_track + assert tf.track.title == 'Ping' + assert tf.track.artist.name == 'Hello' + assert tf.track.album.title == 'World' + + +def test_import_job_from_federation_musicbrainz_recording(factories, mocker): + t = factories['music.Track']() + track_from_api = mocker.patch( + 'funkwhale_api.music.models.Track.get_or_create_from_api', + return_value=t) + lt = factories['federation.LibraryTrack']( + metadata__recording__musicbrainz=True, + artist_name='Hello', + album_title='World', + ) + job = factories['music.ImportJob']( + federation=True, + library_track=lt, + ) + + tasks.import_job_run(import_job_id=job.pk) + job.refresh_from_db() + + tf = job.track_file + assert tf.mimetype == lt.audio_mimetype + assert tf.library_track == job.library_track + assert tf.track == t + track_from_api.assert_called_once_with( + mbid=lt.metadata['recording']['musicbrainz_id']) + + +def test_import_job_from_federation_musicbrainz_release(factories, mocker): + a = factories['music.Album']() + album_from_api = mocker.patch( + 'funkwhale_api.music.models.Album.get_or_create_from_api', + return_value=a) + lt = factories['federation.LibraryTrack']( + metadata__release__musicbrainz=True, + artist_name='Hello', + title='Ping', + ) + job = factories['music.ImportJob']( + federation=True, + library_track=lt, + ) + + tasks.import_job_run(import_job_id=job.pk) + job.refresh_from_db() + + tf = job.track_file + assert tf.mimetype == lt.audio_mimetype + assert tf.library_track == job.library_track + assert tf.track.title == 'Ping' + assert tf.track.artist == a.artist + assert tf.track.album == a + + album_from_api.assert_called_once_with( + mbid=lt.metadata['release']['musicbrainz_id']) + + +def test_import_job_from_federation_musicbrainz_artist(factories, mocker): + a = factories['music.Artist']() + artist_from_api = mocker.patch( + 'funkwhale_api.music.models.Artist.get_or_create_from_api', + return_value=a) + lt = factories['federation.LibraryTrack']( + metadata__artist__musicbrainz=True, + album_title='World', + title='Ping', + ) + job = factories['music.ImportJob']( + federation=True, + library_track=lt, + ) + + tasks.import_job_run(import_job_id=job.pk) + job.refresh_from_db() + + tf = job.track_file + assert tf.mimetype == lt.audio_mimetype + assert tf.library_track == job.library_track + + assert tf.track.title == 'Ping' + assert tf.track.artist == a + assert tf.track.album.artist == a + assert tf.track.album.title == 'World' + + artist_from_api.assert_called_once_with( + mbid=lt.metadata['artist']['musicbrainz_id']) diff --git a/api/tests/music/test_permissions.py b/api/tests/music/test_permissions.py new file mode 100644 index 000000000..6cce85e08 --- /dev/null +++ b/api/tests/music/test_permissions.py @@ -0,0 +1,56 @@ +from rest_framework.views import APIView + +from funkwhale_api.federation import actors +from funkwhale_api.music import permissions + + +def test_list_permission_no_protect(anonymous_user, api_request, settings): + settings.PROTECT_AUDIO_FILES = False + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + assert permission.has_permission(request, view) is True + + +def test_list_permission_protect_anonymous( + anonymous_user, api_request, settings): + settings.PROTECT_AUDIO_FILES = True + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + assert permission.has_permission(request, view) is False + + +def test_list_permission_protect_authenticated( + factories, api_request, settings): + settings.PROTECT_AUDIO_FILES = True + user = factories['users.User']() + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + setattr(request, 'user', user) + assert permission.has_permission(request, view) is True + + +def test_list_permission_protect_not_following_actor( + factories, api_request, settings): + settings.PROTECT_AUDIO_FILES = True + actor = factories['federation.Actor']() + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + setattr(request, 'actor', actor) + assert permission.has_permission(request, view) is False + + +def test_list_permission_protect_following_actor( + factories, api_request, settings): + settings.PROTECT_AUDIO_FILES = True + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow = factories['federation.Follow'](target=library_actor) + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + setattr(request, 'actor', follow.actor) + + assert permission.has_permission(request, view) is True diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py index 295604616..468ea77e3 100644 --- a/api/tests/music/test_views.py +++ b/api/tests/music/test_views.py @@ -1,6 +1,8 @@ +import io import pytest from funkwhale_api.music import views +from funkwhale_api.federation import actors @pytest.mark.parametrize('param,expected', [ @@ -43,3 +45,41 @@ def test_album_view_filter_listenable( queryset = view.filter_queryset(view.get_queryset()) assert list(queryset) == expected + + +def test_can_serve_track_file_as_remote_library( + factories, authenticated_actor, settings, api_client): + settings.PROTECT_AUDIO_FILES = True + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow = factories['federation.Follow']( + actor=authenticated_actor, target=library_actor) + + track_file = factories['music.TrackFile']() + response = api_client.get(track_file.path) + + assert response.status_code == 200 + assert response['X-Accel-Redirect'] == "{}{}".format( + settings.PROTECT_FILES_PATH, + track_file.audio_file.url) + + +def test_can_serve_track_file_as_remote_library_deny_not_following( + factories, authenticated_actor, settings, api_client): + settings.PROTECT_AUDIO_FILES = True + track_file = factories['music.TrackFile']() + response = api_client.get(track_file.path) + + assert response.status_code == 403 + + +def test_can_proxy_remote_track( + factories, settings, api_client, r_mock): + settings.PROTECT_AUDIO_FILES = False + track_file = factories['music.TrackFile'](federation=True) + + r_mock.get(track_file.library_track.audio_url, body=io.StringIO('test')) + response = api_client.get(track_file.path) + + assert response.status_code == 200 + assert list(response.streaming_content) == [b't', b'e', b's', b't'] + assert response['Content-Type'] == track_file.library_track.audio_mimetype diff --git a/deploy/env.prod.sample b/deploy/env.prod.sample index a016b34c7..9e9938500 100644 --- a/deploy/env.prod.sample +++ b/deploy/env.prod.sample @@ -85,3 +85,12 @@ API_AUTHENTICATION_REQUIRED=True # This will help us detect and correct bugs RAVEN_ENABLED=false RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5 + +# This settings enable/disable federation on the instance level +FEDERATION_ENABLED=True +# This setting decide wether music library is shared automatically +# to followers or if it requires manual approval before. +# FEDERATION_MUSIC_NEEDS_APPROVAL=False +# means anyone can subscribe to your library and import your file, +# use with caution. +FEDERATION_MUSIC_NEEDS_APPROVAL=True