MirrorMirror
/
funkwhale
mirror of https://dev.funkwhale.audio/funkwhale/funkwhale.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'federation-follow-request' into 'develop' 

Federation follow request

See merge request funkwhale/funkwhale!123
This commit is contained in:
Eliot Berriot 2018-04-07 13:55:53 +00:00
parent 17edc45866 bf70fa1f53
commit 99200ad077
40 changed files with 2474 additions and 270 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
api/config/settings/common.py
View File

@ -30,7 +30,12 @@ FUNKWHALE_HOSTNAME = urlsplit(FUNKWHALE_URL).netloc
FEDERATION_ENABLED = env.bool('FEDERATION_ENABLED', default=True)
FEDERATION_HOSTNAME = env('FEDERATION_HOSTNAME', default=FUNKWHALE_HOSTNAME)
FEDERATION_COLLECTION_PAGE_SIZE = env.int(
'FEDERATION_COLLECTION_PAGE_SIZE', default=50
)
FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
'FEDERATION_MUSIC_NEEDS_APPROVAL', default=True
)
ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
# APP CONFIGURATION

18
api/funkwhale_api/common/utils.py
View File

@ -1,3 +1,4 @@
from urllib.parse import urlencode, parse_qs, urlsplit, urlunsplit
import os
import shutil
@ -25,3 +26,20 @@ def on_commit(f, *args, **kwargs):
return transaction.on_commit(
lambda: f(*args, **kwargs)
)
def set_query_parameter(url, **kwargs):
"""Given a URL, set or replace a query parameter and return the
modified URL.
>>> set_query_parameter('http://example.com?foo=bar&biz=baz', 'foo', 'stuff')
'http://example.com?foo=stuff&biz=baz'
"""
scheme, netloc, path, query_string, fragment = urlsplit(url)
query_params = parse_qs(query_string)
for param_name, param_value in kwargs.items():
query_params[param_name] = [param_value]
new_query_string = urlencode(query_params, doseq=True)
return urlunsplit((scheme, netloc, path, new_query_string, fragment))

86
api/funkwhale_api/federation/activity.py
View File

@ -2,7 +2,9 @@ import logging
import json
import requests
import requests_http_signature
import uuid
from . import models
from . import signing
logger = logging.getLogger(__name__)
@ -42,33 +44,26 @@ ACTIVITY_TYPES = [
OBJECT_TYPES = [
'Article',
'Audio',
'Collection',
'Document',
'Event',
'Image',
'Note',
'OrderedCollection',
'Page',
'Place',
'Profile',
'Relationship',
'Tombstone',
'Video',
]
] + ACTIVITY_TYPES
def deliver(activity, on_behalf_of, to=[]):
from . import actors
logger.info('Preparing activity delivery to %s', to)
auth = requests_http_signature.HTTPSignatureAuth(
use_auth_header=False,
headers=[
'(request-target)',
'user-agent',
'host',
'date',
'content-type',],
algorithm='rsa-sha256',
key=on_behalf_of.private_key.encode('utf-8'),
key_id=on_behalf_of.private_key_id,
)
auth = signing.get_auth(
on_behalf_of.private_key, on_behalf_of.private_key_id)
for url in to:
recipient_actor = actors.get_actor(url)
logger.debug('delivering to %s', recipient_actor.inbox_url)
@ -83,3 +78,68 @@ def deliver(activity, on_behalf_of, to=[]):
)
response.raise_for_status()
logger.debug('Remote answered with %s', response.status_code)
def get_follow(follow_id, follower, followed):
return {
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{}
],
'actor': follower.url,
'id': follower.url + '#follows/{}'.format(follow_id),
'object': followed.url,
'type': 'Follow'
}
def get_undo(id, actor, object):
return {
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{}
],
'type': 'Undo',
'id': id + '/undo',
'actor': actor.url,
'object': object,
}
def get_accept_follow(accept_id, accept_actor, follow, follow_actor):
return {
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{}
],
"id": accept_actor.url + '#accepts/follows/{}'.format(
accept_id),
"type": "Accept",
"actor": accept_actor.url,
"object": {
"id": follow['id'],
"type": "Follow",
"actor": follow_actor.url,
"object": accept_actor.url
},
}
def accept_follow(target, follow, actor):
accept_uuid = uuid.uuid4()
accept = get_accept_follow(
accept_id=accept_uuid,
accept_actor=target,
follow=follow,
follow_actor=actor)
deliver(
accept,
to=[actor.url],
on_behalf_of=target)
return models.Follow.objects.get_or_create(
actor=actor,
target=target,
)

223
api/funkwhale_api/federation/actors.py
View File

@ -1,8 +1,10 @@
import logging
import requests
import uuid
import xml
from django.conf import settings
from django.db import transaction
from django.urls import reverse
from django.utils import timezone
@ -11,8 +13,10 @@ from rest_framework.exceptions import PermissionDenied
from dynamic_preferences.registries import global_preferences_registry
from . import activity
from . import keys
from . import models
from . import serializers
from . import signing
from . import utils
logger = logging.getLogger(__name__)
@ -47,31 +51,48 @@ def get_actor(actor_url):
class SystemActor(object):
additional_attributes = {}
manually_approves_followers = False
def get_request_auth(self):
actor = self.get_actor_instance()
return signing.get_auth(
actor.private_key, actor.private_key_id)
def serialize(self):
actor = self.get_actor_instance()
serializer = serializers.ActorSerializer(actor)
return serializer.data
def get_actor_instance(self):
a = models.Actor(
**self.get_instance_argument(
try:
return models.Actor.objects.get(url=self.get_actor_url())
except models.Actor.DoesNotExist:
pass
private, public = keys.get_key_pair()
args = self.get_instance_argument(
self.id,
name=self.name,
summary=self.summary,
**self.additional_attributes
)
)
a.pk = self.id
return a
args['private_key'] = private.decode('utf-8')
args['public_key'] = public.decode('utf-8')
return models.Actor.objects.create(**args)
def get_actor_url(self):
return utils.full_url(
reverse(
'federation:instance-actors-detail',
kwargs={'actor': self.id}))
def get_instance_argument(self, id, name, summary, **kwargs):
preferences = global_preferences_registry.manager()
p = {
'preferred_username': id,
'domain': settings.FEDERATION_HOSTNAME,
'type': 'Person',
'name': name.format(host=settings.FEDERATION_HOSTNAME),
'manually_approves_followers': True,
'url': utils.full_url(
reverse(
'federation:instance-actors-detail',
kwargs={'actor': id})),
'url': self.get_actor_url(),
'shared_inbox_url': utils.full_url(
reverse(
'federation:instance-actors-inbox',
@ -84,8 +105,6 @@ class SystemActor(object):
reverse(
'federation:instance-actors-outbox',
kwargs={'actor': id})),
'public_key': preferences['federation__public_key'],
'private_key': preferences['federation__private_key'],
'summary': summary.format(host=settings.FEDERATION_HOSTNAME)
}
p.update(kwargs)
@ -95,7 +114,7 @@ class SystemActor(object):
raise NotImplementedError
def post_inbox(self, data, actor=None):
raise NotImplementedError
return self.handle(data, actor=actor)
def get_outbox(self, data, actor=None):
raise NotImplementedError
@ -103,6 +122,62 @@ class SystemActor(object):
def post_outbox(self, data, actor=None):
raise NotImplementedError
def handle(self, data, actor=None):
"""
Main entrypoint for handling activities posted to the
actor's inbox
"""
logger.info('Received activity on %s inbox', self.id)
if actor is None:
raise PermissionDenied('Actor not authenticated')
serializer = serializers.ActivitySerializer(
data=data, context={'actor': actor})
serializer.is_valid(raise_exception=True)
ac = serializer.data
try:
handler = getattr(
self, 'handle_{}'.format(ac['type'].lower()))
except (KeyError, AttributeError):
logger.debug(
'No handler for activity %s', ac['type'])
return
return handler(data, actor)
def handle_follow(self, ac, sender):
system_actor = self.get_actor_instance()
if self.manually_approves_followers:
fr, created = models.FollowRequest.objects.get_or_create(
actor=sender,
target=system_actor,
approved=None,
)
return fr
return activity.accept_follow(
system_actor, ac, sender
)
def handle_undo_follow(self, ac, sender):
actor = self.get_actor_instance()
models.Follow.objects.filter(
actor=sender,
target=actor,
).delete()
def handle_undo(self, ac, sender):
if ac['object']['type'] != 'Follow':
return
if ac['object']['actor'] != sender.url:
# not the same actor, permission issue
return
self.handle_undo_follow(ac, sender)
class LibraryActor(SystemActor):
id = 'library'
@ -112,6 +187,62 @@ class LibraryActor(SystemActor):
'manually_approves_followers': True
}
def serialize(self):
data = super().serialize()
urls = data.setdefault('url', [])
urls.append({
'type': 'Link',
'mediaType': 'application/activity+json',
'name': 'library',
'href': utils.full_url(reverse('federation:music:files-list'))
})
return data
@property
def manually_approves_followers(self):
return settings.FEDERATION_MUSIC_NEEDS_APPROVAL
@transaction.atomic
def handle_create(self, ac, sender):
try:
remote_library = models.Library.objects.get(
actor=sender,
federation_enabled=True,
)
except models.Library.DoesNotExist:
logger.info(
'Skipping import, we\'re not following %s', sender.url)
return
if ac['object']['type'] != 'Collection':
return
if ac['object']['totalItems'] <= 0:
return
try:
items = ac['object']['items']
except KeyError:
logger.warning('No items in collection!')
return
item_serializers = [
serializers.AudioSerializer(
data=i, context={'library': remote_library})
for i in items
]
valid_serializers = []
for s in item_serializers:
if s.is_valid():
valid_serializers.append(s)
else:
logger.debug(
'Skipping invalid item %s, %s', s.initial_data, s.errors)
for s in valid_serializers:
s.save()
class TestActor(SystemActor):
id = 'test'
@ -123,6 +254,7 @@ class TestActor(SystemActor):
additional_attributes = {
'manually_approves_followers': False
}
manually_approves_followers = False
def get_outbox(self, data, actor=None):
return {
@ -140,23 +272,6 @@ class TestActor(SystemActor):
"orderedItems": []
}
def post_inbox(self, data, actor=None):
if actor is None:
raise PermissionDenied('Actor not authenticated')
serializer = serializers.ActivitySerializer(
data=data, context={'actor': actor})
serializer.is_valid(raise_exception=True)
ac = serializer.validated_data
logger.info('Received activity on %s inbox', self.id)
if ac['type'] == 'Create' and ac['object']['type'] == 'Note':
# we received a toot \o/
command = self.parse_command(ac['object']['content'])
logger.debug('Parsed command: %s', command)
if command == 'ping':
self.handle_ping(ac, actor)
def parse_command(self, message):
"""
Remove any links or fancy markup to extract /command from
@ -168,7 +283,16 @@ class TestActor(SystemActor):
except IndexError:
return
def handle_ping(self, ac, sender):
def handle_create(self, ac, sender):
if ac['object']['type'] != 'Note':
return
# we received a toot \o/
command = self.parse_command(ac['object']['content'])
logger.debug('Parsed command: %s', command)
if command != 'ping':
return
now = timezone.now()
test_actor = self.get_actor_instance()
reply_url = 'https://{}/activities/note/{}'.format(
@ -214,6 +338,43 @@ class TestActor(SystemActor):
to=[ac['actor']],
on_behalf_of=test_actor)
def handle_follow(self, ac, sender):
super().handle_follow(ac, sender)
# also, we follow back
test_actor = self.get_actor_instance()
follow_uuid = uuid.uuid4()
follow = activity.get_follow(
follow_id=follow_uuid,
follower=test_actor,
followed=sender)
activity.deliver(
follow,
to=[ac['actor']],
on_behalf_of=test_actor)
def handle_undo_follow(self, ac, sender):
super().handle_undo_follow(ac, sender)
actor = self.get_actor_instance()
# we also unfollow the sender, if possible
try:
follow = models.Follow.objects.get(
target=sender,
actor=actor,
)
except models.Follow.DoesNotExist:
return
undo = activity.get_undo(
id=follow.get_federation_url(),
actor=actor,
object=serializers.FollowSerializer(follow).data,
)
follow.delete()
activity.deliver(
undo,
to=[sender.url],
on_behalf_of=actor)
SYSTEM_ACTORS = {
'library': LibraryActor(),
'test': TestActor(),

8
api/funkwhale_api/federation/authentication.py
View File

@ -7,6 +7,7 @@ from rest_framework import exceptions
from . import actors
from . import keys
from . import models
from . import serializers
from . import signing
from . import utils
@ -42,11 +43,16 @@ class SignatureAuthentication(authentication.BaseAuthentication):
except cryptography.exceptions.InvalidSignature:
raise exceptions.AuthenticationFailed('Invalid signature')
return serializer.build()
try:
return models.Actor.objects.get(url=actor_data['id'])
except models.Actor.DoesNotExist:
return serializer.save()
def authenticate(self, request):
setattr(request, 'actor', None)
actor = self.authenticate_actor(request)
if not actor:
return
user = AnonymousUser()
setattr(request, 'actor', actor)
return (user, None)

28
api/funkwhale_api/federation/dynamic_preferences_registry.py
View File

@ -4,31 +4,3 @@ from dynamic_preferences import types
from dynamic_preferences.registries import global_preferences_registry
federation = types.Section('federation')
@global_preferences_registry.register
class FederationPrivateKey(types.StringPreference):
show_in_api = False
section = federation
name = 'private_key'
default = ''
help_text = (
'Instance private key, used for signing federation HTTP requests'
)
verbose_name = (
'Instance private key (keep it secret, do not change it)'
)
@global_preferences_registry.register
class FederationPublicKey(types.StringPreference):
show_in_api = False
section = federation
name = 'public_key'
default = ''
help_text = (
'Instance public key, used for signing federation HTTP requests'
)
verbose_name = (
'Instance public key (do not change it)'
)

168
api/funkwhale_api/federation/factories.py
View File

@ -1,8 +1,10 @@
import factory
import requests
import requests_http_signature
import uuid
from django.utils import timezone
from django.conf import settings
from funkwhale_api.factories import registry
@ -51,9 +53,23 @@ class SignedRequestFactory(factory.Factory):
self.headers.update(default_headers)
@registry.register(name='federation.Link')
class LinkFactory(factory.Factory):
type = 'Link'
href = factory.Faker('url')
mediaType = 'text/html'
class Meta:
model = dict
class Params:
audio = factory.Trait(
mediaType=factory.Iterator(['audio/mp3', 'audio/ogg'])
)
@registry.register
class ActorFactory(factory.DjangoModelFactory):
public_key = None
private_key = None
preferred_username = factory.Faker('user_name')
@ -66,6 +82,12 @@ class ActorFactory(factory.DjangoModelFactory):
class Meta:
model = models.Actor
class Params:
local = factory.Trait(
domain=factory.LazyAttribute(
lambda o: settings.FEDERATION_HOSTNAME)
)
@classmethod
def _generate(cls, create, attrs):
has_public = attrs.get('public_key') is not None
@ -77,6 +99,102 @@ class ActorFactory(factory.DjangoModelFactory):
return super()._generate(create, attrs)
@registry.register
class FollowFactory(factory.DjangoModelFactory):
target = factory.SubFactory(ActorFactory)
actor = factory.SubFactory(ActorFactory)
class Meta:
model = models.Follow
class Params:
local = factory.Trait(
actor=factory.SubFactory(ActorFactory, local=True)
)
@registry.register
class FollowRequestFactory(factory.DjangoModelFactory):
target = factory.SubFactory(ActorFactory)
actor = factory.SubFactory(ActorFactory)
class Meta:
model = models.FollowRequest
@registry.register
class LibraryFactory(factory.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory)
url = factory.Faker('url')
federation_enabled = True
download_files = False
autoimport = False
class Meta:
model = models.Library
class ArtistMetadataFactory(factory.Factory):
name = factory.Faker('name')
class Meta:
model = dict
class Params:
musicbrainz = factory.Trait(
musicbrainz_id=factory.Faker('uuid4')
)
class ReleaseMetadataFactory(factory.Factory):
title = factory.Faker('sentence')
class Meta:
model = dict
class Params:
musicbrainz = factory.Trait(
musicbrainz_id=factory.Faker('uuid4')
)
class RecordingMetadataFactory(factory.Factory):
title = factory.Faker('sentence')
class Meta:
model = dict
class Params:
musicbrainz = factory.Trait(
musicbrainz_id=factory.Faker('uuid4')
)
@registry.register(name='federation.LibraryTrackMetadata')
class LibraryTrackMetadataFactory(factory.Factory):
artist = factory.SubFactory(ArtistMetadataFactory)
recording = factory.SubFactory(RecordingMetadataFactory)
release = factory.SubFactory(ReleaseMetadataFactory)
class Meta:
model = dict
@registry.register
class LibraryTrackFactory(factory.DjangoModelFactory):
library = factory.SubFactory(LibraryFactory)
url = factory.Faker('url')
title = factory.Faker('sentence')
artist_name = factory.Faker('sentence')
album_title = factory.Faker('sentence')
audio_url = factory.Faker('url')
audio_mimetype = 'audio/ogg'
metadata = factory.SubFactory(LibraryTrackMetadataFactory)
class Meta:
model = models.LibraryTrack
@registry.register(name='federation.Note')
class NoteFactory(factory.Factory):
type = 'Note'
@ -89,3 +207,51 @@ class NoteFactory(factory.Factory):
class Meta:
model = dict
@registry.register(name='federation.Activity')
class ActivityFactory(factory.Factory):
type = 'Create'
id = factory.Faker('url')
published = factory.LazyFunction(
lambda: timezone.now().isoformat()
)
actor = factory.Faker('url')
object = factory.SubFactory(
NoteFactory,
actor=factory.SelfAttribute('..actor'),
published=factory.SelfAttribute('..published'))
class Meta:
model = dict
@registry.register(name='federation.AudioMetadata')
class AudioMetadataFactory(factory.Factory):
recording = factory.LazyAttribute(
lambda o: 'https://musicbrainz.org/recording/{}'.format(uuid.uuid4())
)
artist = factory.LazyAttribute(
lambda o: 'https://musicbrainz.org/artist/{}'.format(uuid.uuid4())
)
release = factory.LazyAttribute(
lambda o: 'https://musicbrainz.org/release/{}'.format(uuid.uuid4())
)
class Meta:
model = dict
@registry.register(name='federation.Audio')
class AudioFactory(factory.Factory):
type = 'Audio'
id = factory.Faker('url')
published = factory.LazyFunction(
lambda: timezone.now().isoformat()
)
actor = factory.Faker('url')
url = factory.SubFactory(LinkFactory, audio=True)
metadata = factory.SubFactory(LibraryTrackMetadataFactory)
class Meta:
model = dict

0
api/funkwhale_api/federation/management/__init__.py
View File

0
api/funkwhale_api/federation/management/commands/__init__.py
View File

53
api/funkwhale_api/federation/management/commands/generate_keys.py
View File

@ -1,53 +0,0 @@
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from dynamic_preferences.registries import global_preferences_registry
from funkwhale_api.federation import keys
class Command(BaseCommand):
help = (
'Generate a public/private key pair for your instance,'
' for federation purposes. If a key pair already exists, does nothing.'
)
def add_arguments(self, parser):
parser.add_argument(
'--replace',
action='store_true',
dest='replace',
default=False,
help='Replace existing key pair, if any',
)
parser.add_argument(
'--noinput', '--no-input', action='store_false', dest='interactive',
help="Do NOT prompt the user for input of any kind.",
)
@transaction.atomic
def handle(self, *args, **options):
preferences = global_preferences_registry.manager()
existing_public = preferences['federation__public_key']
existing_private = preferences['federation__public_key']
if existing_public or existing_private and not options['replace']:
raise CommandError(
'Keys are already present! '
'Replace them with --replace if you know what you are doing.')
if options['interactive']:
message = (
'Are you sure you want to do this?\n\n'
"Type 'yes' to continue, or 'no' to cancel: "
)
if input(''.join(message)) != 'yes':
raise CommandError("Operation cancelled.")
private, public = keys.get_key_pair()
preferences['federation__public_key'] = public.decode('utf-8')
preferences['federation__private_key'] = private.decode('utf-8')
self.stdout.write(
'Your new key pair was generated.'
'Your public key is now:\n\n{}'.format(public.decode('utf-8'))
)

17
api/funkwhale_api/federation/migrations/0002_auto_20180403_1620.py Normal file
View File

@ -0,0 +1,17 @@
# Generated by Django 2.0.3 on 2018-04-03 16:20
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('federation', '0001_initial'),
]
operations = [
migrations.AlterUniqueTogether(
name='actor',
unique_together={('domain', 'preferred_username')},
),
]

94
api/funkwhale_api/federation/migrations/0003_auto_20180407_1010.py Normal file
View File

@ -0,0 +1,94 @@
# Generated by Django 2.0.3 on 2018-04-07 10:10
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import uuid
def delete_system_actors(apps, schema_editor):
"""Revert site domain and name to default."""
Actor = apps.get_model("federation", "Actor")
Actor.objects.filter(preferred_username__in=['test', 'library']).delete()
def backward(apps, schema_editor):
pass
class Migration(migrations.Migration):
dependencies = [
('federation', '0002_auto_20180403_1620'),
]
operations = [
migrations.RunPython(delete_system_actors, backward),
migrations.CreateModel(
name='Follow',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('modification_date', models.DateTimeField(auto_now=True)),
('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='emitted_follows', to='federation.Actor')),
('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='received_follows', to='federation.Actor')),
],
),
migrations.CreateModel(
name='FollowRequest',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('modification_date', models.DateTimeField(auto_now=True)),
('approved', models.NullBooleanField(default=None)),
('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='emmited_follow_requests', to='federation.Actor')),
('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='received_follow_requests', to='federation.Actor')),
],
),
migrations.CreateModel(
name='Library',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('modification_date', models.DateTimeField(auto_now=True)),
('fetched_date', models.DateTimeField(blank=True, null=True)),
('uuid', models.UUIDField(default=uuid.uuid4)),
('url', models.URLField()),
('federation_enabled', models.BooleanField()),
('download_files', models.BooleanField()),
('autoimport', models.BooleanField()),
('tracks_count', models.PositiveIntegerField(blank=True, null=True)),
('actor', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='library', to='federation.Actor')),
],
),
migrations.CreateModel(
name='LibraryTrack',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.URLField(unique=True)),
('audio_url', models.URLField()),
('audio_mimetype', models.CharField(max_length=200)),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('modification_date', models.DateTimeField(auto_now=True)),
('fetched_date', models.DateTimeField(blank=True, null=True)),
('published_date', models.DateTimeField(blank=True, null=True)),
('artist_name', models.CharField(max_length=500)),
('album_title', models.CharField(max_length=500)),
('title', models.CharField(max_length=500)),
('metadata', django.contrib.postgres.fields.jsonb.JSONField(default={}, max_length=10000)),
('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tracks', to='federation.Library')),
],
),
migrations.AddField(
model_name='actor',
name='followers',
field=models.ManyToManyField(related_name='following', through='federation.Follow', to='federation.Actor'),
),
migrations.AlterUniqueTogether(
name='follow',
unique_together={('actor', 'target')},
),
]

139
api/funkwhale_api/federation/models.py
View File

@ -1,4 +1,7 @@
import uuid
from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.db import models
from django.utils import timezone
@ -12,6 +15,8 @@ TYPE_CHOICES = [
class Actor(models.Model):
ap_type = 'Actor'
url = models.URLField(unique=True, max_length=500, db_index=True)
outbox_url = models.URLField(max_length=500)
inbox_url = models.URLField(max_length=500)
@ -31,6 +36,16 @@ class Actor(models.Model):
last_fetch_date = models.DateTimeField(
default=timezone.now)
manually_approves_followers = models.NullBooleanField(default=None)
followers = models.ManyToManyField(
to='self',
symmetrical=False,
through='Follow',
through_fields=('target', 'actor'),
related_name='following',
)
class Meta:
unique_together = ['domain', 'preferred_username']
@property
def webfinger_subject(self):
@ -57,3 +72,127 @@ class Actor(models.Model):
setattr(self, field, v.lower())
super().save(**kwargs)
@property
def is_local(self):
return self.domain == settings.FEDERATION_HOSTNAME
@property
def is_system(self):
from . import actors
return all([
settings.FEDERATION_HOSTNAME == self.domain,
self.preferred_username in actors.SYSTEM_ACTORS
])
@property
def system_conf(self):
from . import actors
if self.is_system:
return actors.SYSTEM_ACTORS[self.preferred_username]
class Follow(models.Model):
ap_type = 'Follow'
uuid = models.UUIDField(default=uuid.uuid4, unique=True)
actor = models.ForeignKey(
Actor,
related_name='emitted_follows',
on_delete=models.CASCADE,
)
target = models.ForeignKey(
Actor,
related_name='received_follows',
on_delete=models.CASCADE,
)
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(
auto_now=True)
class Meta:
unique_together = ['actor', 'target']
def get_federation_url(self):
return '{}#follows/{}'.format(self.actor.url, self.uuid)
class FollowRequest(models.Model):
uuid = models.UUIDField(default=uuid.uuid4, unique=True)
actor = models.ForeignKey(
Actor,
related_name='emmited_follow_requests',
on_delete=models.CASCADE,
)
target = models.ForeignKey(
Actor,
related_name='received_follow_requests',
on_delete=models.CASCADE,
)
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(
auto_now=True)
approved = models.NullBooleanField(default=None)
def approve(self):
from . import activity
from . import serializers
self.approved = True
self.save(update_fields=['approved'])
Follow.objects.get_or_create(
target=self.target,
actor=self.actor
)
if self.target.is_local:
follow = {
'@context': serializers.AP_CONTEXT,
'actor': self.actor.url,
'id': self.actor.url + '#follows/{}'.format(uuid.uuid4()),
'object': self.target.url,
'type': 'Follow'
}
activity.accept_follow(
self.target, follow, self.actor
)
def refuse(self):
self.approved = False
self.save(update_fields=['approved'])
class Library(models.Model):
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(
auto_now=True)
fetched_date = models.DateTimeField(null=True, blank=True)
actor = models.OneToOneField(
Actor,
on_delete=models.CASCADE,
related_name='library')
uuid = models.UUIDField(default=uuid.uuid4)
url = models.URLField()
# use this flag to disable federation with a library
federation_enabled = models.BooleanField()
# should we mirror files locally or hotlink them?
download_files = models.BooleanField()
# should we automatically import new files from this library?
autoimport = models.BooleanField()
tracks_count = models.PositiveIntegerField(null=True, blank=True)
class LibraryTrack(models.Model):
url = models.URLField(unique=True)
audio_url = models.URLField()
audio_mimetype = models.CharField(max_length=200)
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(
auto_now=True)
fetched_date = models.DateTimeField(null=True, blank=True)
published_date = models.DateTimeField(null=True, blank=True)
library = models.ForeignKey(
Library, related_name='tracks', on_delete=models.CASCADE)
artist_name = models.CharField(max_length=500)
album_title = models.CharField(max_length=500)
title = models.CharField(max_length=500)
metadata = JSONField(default={}, max_length=10000)

19
api/funkwhale_api/federation/permissions.py Normal file
View File

@ -0,0 +1,19 @@
from django.conf import settings
from rest_framework.permissions import BasePermission
from . import actors
class LibraryFollower(BasePermission):
def has_permission(self, request, view):
if not settings.FEDERATION_MUSIC_NEEDS_APPROVAL:
return True
actor = getattr(request, 'actor', None)
if actor is None:
return False
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
return library.followers.filter(url=actor.url).exists()

251
api/funkwhale_api/federation/serializers.py
View File

@ -2,15 +2,25 @@ import urllib.parse
from django.urls import reverse
from django.conf import settings
from django.core.paginator import Paginator
from django.db import transaction
from rest_framework import serializers
from dynamic_preferences.registries import global_preferences_registry
from funkwhale_api.common.utils import set_query_parameter
from . import activity
from . import models
from . import utils
AP_CONTEXT = [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{},
]
class ActorSerializer(serializers.ModelSerializer):
# left maps to activitypub fields, right to our internal models
id = serializers.URLField(source='url')
@ -43,11 +53,7 @@ class ActorSerializer(serializers.ModelSerializer):
def to_representation(self, instance):
ret = super().to_representation(instance)
ret['@context'] = [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{},
]
ret['@context'] = AP_CONTEXT
if instance.public_key:
ret['publicKey'] = {
'owner': instance.url,
@ -87,6 +93,28 @@ class ActorSerializer(serializers.ModelSerializer):
return value[:500]
class FollowSerializer(serializers.ModelSerializer):
# left maps to activitypub fields, right to our internal models
id = serializers.URLField(source='get_federation_url')
object = serializers.URLField(source='target.url')
actor = serializers.URLField(source='actor.url')
type = serializers.CharField(source='ap_type')
class Meta:
model = models.Actor
fields = [
'id',
'object',
'actor',
'type'
]
def to_representation(self, instance):
ret = super().to_representation(instance)
ret['@context'] = AP_CONTEXT
return ret
class ActorWebfingerSerializer(serializers.ModelSerializer):
class Meta:
model = models.Actor
@ -120,7 +148,9 @@ class ActivitySerializer(serializers.Serializer):
type = value['type']
except KeyError:
raise serializers.ValidationError('Missing object type')
except TypeError:
# probably a URL
return value
try:
object_serializer = OBJECT_SERIALIZERS[type]
except KeyError:
@ -173,3 +203,212 @@ OBJECT_SERIALIZERS = {
t: ObjectSerializer
for t in activity.OBJECT_TYPES
}
class PaginatedCollectionSerializer(serializers.Serializer):
def to_representation(self, conf):
paginator = Paginator(
conf['items'],
conf.get('page_size', 20)
)
first = set_query_parameter(conf['id'], page=1)
current = first
last = set_query_parameter(conf['id'], page=paginator.num_pages)
d = {
'id': conf['id'],
'actor': conf['actor'].url,
'totalItems': paginator.count,
'type': 'Collection',
'current': current,
'first': first,
'last': last,
}
if self.context.get('include_ap_context', True):
d['@context'] = AP_CONTEXT
return d
class CollectionPageSerializer(serializers.Serializer):
def to_representation(self, conf):
page = conf['page']
first = set_query_parameter(conf['id'], page=1)
last = set_query_parameter(conf['id'], page=page.paginator.num_pages)
id = set_query_parameter(conf['id'], page=page.number)
d = {
'id': id,
'partOf': conf['id'],
'actor': conf['actor'].url,
'totalItems': page.paginator.count,
'type': 'CollectionPage',
'first': first,
'last': last,
'items': [
conf['item_serializer'](
i,
context={
'actor': conf['actor'],
'include_ap_context': False}
).data
for i in page.object_list
]
}
if page.has_previous():
d['prev'] = set_query_parameter(
conf['id'], page=page.previous_page_number())
if page.has_previous():
d['next'] = set_query_parameter(
conf['id'], page=page.next_page_number())
if self.context.get('include_ap_context', True):
d['@context'] = AP_CONTEXT
return d
class ArtistMetadataSerializer(serializers.Serializer):
musicbrainz_id = serializers.UUIDField(required=False)
name = serializers.CharField()
class ReleaseMetadataSerializer(serializers.Serializer):
musicbrainz_id = serializers.UUIDField(required=False)
title = serializers.CharField()
class RecordingMetadataSerializer(serializers.Serializer):
musicbrainz_id = serializers.UUIDField(required=False)
title = serializers.CharField()
class AudioMetadataSerializer(serializers.Serializer):
artist = ArtistMetadataSerializer()
release = ReleaseMetadataSerializer()
recording = RecordingMetadataSerializer()
class AudioSerializer(serializers.Serializer):
type = serializers.CharField()
id = serializers.URLField()
url = serializers.JSONField()
published = serializers.DateTimeField()
updated = serializers.DateTimeField(required=False)
metadata = AudioMetadataSerializer()
def validate_type(self, v):
if v != 'Audio':
raise serializers.ValidationError('Invalid type for audio')
return v
def validate_url(self, v):
try:
url = v['href']
except (KeyError, TypeError):
raise serializers.ValidationError('Missing href')
try:
media_type = v['mediaType']
except (KeyError, TypeError):
raise serializers.ValidationError('Missing mediaType')
if not media_type.startswith('audio/'):
raise serializers.ValidationError('Invalid mediaType')
return url
def validate_url(self, v):
try:
url = v['href']
except (KeyError, TypeError):
raise serializers.ValidationError('Missing href')
try:
media_type = v['mediaType']
except (KeyError, TypeError):
raise serializers.ValidationError('Missing mediaType')
if not media_type.startswith('audio/'):
raise serializers.ValidationError('Invalid mediaType')
return v
def create(self, validated_data):
defaults = {
'audio_mimetype': validated_data['url']['mediaType'],
'audio_url': validated_data['url']['href'],
'metadata': validated_data['metadata'],
'artist_name': validated_data['metadata']['artist']['name'],
'album_title': validated_data['metadata']['release']['title'],
'title': validated_data['metadata']['recording']['title'],
'published_date': validated_data['published'],
'modification_date': validated_data.get('updated'),
}
return models.LibraryTrack.objects.get_or_create(
library=self.context['library'],
url=validated_data['id'],
defaults=defaults
)[0]
def to_representation(self, instance):
track = instance.track
album = instance.track.album
artist = instance.track.artist
d = {
'type': 'Audio',
'id': instance.get_federation_url(),
'name': instance.track.full_name,
'published': instance.creation_date.isoformat(),
'updated': instance.modification_date.isoformat(),
'metadata': {
'artist': {
'musicbrainz_id': str(artist.mbid) if artist.mbid else None,
'name': artist.name,
},
'release': {
'musicbrainz_id': str(album.mbid) if album.mbid else None,
'title': album.title,
},
'recording': {
'musicbrainz_id': str(track.mbid) if track.mbid else None,
'title': track.title,
},
},
'url': {
'href': utils.full_url(instance.path),
'type': 'Link',
'mediaType': instance.mimetype
},
'attributedTo': [
self.context['actor'].url
]
}
if self.context.get('include_ap_context', True):
d['@context'] = AP_CONTEXT
return d
class CollectionSerializer(serializers.Serializer):
def to_representation(self, conf):
d = {
'id': conf['id'],
'actor': conf['actor'].url,
'totalItems': len(conf['items']),
'type': 'Collection',
'items': [
conf['item_serializer'](
i,
context={
'actor': conf['actor'],
'include_ap_context': False}
).data
for i in conf['items']
]
}
if self.context.get('include_ap_context', True):
d['@context'] = AP_CONTEXT
return d

15
api/funkwhale_api/federation/signing.py
View File

@ -53,3 +53,18 @@ def verify_django(django_request, public_key):
request.headers[h] = str(v)
prepared_request = request.prepare()
return verify(request, public_key)
def get_auth(private_key, private_key_id):
return requests_http_signature.HTTPSignatureAuth(
use_auth_header=False,
headers=[
'(request-target)',
'user-agent',
'host',
'date',
'content-type'],
algorithm='rsa-sha256',
key=private_key.encode('utf-8'),
key_id=private_key_id,
)

13
api/funkwhale_api/federation/urls.py
View File

@ -1,8 +1,10 @@
from rest_framework import routers
from django.conf.urls import include, url
from rest_framework import routers
from . import views
router = routers.SimpleRouter(trailing_slash=False)
music_router = routers.SimpleRouter(trailing_slash=False)
router.register(
r'federation/instance/actors',
views.InstanceActorViewSet,
@ -12,4 +14,11 @@ router.register(
views.WellKnownViewSet,
'well-known')
urlpatterns = router.urls
music_router.register(
r'files',
views.MusicFilesViewSet,
'files',
)
urlpatterns = router.urls + [
url('federation/music/', include((music_router.urls, 'music'), namespace='music'))
]

57
api/funkwhale_api/federation/views.py
View File

@ -1,16 +1,22 @@
from django import forms
from django.conf import settings
from django.core import paginator
from django.http import HttpResponse
from django.urls import reverse
from rest_framework import viewsets
from rest_framework import views
from rest_framework import response
from rest_framework.decorators import list_route, detail_route
from funkwhale_api.music.models import TrackFile
from . import actors
from . import authentication
from . import permissions
from . import renderers
from . import serializers
from . import utils
from . import webfinger
@ -38,8 +44,8 @@ class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet):
def retrieve(self, request, *args, **kwargs):
system_actor = self.get_object()
actor = system_actor.get_actor_instance()
serializer = serializers.ActorSerializer(actor)
return response.Response(serializer.data, status=200)
data = actor.system_conf.serialize()
return response.Response(data, status=200)
@detail_route(methods=['get', 'post'])
def inbox(self, request, *args, **kwargs):
@ -101,3 +107,50 @@ class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet):
username, hostname = clean_result
actor = actors.SYSTEM_ACTORS[username].get_actor_instance()
return serializers.ActorWebfingerSerializer(actor).data
class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet):
authentication_classes = [
authentication.SignatureAuthentication]
permission_classes = [permissions.LibraryFollower]
renderer_classes = [renderers.ActivityPubRenderer]
def list(self, request, *args, **kwargs):
page = request.GET.get('page')
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
qs = TrackFile.objects.order_by('-creation_date').select_related(
'track__artist',
'track__album__artist'
)
if page is None:
conf = {
'id': utils.full_url(reverse('federation:music:files-list')),
'page_size': settings.FEDERATION_COLLECTION_PAGE_SIZE,
'items': qs,
'item_serializer': serializers.AudioSerializer,
'actor': library,
}
serializer = serializers.PaginatedCollectionSerializer(conf)
data = serializer.data
else:
try:
page_number = int(page)
except:
return response.Response(
{'page': ['Invalid page number']}, status=400)
p = paginator.Paginator(
qs, settings.FEDERATION_COLLECTION_PAGE_SIZE)
try:
page = p.page(page_number)
conf = {
'id': utils.full_url(reverse('federation:music:files-list')),
'page': page,
'item_serializer': serializers.AudioSerializer,
'actor': library,
}
serializer = serializers.CollectionPageSerializer(conf)
data = serializer.data
except paginator.EmptyPage:
return response.Response(status=404)
return response.Response(data)

28
api/funkwhale_api/music/factories.py
View File

@ -2,6 +2,9 @@ import factory
import os
from funkwhale_api.factories import registry, ManyToManyFromList
from funkwhale_api.federation.factories import (
LibraryTrackFactory,
)
from funkwhale_api.users.factories import UserFactory
SAMPLES_PATH = os.path.join(
@ -53,6 +56,18 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
class Meta:
model = 'music.TrackFile'
class Params:
federation = factory.Trait(
audio_file=None,
library_track=factory.SubFactory(LibraryTrackFactory),
mimetype=factory.LazyAttribute(
lambda o: o.library_track.audio_mimetype
),
source=factory.LazyAttribute(
lambda o: o.library_track.audio_url
),
)
@registry.register
class ImportBatchFactory(factory.django.DjangoModelFactory):
@ -61,6 +76,12 @@ class ImportBatchFactory(factory.django.DjangoModelFactory):
class Meta:
model = 'music.ImportBatch'
class Params:
federation = factory.Trait(
submitted_by=None,
source='federation',
)
@registry.register
class ImportJobFactory(factory.django.DjangoModelFactory):
@ -71,6 +92,13 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
class Meta:
model = 'music.ImportJob'
class Params:
federation = factory.Trait(
mbid=None,
library_track=factory.SubFactory(LibraryTrackFactory),
batch=factory.SubFactory(ImportBatchFactory, federation=True),
)
@registry.register(name='music.FileImportJob')
class FileImportJobFactory(ImportJobFactory):

88
api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py Normal file
View File

@ -0,0 +1,88 @@
# Generated by Django 2.0.3 on 2018-04-07 10:10
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import uuid
class Migration(migrations.Migration):
dependencies = [
('federation', '0003_auto_20180407_1010'),
('music', '0022_importbatch_import_request'),
]
operations = [
migrations.AddField(
model_name='album',
name='uuid',
field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
),
migrations.AddField(
model_name='artist',
name='uuid',
field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
),
migrations.AddField(
model_name='importbatch',
name='uuid',
field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
),
migrations.AddField(
model_name='importjob',
name='library_track',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='import_jobs', to='federation.LibraryTrack'),
),
migrations.AddField(
model_name='importjob',
name='uuid',
field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
),
migrations.AddField(
model_name='lyrics',
name='uuid',
field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
),
migrations.AddField(
model_name='track',
name='uuid',
field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
),
migrations.AddField(
model_name='trackfile',
name='creation_date',
field=models.DateTimeField(default=django.utils.timezone.now),
),
migrations.AddField(
model_name='trackfile',
name='library_track',
field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='local_track_file', to='federation.LibraryTrack'),
),
migrations.AddField(
model_name='trackfile',
name='modification_date',
field=models.DateTimeField(auto_now=True),
),
migrations.AddField(
model_name='trackfile',
name='uuid',
field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
),
migrations.AddField(
model_name='work',
name='uuid',
field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
),
migrations.AlterField(
model_name='importbatch',
name='source',
field=models.CharField(choices=[('api', 'api'), ('shell', 'shell'), ('federation', 'federation')], default='api', max_length=30),
),
migrations.AlterField(
model_name='importbatch',
name='submitted_by',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to=settings.AUTH_USER_MODEL),
),
]

77
api/funkwhale_api/music/models.py
View File

@ -5,6 +5,7 @@ import datetime
import tempfile
import shutil
import markdown
import uuid
from django.conf import settings
from django.db import models
@ -20,12 +21,15 @@ from versatileimagefield.fields import VersatileImageField
from funkwhale_api import downloader
from funkwhale_api import musicbrainz
from funkwhale_api.federation import utils as federation_utils
from . import importers
from . import utils
class APIModelMixin(models.Model):
mbid = models.UUIDField(unique=True, db_index=True, null=True, blank=True)
uuid = models.UUIDField(
unique=True, db_index=True, default=uuid.uuid4)
api_includes = []
creation_date = models.DateTimeField(default=timezone.now)
import_hooks = []
@ -65,6 +69,13 @@ class APIModelMixin(models.Model):
pass
return cleaned_data
@property
def musicbrainz_url(self):
if self.mbid:
return 'https://musicbrainz.org/{}/{}'.format(
self.musicbrainz_model, self.mbid)
class Artist(APIModelMixin):
name = models.CharField(max_length=255)
@ -90,10 +101,19 @@ class Artist(APIModelMixin):
t.append(tag)
return set(t)
@classmethod
def get_or_create_from_name(cls, name, **kwargs):
kwargs.update({'name': name})
return cls.objects.get_or_create(
name__iexact=name,
defaults=kwargs)[0]
def import_artist(v):
a = Artist.get_or_create_from_api(mbid=v[0]['artist']['id'])[0]
return a
def parse_date(v):
if len(v) == 4:
return datetime.date(int(v), 1, 1)
@ -108,6 +128,7 @@ def import_tracks(instance, cleaned_data, raw_data):
track_cleaned_data['position'] = int(track_data['position'])
track = importers.load(Track, track_cleaned_data, track_data, Track.import_hooks)
class Album(APIModelMixin):
title = models.CharField(max_length=255)
artist = models.ForeignKey(
@ -170,6 +191,14 @@ class Album(APIModelMixin):
t.append(tag)
return set(t)
@classmethod
def get_or_create_from_title(cls, title, **kwargs):
kwargs.update({'title': title})
return cls.objects.get_or_create(
title__iexact=title,
defaults=kwargs)[0]
def import_tags(instance, cleaned_data, raw_data):
MINIMUM_COUNT = 2
tags_to_add = []
@ -182,6 +211,7 @@ def import_tags(instance, cleaned_data, raw_data):
tags_to_add.append(tag_data['name'])
instance.tags.add(*tags_to_add)
def import_album(v):
a = Album.get_or_create_from_api(mbid=v[0]['id'])[0]
return a
@ -248,6 +278,8 @@ class Work(APIModelMixin):
class Lyrics(models.Model):
uuid = models.UUIDField(
unique=True, db_index=True, default=uuid.uuid4)
work = models.ForeignKey(
Work,
related_name='lyrics',
@ -366,16 +398,35 @@ class Track(APIModelMixin):
self.mbid)
return settings.FUNKWHALE_URL + '/tracks/{}'.format(self.pk)
@classmethod
def get_or_create_from_title(cls, title, **kwargs):
kwargs.update({'title': title})
return cls.objects.get_or_create(
title__iexact=title,
defaults=kwargs)[0]
class TrackFile(models.Model):
uuid = models.UUIDField(
unique=True, db_index=True, default=uuid.uuid4)
track = models.ForeignKey(
Track, related_name='files', on_delete=models.CASCADE)
audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255)
source = models.URLField(null=True, blank=True)
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(auto_now=True)
duration = models.IntegerField(null=True, blank=True)
acoustid_track_id = models.UUIDField(null=True, blank=True)
mimetype = models.CharField(null=True, blank=True, max_length=200)
library_track = models.OneToOneField(
'federation.LibraryTrack',
related_name='local_track_file',
on_delete=models.CASCADE,
null=True,
blank=True,
)
def download_file(self):
# import the track file, since there is not any
# we create a tmp dir for the download
@ -391,12 +442,15 @@ class TrackFile(models.Model):
shutil.rmtree(tmp_dir)
return self.audio_file
def get_federation_url(self):
return federation_utils.full_url(
'/federation/music/file/{}'.format(self.uuid)
)
@property
def path(self):
if settings.PROTECT_AUDIO_FILES:
return reverse(
'api:v1:trackfiles-serve', kwargs={'pk': self.pk})
return self.audio_file.url
@property
def filename(self):
@ -417,10 +471,14 @@ IMPORT_STATUS_CHOICES = (
('skipped', 'Skipped'),
)
class ImportBatch(models.Model):
uuid = models.UUIDField(
unique=True, db_index=True, default=uuid.uuid4)
IMPORT_BATCH_SOURCES = [
('api', 'api'),
('shell', 'shell')
('shell', 'shell'),
('federation', 'federation'),
]
source = models.CharField(
max_length=30, default='api', choices=IMPORT_BATCH_SOURCES)
@ -428,6 +486,8 @@ class ImportBatch(models.Model):
submitted_by = models.ForeignKey(
'users.User',
related_name='imports',
null=True,
blank=True,
on_delete=models.CASCADE)
status = models.CharField(
choices=IMPORT_STATUS_CHOICES, default='pending', max_length=30)
@ -437,6 +497,7 @@ class ImportBatch(models.Model):
null=True,
blank=True,
on_delete=models.CASCADE)
class Meta:
ordering = ['-creation_date']
@ -449,6 +510,8 @@ class ImportBatch(models.Model):
class ImportJob(models.Model):
uuid = models.UUIDField(
unique=True, db_index=True, default=uuid.uuid4)
batch = models.ForeignKey(
ImportBatch, related_name='jobs', on_delete=models.CASCADE)
track_file = models.ForeignKey(
@ -465,6 +528,14 @@ class ImportJob(models.Model):
audio_file = models.FileField(
upload_to='imports/%Y/%m/%d', max_length=255, null=True, blank=True)
library_track = models.ForeignKey(
'federation.LibraryTrack',
related_name='import_jobs',
on_delete=models.SET_NULL,
null=True,
blank=True
)
class Meta:
ordering = ('id', )

23
api/funkwhale_api/music/permissions.py Normal file
View File

@ -0,0 +1,23 @@
from django.conf import settings
from rest_framework.permissions import BasePermission
from funkwhale_api.federation import actors
class Listen(BasePermission):
def has_permission(self, request, view):
if not settings.PROTECT_AUDIO_FILES:
return True
user = getattr(request, 'user', None)
if user and user.is_authenticated:
return True
actor = getattr(request, 'actor', None)
if actor is None:
return False
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
return library.followers.filter(url=actor.url).exists()

3
api/funkwhale_api/music/serializers.py
View File

@ -1,7 +1,10 @@
from django.db import transaction
from rest_framework import serializers
from taggit.models import Tag
from funkwhale_api.activity import serializers as activity_serializers
from funkwhale_api.federation.serializers import AP_CONTEXT
from funkwhale_api.federation import utils as federation_utils
from . import models

58
api/funkwhale_api/music/tasks.py
View File

@ -25,6 +25,48 @@ def set_acoustid_on_track_file(track_file):
return update(result['id'])
def import_track_from_remote(library_track):
metadata = library_track.metadata
try:
track_mbid = metadata['recording']['musicbrainz_id']
assert track_mbid # for null/empty values
except (KeyError, AssertionError):
pass
else:
return models.Track.get_or_create_from_api(mbid=track_mbid)
try:
album_mbid = metadata['release']['musicbrainz_id']
assert album_mbid # for null/empty values
except (KeyError, AssertionError):
pass
else:
album = models.Album.get_or_create_from_api(mbid=album_mbid)
return models.Track.get_or_create_from_title(
library_track.title, artist=album.artist, album=album)
try:
artist_mbid = metadata['artist']['musicbrainz_id']
assert artist_mbid # for null/empty values
except (KeyError, AssertionError):
pass
else:
artist = models.Artist.get_or_create_from_api(mbid=artist_mbid)
album = models.Album.get_or_create_from_title(
library_track.album_title, artist=artist)
return models.Track.get_or_create_from_title(
library_track.title, artist=artist, album=album)
# worst case scenario, we have absolutely no way to link to a
# musicbrainz resource, we rely on the name/titles
artist = models.Artist.get_or_create_from_name(
library_track.artist_name)
album = models.Album.get_or_create_from_title(
library_track.album_title, artist=artist)
return models.Track.get_or_create_from_title(
library_track.title, artist=artist, album=album)
def _do_import(import_job, replace, use_acoustid=True):
from_file = bool(import_job.audio_file)
mbid = import_job.mbid
@ -43,8 +85,14 @@ def _do_import(import_job, replace, use_acoustid=True):
acoustid_track_id = match['id']
if mbid:
track, _ = models.Track.get_or_create_from_api(mbid=mbid)
else:
elif import_job.audio_file:
track = import_track_data_from_path(import_job.audio_file.path)
elif import_job.library_track:
track = import_track_from_remote(import_job.library_track)
else:
raise ValueError(
'Not enough data to process import, '
'add a mbid, an audio file or a library track')
track_file = None
if replace:
@ -63,6 +111,14 @@ def _do_import(import_job, replace, use_acoustid=True):
track_file.audio_file = ContentFile(import_job.audio_file.read())
track_file.audio_file.name = import_job.audio_file.name
track_file.duration = duration
elif import_job.library_track:
track_file.library_track = import_job.library_track
track_file.mimetype = import_job.library_track.audio_mimetype
if import_job.library_track.library.download_files:
raise NotImplementedError()
else:
# no downloading, we hotlink
pass
else:
track_file.download_file()
track_file.save()

7
api/funkwhale_api/music/utils.py
View File

@ -60,3 +60,10 @@ def compute_status(jobs):
if pending:
return 'pending'
return 'finished'
def get_ext_from_type(mimetype):
mapping = {
'audio/ogg': 'ogg',
'audio/mpeg': 'mp3',
}

65
api/funkwhale_api/music/views.py
View File

@ -1,36 +1,42 @@
import ffmpeg
import os
import json
import requests
import subprocess
import unicodedata
import urllib
from django.urls import reverse
from django.contrib.auth.decorators import login_required
from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings
from django.db import models, transaction
from django.db.models.functions import Length
from django.conf import settings
from django.http import StreamingHttpResponse
from django.urls import reverse
from django.utils.decorators import method_decorator
from rest_framework import viewsets, views, mixins
from rest_framework.decorators import detail_route, list_route
from rest_framework.response import Response
from rest_framework import settings as rest_settings
from rest_framework import permissions
from musicbrainzngs import ResponseError
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from funkwhale_api.common import utils as funkwhale_utils
from funkwhale_api.federation import actors
from funkwhale_api.requests.models import ImportRequest
from funkwhale_api.musicbrainz import api
from funkwhale_api.common.permissions import (
ConditionalAuthentication, HasModelPermission)
from taggit.models import Tag
from funkwhale_api.federation.authentication import SignatureAuthentication
from . import forms
from . import models
from . import serializers
from . import importers
from . import filters
from . import forms
from . import importers
from . import models
from . import permissions as music_permissions
from . import serializers
from . import tasks
from . import utils
@ -45,6 +51,7 @@ class SearchMixin(object):
serializer = self.serializer_class(queryset, many=True)
return Response(serializer.data)
class TagViewSetMixin(object):
def get_queryset(self):
@ -179,22 +186,54 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
queryset = (models.TrackFile.objects.all().order_by('-id'))
serializer_class = serializers.TrackFileSerializer
permission_classes = [ConditionalAuthentication]
authentication_classes = rest_settings.api_settings.DEFAULT_AUTHENTICATION_CLASSES + [
SignatureAuthentication
]
permission_classes = [music_permissions.Listen]
@detail_route(methods=['get'])
def serve(self, request, *args, **kwargs):
try:
f = models.TrackFile.objects.get(pk=kwargs['pk'])
f = models.TrackFile.objects.select_related(
'library_track',
'track__album__artist',
'track__artist',
).get(pk=kwargs['pk'])
except models.TrackFile.DoesNotExist:
return Response(status=404)
mt = f.mimetype
try:
library_track = f.library_track
except ObjectDoesNotExist:
library_track = None
if library_track and not f.audio_file:
# we proxy the response to the remote library
# since we did not mirror the file locally
mt = library_track.audio_mimetype
file_extension = utils.get_ext_from_type(mt)
filename = '{}.{}'.format(f.track.full_name, file_extension)
auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
remote_response = requests.get(
library_track.audio_url,
auth=auth,
stream=True,
headers={
'Content-Type': 'application/activity+json'
})
response = StreamingHttpResponse(remote_response.iter_content())
else:
response = Response()
filename = "filename*=UTF-8''{}".format(
urllib.parse.quote(f.filename))
response["Content-Disposition"] = "attachment; {}".format(filename)
filename = f.filename
response['X-Accel-Redirect'] = "{}{}".format(
settings.PROTECT_FILES_PATH,
f.audio_file.url)
filename = "filename*=UTF-8''{}".format(
urllib.parse.quote(filename))
response["Content-Disposition"] = "attachment; {}".format(filename)
if mt:
response["Content-Type"] = mt
return response
@list_route(methods=['get'])

9
api/tests/conftest.py
View File

@ -162,3 +162,12 @@ def media_root(settings):
def r_mock():
with requests_mock.mock() as m:
yield m
@pytest.fixture
def authenticated_actor(factories, mocker):
actor = factories['federation.Actor']()
mocker.patch(
'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor',
return_value=actor)
yield actor

10
api/tests/federation/conftest.py
View File

@ -1,10 +0,0 @@
import pytest
@pytest.fixture
def authenticated_actor(nodb_factories, mocker):
actor = nodb_factories['federation.Actor']()
mocker.patch(
'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor',
return_value=actor)
yield actor

42
api/tests/federation/test_activity.py
View File

@ -1,5 +1,8 @@
import uuid
from funkwhale_api.federation import activity
def test_deliver(nodb_factories, r_mock, mocker):
to = nodb_factories['federation.Actor']()
mocker.patch(
@ -30,3 +33,42 @@ def test_deliver(nodb_factories, r_mock, mocker):
assert r_mock.call_count == 1
assert request.url == to.inbox_url
assert request.headers['content-type'] == 'application/activity+json'
def test_accept_follow(mocker, factories):
deliver = mocker.patch(
'funkwhale_api.federation.activity.deliver')
actor = factories['federation.Actor']()
target = factories['federation.Actor'](local=True)
follow = {
'actor': actor.url,
'type': 'Follow',
'id': 'http://test.federation/user#follows/267',
'object': target.url,
}
uid = uuid.uuid4()
mocker.patch('uuid.uuid4', return_value=uid)
expected_accept = {
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{}
],
"id": target.url + '#accepts/follows/{}'.format(uid),
"type": "Accept",
"actor": target.url,
"object": {
"id": follow['id'],
"type": "Follow",
"actor": actor.url,
"object": target.url
},
}
activity.accept_follow(
target, follow, actor
)
deliver.assert_called_once_with(
expected_accept, to=[actor.url], on_behalf_of=target
)
follow_instance = actor.emitted_follows.first()
assert follow_instance.target == target

300
api/tests/federation/test_actors.py
View File

@ -1,4 +1,6 @@
import arrow
import pytest
import uuid
from django.urls import reverse
from django.utils import timezone
@ -6,6 +8,7 @@ from django.utils import timezone
from rest_framework import exceptions
from funkwhale_api.federation import actors
from funkwhale_api.federation import models
from funkwhale_api.federation import serializers
from funkwhale_api.federation import utils
@ -23,14 +26,17 @@ def test_actor_fetching(r_mock):
assert r == payload
def test_get_library(settings, preferences):
preferences['federation__public_key'] = 'public_key'
def test_get_library(db, settings, mocker):
get_key_pair = mocker.patch(
'funkwhale_api.federation.keys.get_key_pair',
return_value=(b'private', b'public'))
expected = {
'preferred_username': 'library',
'domain': settings.FEDERATION_HOSTNAME,
'type': 'Person',
'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME),
'manually_approves_followers': True,
'public_key': 'public',
'url': utils.full_url(
reverse(
'federation:instance-actors-detail',
@ -47,7 +53,6 @@ def test_get_library(settings, preferences):
reverse(
'federation:instance-actors-outbox',
kwargs={'actor': 'library'})),
'public_key': 'public_key',
'summary': 'Bot account to federate with {}\'s library'.format(
settings.FEDERATION_HOSTNAME),
}
@ -56,14 +61,17 @@ def test_get_library(settings, preferences):
assert getattr(actor, key) == value
def test_get_test(settings, preferences):
preferences['federation__public_key'] = 'public_key'
def test_get_test(db, mocker, settings):
get_key_pair = mocker.patch(
'funkwhale_api.federation.keys.get_key_pair',
return_value=(b'private', b'public'))
expected = {
'preferred_username': 'test',
'domain': settings.FEDERATION_HOSTNAME,
'type': 'Person',
'name': '{}\'s test account'.format(settings.FEDERATION_HOSTNAME),
'manually_approves_followers': False,
'public_key': 'public',
'url': utils.full_url(
reverse(
'federation:instance-actors-detail',
@ -80,7 +88,6 @@ def test_get_test(settings, preferences):
reverse(
'federation:instance-actors-outbox',
kwargs={'actor': 'test'})),
'public_key': 'public_key',
'summary': 'Bot account to test federation with {}. Send me /ping and I\'ll answer you.'.format(
settings.FEDERATION_HOSTNAME),
}
@ -126,7 +133,7 @@ def test_test_post_outbox_validates_actor(nodb_factories):
assert msg in exc_info.value
def test_test_post_outbox_handles_create_note(
def test_test_post_inbox_handles_create_note(
settings, mocker, factories):
deliver = mocker.patch(
'funkwhale_api.federation.activity.deliver')
@ -167,11 +174,7 @@ def test_test_post_outbox_handles_create_note(
}]
)
expected_activity = {
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{}
],
'@context': serializers.AP_CONTEXT,
'actor': test_actor.url,
'id': 'https://{}/activities/note/{}/activity'.format(
settings.FEDERATION_HOSTNAME, now.timestamp()
@ -188,3 +191,276 @@ def test_test_post_outbox_handles_create_note(
to=[actor.url],
on_behalf_of=actors.SYSTEM_ACTORS['test'].get_actor_instance()
)
def test_getting_actor_instance_persists_in_db(db):
test = actors.SYSTEM_ACTORS['test'].get_actor_instance()
from_db = models.Actor.objects.get(url=test.url)
for f in test._meta.fields:
assert getattr(from_db, f.name) == getattr(test, f.name)
@pytest.mark.parametrize('username,domain,expected', [
('test', 'wrongdomain.com', False),
('notsystem', '', False),
('test', '', True),
])
def test_actor_is_system(
username, domain, expected, nodb_factories, settings):
if not domain:
domain = settings.FEDERATION_HOSTNAME
actor = nodb_factories['federation.Actor'](
preferred_username=username,
domain=domain,
)
assert actor.is_system is expected
@pytest.mark.parametrize('username,domain,expected', [
('test', 'wrongdomain.com', None),
('notsystem', '', None),
('test', '', actors.SYSTEM_ACTORS['test']),
])
def test_actor_is_system(
username, domain, expected, nodb_factories, settings):
if not domain:
domain = settings.FEDERATION_HOSTNAME
actor = nodb_factories['federation.Actor'](
preferred_username=username,
domain=domain,
)
assert actor.system_conf == expected
@pytest.mark.parametrize('value', [False, True])
def test_library_actor_manually_approves_based_on_setting(
value, settings):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = value
library_conf = actors.SYSTEM_ACTORS['library']
assert library_conf.manually_approves_followers is value
def test_system_actor_handle(mocker, nodb_factories):
handler = mocker.patch(
'funkwhale_api.federation.actors.TestActor.handle_create')
actor = nodb_factories['federation.Actor']()
activity = nodb_factories['federation.Activity'](
type='Create', actor=actor.url)
serializer = serializers.ActivitySerializer(
data=activity
)
assert serializer.is_valid()
actors.SYSTEM_ACTORS['test'].handle(activity, actor)
handler.assert_called_once_with(activity, actor)
def test_test_actor_handles_follow(
settings, mocker, factories):
deliver = mocker.patch(
'funkwhale_api.federation.activity.deliver')
actor = factories['federation.Actor']()
now = timezone.now()
mocker.patch('django.utils.timezone.now', return_value=now)
accept_follow = mocker.patch(
'funkwhale_api.federation.activity.accept_follow')
test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance()
data = {
'actor': actor.url,
'type': 'Follow',
'id': 'http://test.federation/user#follows/267',
'object': test_actor.url,
}
uid = uuid.uuid4()
mocker.patch('uuid.uuid4', return_value=uid)
expected_follow = {
'@context': serializers.AP_CONTEXT,
'actor': test_actor.url,
'id': test_actor.url + '#follows/{}'.format(uid),
'object': actor.url,
'type': 'Follow'
}
actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor)
accept_follow.assert_called_once_with(
test_actor, data, actor
)
expected_calls = [
mocker.call(
expected_follow,
to=[actor.url],
on_behalf_of=test_actor,
)
]
deliver.assert_has_calls(expected_calls)
def test_test_actor_handles_undo_follow(
settings, mocker, factories):
deliver = mocker.patch(
'funkwhale_api.federation.activity.deliver')
test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance()
follow = factories['federation.Follow'](target=test_actor)
reverse_follow = factories['federation.Follow'](
actor=test_actor, target=follow.actor)
follow_serializer = serializers.FollowSerializer(follow)
reverse_follow_serializer = serializers.FollowSerializer(
reverse_follow)
undo = {
'@context': serializers.AP_CONTEXT,
'type': 'Undo',
'id': follow_serializer.data['id'] + '/undo',
'actor': follow.actor.url,
'object': follow_serializer.data,
}
expected_undo = {
'@context': serializers.AP_CONTEXT,
'type': 'Undo',
'id': reverse_follow_serializer.data['id'] + '/undo',
'actor': reverse_follow.actor.url,
'object': reverse_follow_serializer.data,
}
actors.SYSTEM_ACTORS['test'].post_inbox(undo, actor=follow.actor)
deliver.assert_called_once_with(
expected_undo,
to=[follow.actor.url],
on_behalf_of=test_actor,)
assert models.Follow.objects.count() == 0
def test_library_actor_handles_follow_manual_approval(
settings, mocker, factories):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True
actor = factories['federation.Actor']()
now = timezone.now()
mocker.patch('django.utils.timezone.now', return_value=now)
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
data = {
'actor': actor.url,
'type': 'Follow',
'id': 'http://test.federation/user#follows/267',
'object': library_actor.url,
}
library_actor.system_conf.post_inbox(data, actor=actor)
fr = library_actor.received_follow_requests.first()
assert library_actor.received_follow_requests.count() == 1
assert fr.target == library_actor
assert fr.actor == actor
assert fr.approved is None
def test_library_actor_handles_follow_auto_approval(
settings, mocker, factories):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False
actor = factories['federation.Actor']()
accept_follow = mocker.patch(
'funkwhale_api.federation.activity.accept_follow')
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
data = {
'actor': actor.url,
'type': 'Follow',
'id': 'http://test.federation/user#follows/267',
'object': library_actor.url,
}
library_actor.system_conf.post_inbox(data, actor=actor)
assert library_actor.received_follow_requests.count() == 0
accept_follow.assert_called_once_with(
library_actor, data, actor
)
def test_library_actor_handle_create_audio_no_library(mocker, factories):
# when we receive inbox create audio, we should not do anything
# if we don't have a configured library matching the sender
mocked_create = mocker.patch(
'funkwhale_api.federation.serializers.AudioSerializer.create'
)
actor = factories['federation.Actor']()
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
data = {
'actor': actor.url,
'type': 'Create',
'id': 'http://test.federation/audio/create',
'object': {
'id': 'https://batch.import',
'type': 'Collection',
'totalItems': 2,
'items': factories['federation.Audio'].create_batch(size=2)
},
}
library_actor.system_conf.post_inbox(data, actor=actor)
mocked_create.assert_not_called()
models.LibraryTrack.objects.count() == 0
def test_library_actor_handle_create_audio_no_library_enabled(
mocker, factories):
# when we receive inbox create audio, we should not do anything
# if we don't have an enabled library
mocked_create = mocker.patch(
'funkwhale_api.federation.serializers.AudioSerializer.create'
)
disabled_library = factories['federation.Library'](
federation_enabled=False)
actor = disabled_library.actor
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
data = {
'actor': actor.url,
'type': 'Create',
'id': 'http://test.federation/audio/create',
'object': {
'id': 'https://batch.import',
'type': 'Collection',
'totalItems': 2,
'items': factories['federation.Audio'].create_batch(size=2)
},
}
library_actor.system_conf.post_inbox(data, actor=actor)
mocked_create.assert_not_called()
models.LibraryTrack.objects.count() == 0
def test_library_actor_handle_create_audio(mocker, factories):
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
remote_library = factories['federation.Library'](
federation_enabled=True
)
data = {
'actor': remote_library.actor.url,
'type': 'Create',
'id': 'http://test.federation/audio/create',
'object': {
'id': 'https://batch.import',
'type': 'Collection',
'totalItems': 2,
'items': factories['federation.Audio'].create_batch(size=2)
},
}
library_actor.system_conf.post_inbox(data, actor=remote_library.actor)
lts = list(remote_library.tracks.order_by('id'))
assert len(lts) == 2
for i, a in enumerate(data['object']['items']):
lt = lts[i]
assert lt.pk is not None
assert lt.url == a['id']
assert lt.library == remote_library
assert lt.audio_url == a['url']['href']
assert lt.audio_mimetype == a['url']['mediaType']
assert lt.metadata == a['metadata']
assert lt.title == a['metadata']['recording']['title']
assert lt.artist_name == a['metadata']['artist']['name']
assert lt.album_title == a['metadata']['release']['title']
assert lt.published_date == arrow.get(a['published'])

4
api/tests/federation/test_authentication.py
View File

@ -3,7 +3,7 @@ from funkwhale_api.federation import keys
from funkwhale_api.federation import signing
def test_authenticate(nodb_factories, mocker, api_request):
def test_authenticate(factories, mocker, api_request):
private, public = keys.get_key_pair()
actor_url = 'https://test.federation/actor'
mocker.patch(
@ -18,7 +18,7 @@ def test_authenticate(nodb_factories, mocker, api_request):
'id': actor_url + '#main-key',
}
})
signed_request = nodb_factories['federation.SignedRequest'](
signed_request = factories['federation.SignedRequest'](
auth__key=private,
auth__key_id=actor_url + '#main-key',
auth__headers=[

14
api/tests/federation/test_commands.py
View File

@ -1,14 +0,0 @@
from django.core.management import call_command
def test_generate_instance_key_pair(preferences, mocker):
mocker.patch(
'funkwhale_api.federation.keys.get_key_pair',
return_value=(b'private', b'public'))
assert preferences['federation__public_key'] == ''
assert preferences['federation__private_key'] == ''
call_command('generate_keys', interactive=False)
assert preferences['federation__private_key'] == 'private'
assert preferences['federation__public_key'] == 'public'

85
api/tests/federation/test_models.py Normal file
View File

@ -0,0 +1,85 @@
import pytest
import uuid
from django import db
from funkwhale_api.federation import models
from funkwhale_api.federation import serializers
def test_cannot_duplicate_actor(factories):
actor = factories['federation.Actor']()
with pytest.raises(db.IntegrityError):
factories['federation.Actor'](
domain=actor.domain,
preferred_username=actor.preferred_username,
)
def test_cannot_duplicate_follow(factories):
follow = factories['federation.Follow']()
with pytest.raises(db.IntegrityError):
factories['federation.Follow'](
target=follow.target,
actor=follow.actor,
)
def test_follow_federation_url(factories):
follow = factories['federation.Follow'](local=True)
expected = '{}#follows/{}'.format(
follow.actor.url, follow.uuid)
assert follow.get_federation_url() == expected
def test_follow_request_approve(mocker, factories):
uid = uuid.uuid4()
mocker.patch('uuid.uuid4', return_value=uid)
accept_follow = mocker.patch(
'funkwhale_api.federation.activity.accept_follow')
fr = factories['federation.FollowRequest'](target__local=True)
fr.approve()
follow = {
'@context': serializers.AP_CONTEXT,
'actor': fr.actor.url,
'id': fr.actor.url + '#follows/{}'.format(uid),
'object': fr.target.url,
'type': 'Follow'
}
assert fr.approved is True
assert list(fr.target.followers.all()) == [fr.actor]
accept_follow.assert_called_once_with(
fr.target, follow, fr.actor
)
def test_follow_request_approve_non_local(mocker, factories):
uid = uuid.uuid4()
mocker.patch('uuid.uuid4', return_value=uid)
accept_follow = mocker.patch(
'funkwhale_api.federation.activity.accept_follow')
fr = factories['federation.FollowRequest']()
fr.approve()
assert fr.approved is True
assert list(fr.target.followers.all()) == [fr.actor]
accept_follow.assert_not_called()
def test_follow_request_refused(mocker, factories):
fr = factories['federation.FollowRequest']()
fr.refuse()
assert fr.approved is False
assert fr.target.followers.count() == 0
def test_library_model_unique_per_actor(factories):
library = factories['federation.Library']()
with pytest.raises(db.IntegrityError):
factories['federation.Library'](actor=library.actor)

45
api/tests/federation/test_permissions.py Normal file
View File

@ -0,0 +1,45 @@
from rest_framework.views import APIView
from funkwhale_api.federation import actors
from funkwhale_api.federation import permissions
def test_library_follower(
factories, api_request, anonymous_user, settings):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True
view = APIView.as_view()
permission = permissions.LibraryFollower()
request = api_request.get('/')
setattr(request, 'user', anonymous_user)
check = permission.has_permission(request, view)
assert check is False
def test_library_follower_actor_non_follower(
factories, api_request, anonymous_user, settings):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True
actor = factories['federation.Actor']()
view = APIView.as_view()
permission = permissions.LibraryFollower()
request = api_request.get('/')
setattr(request, 'user', anonymous_user)
setattr(request, 'actor', actor)
check = permission.has_permission(request, view)
assert check is False
def test_library_follower_actor_follower(
factories, api_request, anonymous_user, settings):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
follow = factories['federation.Follow'](target=library)
view = APIView.as_view()
permission = permissions.LibraryFollower()
request = api_request.get('/')
setattr(request, 'user', anonymous_user)
setattr(request, 'actor', follow.actor)
check = permission.has_permission(request, view)
assert check is True

233
api/tests/federation/test_serializers.py
View File

@ -1,8 +1,13 @@
from django.urls import reverse
import arrow
from django.urls import reverse
from django.core.paginator import Paginator
from funkwhale_api.federation import actors
from funkwhale_api.federation import keys
from funkwhale_api.federation import models
from funkwhale_api.federation import serializers
from funkwhale_api.federation import utils
def test_actor_serializer_from_ap(db):
@ -144,3 +149,229 @@ def test_webfinger_serializer():
serializer = serializers.ActorWebfingerSerializer(actor)
assert serializer.data == expected
def test_follow_serializer_to_ap(factories):
follow = factories['federation.Follow'](local=True)
serializer = serializers.FollowSerializer(follow)
expected = {
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{},
],
'id': follow.get_federation_url(),
'type': 'Follow',
'actor': follow.actor.url,
'object': follow.target.url,
}
assert serializer.data == expected
def test_paginated_collection_serializer(factories):
tfs = factories['music.TrackFile'].create_batch(size=5)
actor = factories['federation.Actor'](local=True)
conf = {
'id': 'https://test.federation/test',
'items': tfs,
'item_serializer': serializers.AudioSerializer,
'actor': actor,
'page_size': 2,
}
expected = {
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{},
],
'type': 'Collection',
'id': conf['id'],
'actor': actor.url,
'totalItems': len(tfs),
'current': conf['id'] + '?page=1',
'last': conf['id'] + '?page=3',
'first': conf['id'] + '?page=1',
}
serializer = serializers.PaginatedCollectionSerializer(conf)
assert serializer.data == expected
def test_collection_page_serializer(factories):
tfs = factories['music.TrackFile'].create_batch(size=5)
actor = factories['federation.Actor'](local=True)
conf = {
'id': 'https://test.federation/test',
'item_serializer': serializers.AudioSerializer,
'actor': actor,
'page': Paginator(tfs, 2).page(2),
}
expected = {
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{},
],
'type': 'CollectionPage',
'id': conf['id'] + '?page=2',
'actor': actor.url,
'totalItems': len(tfs),
'partOf': conf['id'],
'prev': conf['id'] + '?page=1',
'next': conf['id'] + '?page=3',
'first': conf['id'] + '?page=1',
'last': conf['id'] + '?page=3',
'items': [
conf['item_serializer'](
i,
context={'actor': actor, 'include_ap_context': False}
).data
for i in conf['page'].object_list
]
}
serializer = serializers.CollectionPageSerializer(conf)
assert serializer.data == expected
def test_activity_pub_audio_serializer_to_library_track(factories):
remote_library = factories['federation.Library']()
audio = factories['federation.Audio']()
serializer = serializers.AudioSerializer(
data=audio, context={'library': remote_library})
assert serializer.is_valid(raise_exception=True)
lt = serializer.save()
assert lt.pk is not None
assert lt.url == audio['id']
assert lt.library == remote_library
assert lt.audio_url == audio['url']['href']
assert lt.audio_mimetype == audio['url']['mediaType']
assert lt.metadata == audio['metadata']
assert lt.title == audio['metadata']['recording']['title']
assert lt.artist_name == audio['metadata']['artist']['name']
assert lt.album_title == audio['metadata']['release']['title']
assert lt.published_date == arrow.get(audio['published'])
def test_activity_pub_audio_serializer_to_ap(factories):
tf = factories['music.TrackFile'](mimetype='audio/mp3')
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
expected = {
'@context': serializers.AP_CONTEXT,
'type': 'Audio',
'id': tf.get_federation_url(),
'name': tf.track.full_name,
'published': tf.creation_date.isoformat(),
'updated': tf.modification_date.isoformat(),
'metadata': {
'artist': {
'musicbrainz_id': tf.track.artist.mbid,
'name': tf.track.artist.name,
},
'release': {
'musicbrainz_id': tf.track.album.mbid,
'title': tf.track.album.title,
},
'recording': {
'musicbrainz_id': tf.track.mbid,
'title': tf.track.title,
},
},
'url': {
'href': utils.full_url(tf.path),
'type': 'Link',
'mediaType': 'audio/mp3'
},
'attributedTo': [
library.url
]
}
serializer = serializers.AudioSerializer(tf, context={'actor': library})
assert serializer.data == expected
def test_activity_pub_audio_serializer_to_ap_no_mbid(factories):
tf = factories['music.TrackFile'](
mimetype='audio/mp3',
track__mbid=None,
track__album__mbid=None,
track__album__artist__mbid=None,
)
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
expected = {
'@context': serializers.AP_CONTEXT,
'type': 'Audio',
'id': tf.get_federation_url(),
'name': tf.track.full_name,
'published': tf.creation_date.isoformat(),
'updated': tf.modification_date.isoformat(),
'metadata': {
'artist': {
'name': tf.track.artist.name,
'musicbrainz_id': None,
},
'release': {
'title': tf.track.album.title,
'musicbrainz_id': None,
},
'recording': {
'title': tf.track.title,
'musicbrainz_id': None,
},
},
'url': {
'href': utils.full_url(tf.path),
'type': 'Link',
'mediaType': 'audio/mp3'
},
'attributedTo': [
library.url
]
}
serializer = serializers.AudioSerializer(tf, context={'actor': library})
assert serializer.data == expected
def test_collection_serializer_to_ap(factories):
tf1 = factories['music.TrackFile'](mimetype='audio/mp3')
tf2 = factories['music.TrackFile'](mimetype='audio/ogg')
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
expected = {
'@context': serializers.AP_CONTEXT,
'id': 'https://test.id',
'actor': library.url,
'totalItems': 2,
'type': 'Collection',
'items': [
serializers.AudioSerializer(
tf1, context={'actor': library, 'include_ap_context': False}
).data,
serializers.AudioSerializer(
tf2, context={'actor': library, 'include_ap_context': False}
).data,
]
}
collection = {
'id': expected['id'],
'actor': library,
'items': [tf1, tf2],
'item_serializer': serializers.AudioSerializer
}
serializer = serializers.CollectionSerializer(
collection, context={'actor': library, 'id': 'https://test.id'})
assert serializer.data == expected

91
api/tests/federation/test_views.py
View File

@ -1,13 +1,14 @@
from django.urls import reverse
from django.core.paginator import Paginator
import pytest
from funkwhale_api.federation import actors
from funkwhale_api.federation import serializers
from funkwhale_api.federation import utils
from funkwhale_api.federation import webfinger
@pytest.mark.parametrize('system_actor', actors.SYSTEM_ACTORS.keys())
def test_instance_actors(system_actor, db, settings, api_client):
actor = actors.SYSTEM_ACTORS[system_actor].get_actor_instance()
@ -17,6 +18,8 @@ def test_instance_actors(system_actor, db, settings, api_client):
response = api_client.get(url)
serializer = serializers.ActorSerializer(actor)
if system_actor == 'library':
response.data.pop('url')
assert response.status_code == 200
assert response.data == serializer.data
@ -62,3 +65,89 @@ def test_wellknown_webfinger_system(
assert response.status_code == 200
assert response['Content-Type'] == 'application/jrd+json'
assert response.data == serializer.data
def test_audio_file_list_requires_authenticated_actor(
db, settings, api_client):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True
url = reverse('federation:music:files-list')
response = api_client.get(url)
assert response.status_code == 403
def test_audio_file_list_actor_no_page(
db, settings, api_client, factories):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False
settings.FEDERATION_COLLECTION_PAGE_SIZE = 2
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
tfs = factories['music.TrackFile'].create_batch(size=5)
conf = {
'id': utils.full_url(reverse('federation:music:files-list')),
'page_size': 2,
'items': list(reversed(tfs)), # we order by -creation_date
'item_serializer': serializers.AudioSerializer,
'actor': library
}
expected = serializers.PaginatedCollectionSerializer(conf).data
url = reverse('federation:music:files-list')
response = api_client.get(url)
assert response.status_code == 200
assert response.data == expected
def test_audio_file_list_actor_page(
db, settings, api_client, factories):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False
settings.FEDERATION_COLLECTION_PAGE_SIZE = 2
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
tfs = factories['music.TrackFile'].create_batch(size=5)
conf = {
'id': utils.full_url(reverse('federation:music:files-list')),
'page': Paginator(list(reversed(tfs)), 2).page(2),
'item_serializer': serializers.AudioSerializer,
'actor': library
}
expected = serializers.CollectionPageSerializer(conf).data
url = reverse('federation:music:files-list')
response = api_client.get(url, data={'page': 2})
assert response.status_code == 200
assert response.data == expected
def test_audio_file_list_actor_page_error(
db, settings, api_client, factories):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False
url = reverse('federation:music:files-list')
response = api_client.get(url, data={'page': 'nope'})
assert response.status_code == 400
def test_audio_file_list_actor_page_error_too_far(
db, settings, api_client, factories):
settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False
url = reverse('federation:music:files-list')
response = api_client.get(url, data={'page': 5000})
assert response.status_code == 404
def test_library_actor_includes_library_link(db, settings, api_client):
actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
url = reverse(
'federation:instance-actors-detail',
kwargs={'actor': 'library'})
response = api_client.get(url)
expected_links = [
{
'type': 'Link',
'name': 'library',
'mediaType': 'application/activity+json',
'href': utils.full_url(reverse('federation:music:files-list'))
}
]
assert response.status_code == 200
assert response.data['url'] == expected_links

111
api/tests/music/test_import.py
View File

@ -1,7 +1,10 @@
import json
import pytest
from django.urls import reverse
from funkwhale_api.music import tasks
def test_create_import_can_bind_to_request(
artists, albums, mocker, factories, superuser_api_client):
@ -33,3 +36,111 @@ def test_create_import_can_bind_to_request(
batch = request.import_batches.latest('id')
assert batch.import_request == request
def test_import_job_from_federation_no_musicbrainz(factories):
lt = factories['federation.LibraryTrack'](
artist_name='Hello',
album_title='World',
title='Ping',
)
job = factories['music.ImportJob'](
federation=True,
library_track=lt,
)
tasks.import_job_run(import_job_id=job.pk)
job.refresh_from_db()
tf = job.track_file
assert tf.mimetype == lt.audio_mimetype
assert tf.library_track == job.library_track
assert tf.track.title == 'Ping'
assert tf.track.artist.name == 'Hello'
assert tf.track.album.title == 'World'
def test_import_job_from_federation_musicbrainz_recording(factories, mocker):
t = factories['music.Track']()
track_from_api = mocker.patch(
'funkwhale_api.music.models.Track.get_or_create_from_api',
return_value=t)
lt = factories['federation.LibraryTrack'](
metadata__recording__musicbrainz=True,
artist_name='Hello',
album_title='World',
)
job = factories['music.ImportJob'](
federation=True,
library_track=lt,
)
tasks.import_job_run(import_job_id=job.pk)
job.refresh_from_db()
tf = job.track_file
assert tf.mimetype == lt.audio_mimetype
assert tf.library_track == job.library_track
assert tf.track == t
track_from_api.assert_called_once_with(
mbid=lt.metadata['recording']['musicbrainz_id'])
def test_import_job_from_federation_musicbrainz_release(factories, mocker):
a = factories['music.Album']()
album_from_api = mocker.patch(
'funkwhale_api.music.models.Album.get_or_create_from_api',
return_value=a)
lt = factories['federation.LibraryTrack'](
metadata__release__musicbrainz=True,
artist_name='Hello',
title='Ping',
)
job = factories['music.ImportJob'](
federation=True,
library_track=lt,
)
tasks.import_job_run(import_job_id=job.pk)
job.refresh_from_db()
tf = job.track_file
assert tf.mimetype == lt.audio_mimetype
assert tf.library_track == job.library_track
assert tf.track.title == 'Ping'
assert tf.track.artist == a.artist
assert tf.track.album == a
album_from_api.assert_called_once_with(
mbid=lt.metadata['release']['musicbrainz_id'])
def test_import_job_from_federation_musicbrainz_artist(factories, mocker):
a = factories['music.Artist']()
artist_from_api = mocker.patch(
'funkwhale_api.music.models.Artist.get_or_create_from_api',
return_value=a)
lt = factories['federation.LibraryTrack'](
metadata__artist__musicbrainz=True,
album_title='World',
title='Ping',
)
job = factories['music.ImportJob'](
federation=True,
library_track=lt,
)
tasks.import_job_run(import_job_id=job.pk)
job.refresh_from_db()
tf = job.track_file
assert tf.mimetype == lt.audio_mimetype
assert tf.library_track == job.library_track
assert tf.track.title == 'Ping'
assert tf.track.artist == a
assert tf.track.album.artist == a
assert tf.track.album.title == 'World'
artist_from_api.assert_called_once_with(
mbid=lt.metadata['artist']['musicbrainz_id'])

56
api/tests/music/test_permissions.py Normal file
View File

@ -0,0 +1,56 @@
from rest_framework.views import APIView
from funkwhale_api.federation import actors
from funkwhale_api.music import permissions
def test_list_permission_no_protect(anonymous_user, api_request, settings):
settings.PROTECT_AUDIO_FILES = False
view = APIView.as_view()
permission = permissions.Listen()
request = api_request.get('/')
assert permission.has_permission(request, view) is True
def test_list_permission_protect_anonymous(
anonymous_user, api_request, settings):
settings.PROTECT_AUDIO_FILES = True
view = APIView.as_view()
permission = permissions.Listen()
request = api_request.get('/')
assert permission.has_permission(request, view) is False
def test_list_permission_protect_authenticated(
factories, api_request, settings):
settings.PROTECT_AUDIO_FILES = True
user = factories['users.User']()
view = APIView.as_view()
permission = permissions.Listen()
request = api_request.get('/')
setattr(request, 'user', user)
assert permission.has_permission(request, view) is True
def test_list_permission_protect_not_following_actor(
factories, api_request, settings):
settings.PROTECT_AUDIO_FILES = True
actor = factories['federation.Actor']()
view = APIView.as_view()
permission = permissions.Listen()
request = api_request.get('/')
setattr(request, 'actor', actor)
assert permission.has_permission(request, view) is False
def test_list_permission_protect_following_actor(
factories, api_request, settings):
settings.PROTECT_AUDIO_FILES = True
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
follow = factories['federation.Follow'](target=library_actor)
view = APIView.as_view()
permission = permissions.Listen()
request = api_request.get('/')
setattr(request, 'actor', follow.actor)
assert permission.has_permission(request, view) is True

40
api/tests/music/test_views.py
View File

@ -1,6 +1,8 @@
import io
import pytest
from funkwhale_api.music import views
from funkwhale_api.federation import actors
@pytest.mark.parametrize('param,expected', [
@ -43,3 +45,41 @@ def test_album_view_filter_listenable(
queryset = view.filter_queryset(view.get_queryset())
assert list(queryset) == expected
def test_can_serve_track_file_as_remote_library(
factories, authenticated_actor, settings, api_client):
settings.PROTECT_AUDIO_FILES = True
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
follow = factories['federation.Follow'](
actor=authenticated_actor, target=library_actor)
track_file = factories['music.TrackFile']()
response = api_client.get(track_file.path)
assert response.status_code == 200
assert response['X-Accel-Redirect'] == "{}{}".format(
settings.PROTECT_FILES_PATH,
track_file.audio_file.url)
def test_can_serve_track_file_as_remote_library_deny_not_following(
factories, authenticated_actor, settings, api_client):
settings.PROTECT_AUDIO_FILES = True
track_file = factories['music.TrackFile']()
response = api_client.get(track_file.path)
assert response.status_code == 403
def test_can_proxy_remote_track(
factories, settings, api_client, r_mock):
settings.PROTECT_AUDIO_FILES = False
track_file = factories['music.TrackFile'](federation=True)
r_mock.get(track_file.library_track.audio_url, body=io.StringIO('test'))
response = api_client.get(track_file.path)
assert response.status_code == 200
assert list(response.streaming_content) == [b't', b'e', b's', b't']
assert response['Content-Type'] == track_file.library_track.audio_mimetype

9
deploy/env.prod.sample
View File

@ -85,3 +85,12 @@ API_AUTHENTICATION_REQUIRED=True
# This will help us detect and correct bugs
RAVEN_ENABLED=false
RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5
# This settings enable/disable federation on the instance level
FEDERATION_ENABLED=True
# This setting decide wether music library is shared automatically
# to followers or if it requires manual approval before.
# FEDERATION_MUSIC_NEEDS_APPROVAL=False
# means anyone can subscribe to your library and import your file,
# use with caution.
FEDERATION_MUSIC_NEEDS_APPROVAL=True