MirrorMirror
/
funkwhale
mirror of https://dev.funkwhale.audio/funkwhale/funkwhale.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Can now serve track from remote library

This commit is contained in:
Eliot Berriot 2018-04-07 15:34:35 +02:00
parent b29ca44797
commit 9612b1bace
No known key found for this signature in database
GPG Key ID: DD6965E2476E5C27
22 changed files with 272 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
api/funkwhale_api/federation/activity.py
View File

@ -58,21 +58,12 @@ OBJECT_TYPES = [
'Video',
] + ACTIVITY_TYPES
def deliver(activity, on_behalf_of, to=[]):
from . import actors
logger.info('Preparing activity delivery to %s', to)
auth = requests_http_signature.HTTPSignatureAuth(
use_auth_header=False,
headers=[
'(request-target)',
'user-agent',
'host',
'date',
'content-type',],
algorithm='rsa-sha256',
key=on_behalf_of.private_key.encode('utf-8'),
key_id=on_behalf_of.private_key_id,
)
auth = signing.get_auth(
on_behalf_of.private_key, on_behalf_of.private_key_id)
for url in to:
recipient_actor = actors.get_actor(url)
logger.debug('delivering to %s', recipient_actor.inbox_url)

32
api/funkwhale_api/federation/actors.py
View File

@ -13,8 +13,10 @@ from rest_framework.exceptions import PermissionDenied
from dynamic_preferences.registries import global_preferences_registry
from . import activity
from . import keys
from . import models
from . import serializers
from . import signing
from . import utils
logger = logging.getLogger(__name__)
@ -51,24 +53,37 @@ class SystemActor(object):
additional_attributes = {}
manually_approves_followers = False
def get_request_auth(self):
actor = self.get_actor_instance()
return signing.get_auth(
actor.private_key, actor.private_key_id)
def serialize(self):
actor = self.get_actor_instance()
serializer = serializers.ActorSerializer(actor)
return serializer.data
def get_actor_instance(self):
try:
return models.Actor.objects.get(url=self.get_actor_url())
except models.Actor.DoesNotExist:
pass
private, public = keys.get_key_pair()
args = self.get_instance_argument(
self.id,
name=self.name,
summary=self.summary,
**self.additional_attributes
)
url = args.pop('url')
a, created = models.Actor.objects.get_or_create(
url=url,
defaults=args,
)
return a
args['private_key'] = private.decode('utf-8')
args['public_key'] = public.decode('utf-8')
return models.Actor.objects.create(**args)
def get_actor_url(self):
return utils.full_url(
reverse(
'federation:instance-actors-detail',
kwargs={'actor': self.id}))
def get_instance_argument(self, id, name, summary, **kwargs):
preferences = global_preferences_registry.manager()
@ -78,10 +93,7 @@ class SystemActor(object):
'type': 'Person',
'name': name.format(host=settings.FEDERATION_HOSTNAME),
'manually_approves_followers': True,
'url': utils.full_url(
reverse(
'federation:instance-actors-detail',
kwargs={'actor': id})),
'url': self.get_actor_url(),
'shared_inbox_url': utils.full_url(
reverse(
'federation:instance-actors-inbox',

2
api/funkwhale_api/federation/authentication.py
View File

@ -51,6 +51,8 @@ class SignatureAuthentication(authentication.BaseAuthentication):
def authenticate(self, request):
setattr(request, 'actor', None)
actor = self.authenticate_actor(request)
if not actor:
return
user = AnonymousUser()
setattr(request, 'actor', actor)
return (user, None)

0
api/funkwhale_api/federation/management/__init__.py
View File

0
api/funkwhale_api/federation/management/commands/__init__.py
View File

53
api/funkwhale_api/federation/management/commands/generate_keys.py
View File

@ -1,53 +0,0 @@
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from dynamic_preferences.registries import global_preferences_registry
from funkwhale_api.federation import keys
class Command(BaseCommand):
help = (
'Generate a public/private key pair for your instance,'
' for federation purposes. If a key pair already exists, does nothing.'
)
def add_arguments(self, parser):
parser.add_argument(
'--replace',
action='store_true',
dest='replace',
default=False,
help='Replace existing key pair, if any',
)
parser.add_argument(
'--noinput', '--no-input', action='store_false', dest='interactive',
help="Do NOT prompt the user for input of any kind.",
)
@transaction.atomic
def handle(self, *args, **options):
preferences = global_preferences_registry.manager()
existing_public = preferences['federation__public_key']
existing_private = preferences['federation__public_key']
if existing_public or existing_private and not options['replace']:
raise CommandError(
'Keys are already present! '
'Replace them with --replace if you know what you are doing.')
if options['interactive']:
message = (
'Are you sure you want to do this?\n\n'
"Type 'yes' to continue, or 'no' to cancel: "
)
if input(''.join(message)) != 'yes':
raise CommandError("Operation cancelled.")
private, public = keys.get_key_pair()
preferences['federation__public_key'] = public.decode('utf-8')
preferences['federation__private_key'] = private.decode('utf-8')
self.stdout.write(
'Your new key pair was generated.'
'Your public key is now:\n\n{}'.format(public.decode('utf-8'))
)

4
api/funkwhale_api/federation/migrations/0003_auto_20180407_0852.py → api/funkwhale_api/federation/migrations/0003_auto_20180407_1010.py
View File

@ -1,4 +1,4 @@
# Generated by Django 2.0.3 on 2018-04-07 08:52
# Generated by Django 2.0.3 on 2018-04-07 10:10
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
@ -10,7 +10,6 @@ import uuid
class Migration(migrations.Migration):
dependencies = [
('music', '0022_importbatch_import_request'),
('federation', '0002_auto_20180403_1620'),
]
@ -70,7 +69,6 @@ class Migration(migrations.Migration):
('title', models.CharField(max_length=500)),
('metadata', django.contrib.postgres.fields.jsonb.JSONField(default={}, max_length=10000)),
('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tracks', to='federation.Library')),
('local_track_file', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='library_track', to='music.TrackFile')),
],
),
migrations.AddField(

7
api/funkwhale_api/federation/models.py
View File

@ -192,13 +192,6 @@ class LibraryTrack(models.Model):
published_date = models.DateTimeField(null=True, blank=True)
library = models.ForeignKey(
Library, related_name='tracks', on_delete=models.CASCADE)
local_track_file = models.OneToOneField(
'music.TrackFile',
related_name='library_track',
on_delete=models.CASCADE,
null=True,
blank=True,
)
artist_name = models.CharField(max_length=500)
album_title = models.CharField(max_length=500)
title = models.CharField(max_length=500)

15
api/funkwhale_api/federation/signing.py
View File

@ -53,3 +53,18 @@ def verify_django(django_request, public_key):
request.headers[h] = str(v)
prepared_request = request.prepare()
return verify(request, public_key)
def get_auth(private_key, private_key_id):
return requests_http_signature.HTTPSignatureAuth(
use_auth_header=False,
headers=[
'(request-target)',
'user-agent',
'host',
'date',
'content-type'],
algorithm='rsa-sha256',
key=private_key.encode('utf-8'),
key_id=private_key_id,
)

12
api/funkwhale_api/music/factories.py
View File

@ -56,6 +56,18 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
class Meta:
model = 'music.TrackFile'
class Params:
federation = factory.Trait(
audio_file=None,
library_track=factory.SubFactory(LibraryTrackFactory),
mimetype=factory.LazyAttribute(
lambda o: o.library_track.audio_mimetype
),
source=factory.LazyAttribute(
lambda o: o.library_track.audio_url
),
)
@registry.register
class ImportBatchFactory(factory.django.DjangoModelFactory):

9
api/funkwhale_api/music/migrations/0023_auto_20180407_0852.py → api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py
View File

@ -1,4 +1,4 @@
# Generated by Django 2.0.3 on 2018-04-07 08:52
# Generated by Django 2.0.3 on 2018-04-07 10:10
from django.conf import settings
from django.db import migrations, models
@ -10,7 +10,7 @@ import uuid
class Migration(migrations.Migration):
dependencies = [
('federation', '0003_auto_20180407_0852'),
('federation', '0003_auto_20180407_1010'),
('music', '0022_importbatch_import_request'),
]
@ -55,6 +55,11 @@ class Migration(migrations.Migration):
name='creation_date',
field=models.DateTimeField(default=django.utils.timezone.now),
),
migrations.AddField(
model_name='trackfile',
name='library_track',
field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='local_track_file', to='federation.LibraryTrack'),
),
migrations.AddField(
model_name='trackfile',
name='modification_date',

14
api/funkwhale_api/music/models.py
View File

@ -419,6 +419,14 @@ class TrackFile(models.Model):
acoustid_track_id = models.UUIDField(null=True, blank=True)
mimetype = models.CharField(null=True, blank=True, max_length=200)
library_track = models.OneToOneField(
'federation.LibraryTrack',
related_name='local_track_file',
on_delete=models.CASCADE,
null=True,
blank=True,
)
def download_file(self):
# import the track file, since there is not any
# we create a tmp dir for the download
@ -441,10 +449,8 @@ class TrackFile(models.Model):
@property
def path(self):
if settings.PROTECT_AUDIO_FILES:
return reverse(
'api:v1:trackfiles-serve', kwargs={'pk': self.pk})
return self.audio_file.url
return reverse(
'api:v1:trackfiles-serve', kwargs={'pk': self.pk})
@property
def filename(self):

23
api/funkwhale_api/music/permissions.py Normal file
View File

@ -0,0 +1,23 @@
from django.conf import settings
from rest_framework.permissions import BasePermission
from funkwhale_api.federation import actors
class Listen(BasePermission):
def has_permission(self, request, view):
if not settings.PROTECT_AUDIO_FILES:
return True
user = getattr(request, 'user', None)
if user and user.is_authenticated:
return True
actor = getattr(request, 'actor', None)
if actor is None:
return False
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
return library.followers.filter(url=actor.url).exists()

5
api/funkwhale_api/music/tasks.py
View File

@ -112,6 +112,7 @@ def _do_import(import_job, replace, use_acoustid=True):
track_file.audio_file.name = import_job.audio_file.name
track_file.duration = duration
elif import_job.library_track:
track_file.library_track = import_job.library_track
track_file.mimetype = import_job.library_track.audio_mimetype
if import_job.library_track.library.download_files:
raise NotImplementedError()
@ -121,10 +122,6 @@ def _do_import(import_job, replace, use_acoustid=True):
else:
track_file.download_file()
track_file.save()
if import_job.library_track:
import_job.library_track.local_track_file = track_file
import_job.library_track.save(
update_fields=['modification_date', 'local_track_file'])
import_job.status = 'finished'
import_job.track_file = track_file
if import_job.audio_file:

7
api/funkwhale_api/music/utils.py
View File

@ -60,3 +60,10 @@ def compute_status(jobs):
if pending:
return 'pending'
return 'finished'
def get_ext_from_type(mimetype):
mapping = {
'audio/ogg': 'ogg',
'audio/mpeg': 'mp3',
}

69
api/funkwhale_api/music/views.py
View File

@ -1,36 +1,42 @@
import ffmpeg
import os
import json
import requests
import subprocess
import unicodedata
import urllib
from django.urls import reverse
from django.contrib.auth.decorators import login_required
from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings
from django.db import models, transaction
from django.db.models.functions import Length
from django.conf import settings
from django.http import StreamingHttpResponse
from django.urls import reverse
from django.utils.decorators import method_decorator
from rest_framework import viewsets, views, mixins
from rest_framework.decorators import detail_route, list_route
from rest_framework.response import Response
from rest_framework import settings as rest_settings
from rest_framework import permissions
from musicbrainzngs import ResponseError
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from funkwhale_api.common import utils as funkwhale_utils
from funkwhale_api.federation import actors
from funkwhale_api.requests.models import ImportRequest
from funkwhale_api.musicbrainz import api
from funkwhale_api.common.permissions import (
ConditionalAuthentication, HasModelPermission)
from taggit.models import Tag
from funkwhale_api.federation.authentication import SignatureAuthentication
from . import forms
from . import models
from . import serializers
from . import importers
from . import filters
from . import forms
from . import importers
from . import models
from . import permissions as music_permissions
from . import serializers
from . import tasks
from . import utils
@ -45,6 +51,7 @@ class SearchMixin(object):
serializer = self.serializer_class(queryset, many=True)
return Response(serializer.data)
class TagViewSetMixin(object):
def get_queryset(self):
@ -179,22 +186,54 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
queryset = (models.TrackFile.objects.all().order_by('-id'))
serializer_class = serializers.TrackFileSerializer
permission_classes = [ConditionalAuthentication]
authentication_classes = rest_settings.api_settings.DEFAULT_AUTHENTICATION_CLASSES + [
SignatureAuthentication
]
permission_classes = [music_permissions.Listen]
@detail_route(methods=['get'])
def serve(self, request, *args, **kwargs):
try:
f = models.TrackFile.objects.get(pk=kwargs['pk'])
f = models.TrackFile.objects.select_related(
'library_track',
'track__album__artist',
'track__artist',
).get(pk=kwargs['pk'])
except models.TrackFile.DoesNotExist:
return Response(status=404)
response = Response()
mt = f.mimetype
try:
library_track = f.library_track
except ObjectDoesNotExist:
library_track = None
if library_track and not f.audio_file:
# we proxy the response to the remote library
# since we did not mirror the file locally
mt = library_track.audio_mimetype
file_extension = utils.get_ext_from_type(mt)
filename = '{}.{}'.format(f.track.full_name, file_extension)
auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
remote_response = requests.get(
library_track.audio_url,
auth=auth,
stream=True,
headers={
'Content-Type': 'application/activity+json'
})
response = StreamingHttpResponse(remote_response.iter_content())
else:
response = Response()
filename = f.filename
response['X-Accel-Redirect'] = "{}{}".format(
settings.PROTECT_FILES_PATH,
f.audio_file.url)
filename = "filename*=UTF-8''{}".format(
urllib.parse.quote(f.filename))
urllib.parse.quote(filename))
response["Content-Disposition"] = "attachment; {}".format(filename)
response['X-Accel-Redirect'] = "{}{}".format(
settings.PROTECT_FILES_PATH,
f.audio_file.url)
if mt:
response["Content-Type"] = mt
return response
@list_route(methods=['get'])

9
api/tests/conftest.py
View File

@ -162,3 +162,12 @@ def media_root(settings):
def r_mock():
with requests_mock.mock() as m:
yield m
@pytest.fixture
def authenticated_actor(factories, mocker):
actor = factories['federation.Actor']()
mocker.patch(
'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor',
return_value=actor)
yield actor

10
api/tests/federation/conftest.py
View File

@ -1,10 +0,0 @@
import pytest
@pytest.fixture
def authenticated_actor(nodb_factories, mocker):
actor = nodb_factories['federation.Actor']()
mocker.patch(
'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor',
return_value=actor)
yield actor

16
api/tests/federation/test_actors.py
View File

@ -26,14 +26,17 @@ def test_actor_fetching(r_mock):
assert r == payload
def test_get_library(settings, preferences):
preferences['federation__public_key'] = 'public_key'
def test_get_library(db, settings, mocker):
get_key_pair = mocker.patch(
'funkwhale_api.federation.keys.get_key_pair',
return_value=(b'private', b'public'))
expected = {
'preferred_username': 'library',
'domain': settings.FEDERATION_HOSTNAME,
'type': 'Person',
'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME),
'manually_approves_followers': True,
'public_key': 'public',
'url': utils.full_url(
reverse(
'federation:instance-actors-detail',
@ -50,7 +53,6 @@ def test_get_library(settings, preferences):
reverse(
'federation:instance-actors-outbox',
kwargs={'actor': 'library'})),
'public_key': 'public_key',
'summary': 'Bot account to federate with {}\'s library'.format(
settings.FEDERATION_HOSTNAME),
}
@ -59,14 +61,17 @@ def test_get_library(settings, preferences):
assert getattr(actor, key) == value
def test_get_test(settings, preferences):
preferences['federation__public_key'] = 'public_key'
def test_get_test(db, mocker, settings):
get_key_pair = mocker.patch(
'funkwhale_api.federation.keys.get_key_pair',
return_value=(b'private', b'public'))
expected = {
'preferred_username': 'test',
'domain': settings.FEDERATION_HOSTNAME,
'type': 'Person',
'name': '{}\'s test account'.format(settings.FEDERATION_HOSTNAME),
'manually_approves_followers': False,
'public_key': 'public',
'url': utils.full_url(
reverse(
'federation:instance-actors-detail',
@ -83,7 +88,6 @@ def test_get_test(settings, preferences):
reverse(
'federation:instance-actors-outbox',
kwargs={'actor': 'test'})),
'public_key': 'public_key',
'summary': 'Bot account to test federation with {}. Send me /ping and I\'ll answer you.'.format(
settings.FEDERATION_HOSTNAME),
}

14
api/tests/federation/test_commands.py
View File

@ -1,14 +0,0 @@
from django.core.management import call_command
def test_generate_instance_key_pair(preferences, mocker):
mocker.patch(
'funkwhale_api.federation.keys.get_key_pair',
return_value=(b'private', b'public'))
assert preferences['federation__public_key'] == ''
assert preferences['federation__private_key'] == ''
call_command('generate_keys', interactive=False)
assert preferences['federation__private_key'] == 'private'
assert preferences['federation__public_key'] == 'public'

56
api/tests/music/test_permissions.py Normal file
View File

@ -0,0 +1,56 @@
from rest_framework.views import APIView
from funkwhale_api.federation import actors
from funkwhale_api.music import permissions
def test_list_permission_no_protect(anonymous_user, api_request, settings):
settings.PROTECT_AUDIO_FILES = False
view = APIView.as_view()
permission = permissions.Listen()
request = api_request.get('/')
assert permission.has_permission(request, view) is True
def test_list_permission_protect_anonymous(
anonymous_user, api_request, settings):
settings.PROTECT_AUDIO_FILES = True
view = APIView.as_view()
permission = permissions.Listen()
request = api_request.get('/')
assert permission.has_permission(request, view) is False
def test_list_permission_protect_authenticated(
factories, api_request, settings):
settings.PROTECT_AUDIO_FILES = True
user = factories['users.User']()
view = APIView.as_view()
permission = permissions.Listen()
request = api_request.get('/')
setattr(request, 'user', user)
assert permission.has_permission(request, view) is True
def test_list_permission_protect_not_following_actor(
factories, api_request, settings):
settings.PROTECT_AUDIO_FILES = True
actor = factories['federation.Actor']()
view = APIView.as_view()
permission = permissions.Listen()
request = api_request.get('/')
setattr(request, 'actor', actor)
assert permission.has_permission(request, view) is False
def test_list_permission_protect_following_actor(
factories, api_request, settings):
settings.PROTECT_AUDIO_FILES = True
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
follow = factories['federation.Follow'](target=library_actor)
view = APIView.as_view()
permission = permissions.Listen()
request = api_request.get('/')
setattr(request, 'actor', follow.actor)
assert permission.has_permission(request, view) is True

40
api/tests/music/test_views.py
View File

@ -1,6 +1,8 @@
import io
import pytest
from funkwhale_api.music import views
from funkwhale_api.federation import actors
@pytest.mark.parametrize('param,expected', [
@ -43,3 +45,41 @@ def test_album_view_filter_listenable(
queryset = view.filter_queryset(view.get_queryset())
assert list(queryset) == expected
def test_can_serve_track_file_as_remote_library(
factories, authenticated_actor, settings, api_client):
settings.PROTECT_AUDIO_FILES = True
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
follow = factories['federation.Follow'](
actor=authenticated_actor, target=library_actor)
track_file = factories['music.TrackFile']()
response = api_client.get(track_file.path)
assert response.status_code == 200
assert response['X-Accel-Redirect'] == "{}{}".format(
settings.PROTECT_FILES_PATH,
track_file.audio_file.url)
def test_can_serve_track_file_as_remote_library_deny_not_following(
factories, authenticated_actor, settings, api_client):
settings.PROTECT_AUDIO_FILES = True
track_file = factories['music.TrackFile']()
response = api_client.get(track_file.path)
assert response.status_code == 403
def test_can_proxy_remote_track(
factories, settings, api_client, r_mock):
settings.PROTECT_AUDIO_FILES = False
track_file = factories['music.TrackFile'](federation=True)
r_mock.get(track_file.library_track.audio_url, body=io.StringIO('test'))
response = api_client.get(track_file.path)
assert response.status_code == 200
assert list(response.streaming_content) == [b't', b'e', b's', b't']
assert response['Content-Type'] == track_file.library_track.audio_mimetype