From e19fbf543605542628a60cb0ce74e720b19a84be Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 7 Apr 2018 16:26:07 +0200 Subject: [PATCH 01/42] Exclude federated files from library endpoint --- api/funkwhale_api/federation/views.py | 2 +- api/tests/federation/test_views.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 35d8a75a5..da2b193a2 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -121,7 +121,7 @@ class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet): qs = TrackFile.objects.order_by('-creation_date').select_related( 'track__artist', 'track__album__artist' - ) + ).filter(library_track__isnull=True) if page is None: conf = { 'id': utils.full_url(reverse('federation:music:files-list')), diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index c26810dad..c5d651dce 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -116,6 +116,18 @@ def test_audio_file_list_actor_page( assert response.status_code == 200 assert response.data == expected +def test_audio_file_list_actor_page_exclude_federated_files( + db, settings, api_client, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + tfs = factories['music.TrackFile'].create_batch(size=5, federation=True) + + url = reverse('federation:music:files-list') + response = api_client.get(url) + + assert response.status_code == 200 + assert response.data['totalItems'] == 0 + def test_audio_file_list_actor_page_error( db, settings, api_client, factories): From 6bf4d463624015b7e745063bd509c74d30cee350 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 7 Apr 2018 16:28:52 +0200 Subject: [PATCH 02/42] Ensure we don't duplicate libray tracks --- api/tests/federation/test_serializers.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index 45778ed48..71407dc43 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -262,6 +262,25 @@ def test_activity_pub_audio_serializer_to_library_track(factories): assert lt.published_date == arrow.get(audio['published']) +def test_activity_pub_audio_serializer_to_library_track_no_duplicate( + factories): + remote_library = factories['federation.Library']() + audio = factories['federation.Audio']() + serializer1 = serializers.AudioSerializer( + data=audio, context={'library': remote_library}) + serializer2 = serializers.AudioSerializer( + data=audio, context={'library': remote_library}) + + assert serializer1.is_valid() is True + assert serializer2.is_valid() is True + + lt1 = serializer1.save() + lt2 = serializer2.save() + + assert lt1 == lt2 + assert models.LibraryTrack.objects.count() == 1 + + def test_activity_pub_audio_serializer_to_ap(factories): tf = factories['music.TrackFile'](mimetype='audio/mp3') library = actors.SYSTEM_ACTORS['library'].get_actor_instance() From 33972f1f40d75f676cc0fc756bb1bce90fec0a76 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 7 Apr 2018 17:09:28 +0200 Subject: [PATCH 03/42] Fixed broken uuid migration --- .../migrations/0023_auto_20180407_1010.py | 17 ++-- .../music/migrations/0024_populate_uuid.py | 80 +++++++++++++++++++ 2 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 api/funkwhale_api/music/migrations/0024_populate_uuid.py diff --git a/api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py b/api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py index 0539d90f6..ed7404ac4 100644 --- a/api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py +++ b/api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py @@ -4,7 +4,6 @@ from django.conf import settings from django.db import migrations, models import django.db.models.deletion import django.utils.timezone -import uuid class Migration(migrations.Migration): @@ -18,17 +17,17 @@ class Migration(migrations.Migration): migrations.AddField( model_name='album', name='uuid', - field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + field=models.UUIDField(db_index=True, null=True, unique=True), ), migrations.AddField( model_name='artist', name='uuid', - field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + field=models.UUIDField(db_index=True, null=True, unique=True), ), migrations.AddField( model_name='importbatch', name='uuid', - field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + field=models.UUIDField(db_index=True, null=True, unique=True), ), migrations.AddField( model_name='importjob', @@ -38,17 +37,17 @@ class Migration(migrations.Migration): migrations.AddField( model_name='importjob', name='uuid', - field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + field=models.UUIDField(db_index=True, null=True, unique=True), ), migrations.AddField( model_name='lyrics', name='uuid', - field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + field=models.UUIDField(db_index=True, null=True, unique=True), ), migrations.AddField( model_name='track', name='uuid', - field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + field=models.UUIDField(db_index=True, null=True, unique=True), ), migrations.AddField( model_name='trackfile', @@ -68,12 +67,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='trackfile', name='uuid', - field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + field=models.UUIDField(db_index=True, null=True, unique=True), ), migrations.AddField( model_name='work', name='uuid', - field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + field=models.UUIDField(db_index=True, null=True, unique=True), ), migrations.AlterField( model_name='importbatch', diff --git a/api/funkwhale_api/music/migrations/0024_populate_uuid.py b/api/funkwhale_api/music/migrations/0024_populate_uuid.py new file mode 100644 index 000000000..10c78a3db --- /dev/null +++ b/api/funkwhale_api/music/migrations/0024_populate_uuid.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +import os +import uuid +from django.db import migrations, models + + +def populate_uuids(apps, schema_editor): + models = [ + 'Album', + 'Artist', + 'Importbatch', + 'Importjob', + 'Lyrics', + 'Track', + 'Trackfile', + 'Work', + ] + for m in models: + kls = apps.get_model('music', m) + qs = kls.objects.filter(uuid__isnull=True).only('id') + print('Setting uuids for {} ({} objects)'.format(m, len(qs))) + for o in qs: + o.uuid = uuid.uuid4() + o.save(update_fields=['uuid']) + + +def rewind(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('music', '0023_auto_20180407_1010'), + ] + + operations = [ + migrations.RunPython(populate_uuids, rewind), + migrations.AlterField( + model_name='album', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='artist', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='importbatch', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='importjob', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='lyrics', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='track', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='trackfile', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='work', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + ] From 4320fc77b2229d87956a95d44ae4bb83085d6f96 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 7 Apr 2018 17:18:54 +0200 Subject: [PATCH 04/42] Added validation on collection serializers --- api/funkwhale_api/federation/serializers.py | 14 ++++++ api/tests/federation/test_serializers.py | 51 ++++++++++++++++++++- api/tests/federation/test_views.py | 1 + 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 17541c50f..d8e5ddb2d 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -21,6 +21,7 @@ AP_CONTEXT = [ {}, ] + class ActorSerializer(serializers.ModelSerializer): # left maps to activitypub fields, right to our internal models id = serializers.URLField(source='url') @@ -206,6 +207,11 @@ OBJECT_SERIALIZERS = { class PaginatedCollectionSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=['Collection']) + totalItems = serializers.IntegerField(min_value=0) + items = serializers.ListField() + actor = serializers.URLField() + id = serializers.URLField() def to_representation(self, conf): paginator = Paginator( @@ -230,6 +236,14 @@ class PaginatedCollectionSerializer(serializers.Serializer): class CollectionPageSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=['CollectionPage']) + totalItems = serializers.IntegerField(min_value=0) + items = serializers.ListField() + actor = serializers.URLField() + id = serializers.URLField() + prev = serializers.URLField(required=False) + next = serializers.URLField(required=False) + partOf = serializers.URLField() def to_representation(self, conf): page = conf['page'] diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index 71407dc43..8c76bb443 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -34,7 +34,7 @@ def test_actor_serializer_from_ap(db): } serializer = serializers.ActorSerializer(data=payload) - assert serializer.is_valid() + assert serializer.is_valid(raise_exception=True) actor = serializer.build() @@ -65,7 +65,7 @@ def test_actor_serializer_only_mandatory_field_from_ap(db): } serializer = serializers.ActorSerializer(data=payload) - assert serializer.is_valid() + assert serializer.is_valid(raise_exception=True) actor = serializer.build() @@ -201,6 +201,53 @@ def test_paginated_collection_serializer(factories): assert serializer.data == expected +def test_paginated_collection_serializer_validation(): + data = { + 'type': 'Collection', + 'id': 'https://test.federation/test', + 'totalItems': 5, + 'actor': 'http://test.actor', + 'items': [] + } + + serializer = serializers.PaginatedCollectionSerializer( + data=data + ) + + assert serializer.is_valid(raise_exception=True) is True + assert serializer.validated_data['totalItems'] == 5 + assert serializer.validated_data['id'] == data['id'] + assert serializer.validated_data['actor'] == data['actor'] + assert serializer.validated_data['items'] == [] + + +def test_collection_page_serializer_validdation(): + base = 'https://test.federation/test' + data = { + 'type': 'CollectionPage', + 'id': base + '?page=2', + 'totalItems': 5, + 'actor': 'https://test.actor', + 'items': [], + 'prev': base + '?page=1', + 'next': base + '?page=3', + 'partOf': base, + } + + serializer = serializers.CollectionPageSerializer( + data=data + ) + + assert serializer.is_valid(raise_exception=True) is True + assert serializer.validated_data['totalItems'] == 5 + assert serializer.validated_data['id'] == data['id'] + assert serializer.validated_data['actor'] == data['actor'] + assert serializer.validated_data['items'] == [] + assert serializer.validated_data['prev'] == data['prev'] + assert serializer.validated_data['next'] == data['next'] + assert serializer.validated_data['partOf'] == data['partOf'] + + def test_collection_page_serializer(factories): tfs = factories['music.TrackFile'].create_batch(size=5) actor = factories['federation.Actor'](local=True) diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index c5d651dce..b3fd85910 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -116,6 +116,7 @@ def test_audio_file_list_actor_page( assert response.status_code == 200 assert response.data == expected + def test_audio_file_list_actor_page_exclude_federated_files( db, settings, api_client, factories): settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False From 514e48d3fe28b27ff83f5f5e392fef63fa08c124 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 7 Apr 2018 17:24:33 +0200 Subject: [PATCH 05/42] Library can now automatically accept follows --- api/funkwhale_api/federation/actors.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index a461eb76a..2f6c04de0 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -203,6 +203,20 @@ class LibraryActor(SystemActor): def manually_approves_followers(self): return settings.FEDERATION_MUSIC_NEEDS_APPROVAL + def handle_follow(self, ac, sender): + system_actor = self.get_actor_instance() + if self.manually_approves_followers: + fr, created = models.FollowRequest.objects.get_or_create( + actor=sender, + target=system_actor, + approved=None, + ) + return fr + + return activity.accept_follow( + system_actor, ac, sender + ) + @transaction.atomic def handle_create(self, ac, sender): try: From 314587e2eb6f9437ec8e9ad971c25dd54c9ce171 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 7 Apr 2018 18:37:40 +0200 Subject: [PATCH 06/42] Fixed pagination issue --- api/funkwhale_api/federation/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index d8e5ddb2d..a3ba80749 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -273,7 +273,7 @@ class CollectionPageSerializer(serializers.Serializer): d['prev'] = set_query_parameter( conf['id'], page=page.previous_page_number()) - if page.has_previous(): + if page.has_next(): d['next'] = set_query_parameter( conf['id'], page=page.next_page_number()) From b8c7e960c3a073bc9b25adfde5f9dca27466e815 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 8 Apr 2018 10:42:10 +0200 Subject: [PATCH 07/42] Now validate incoming webfinger --- api/funkwhale_api/federation/serializers.py | 25 ++++++++++++++++---- api/funkwhale_api/federation/webfinger.py | 19 +++++++++++++-- api/tests/federation/test_webfinger.py | 26 +++++++++++++++++++++ 3 files changed, 64 insertions(+), 6 deletions(-) diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index a3ba80749..7e84e575a 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -116,10 +116,27 @@ class FollowSerializer(serializers.ModelSerializer): return ret -class ActorWebfingerSerializer(serializers.ModelSerializer): - class Meta: - model = models.Actor - fields = ['url'] +class ActorWebfingerSerializer(serializers.Serializer): + subject = serializers.CharField() + aliases = serializers.ListField(child=serializers.URLField()) + links = serializers.ListField() + actor_url = serializers.URLField(required=False) + + def validate(self, validated_data): + validated_data['actor_url'] = None + for l in validated_data['links']: + try: + if not l['rel'] == 'self': + continue + if not l['type'] == 'application/activity+json': + continue + validated_data['actor_url'] = l['href'] + break + except KeyError: + pass + if validated_data['actor_url'] is None: + raise serializers.ValidationError('No valid actor url found') + return validated_data def to_representation(self, instance): data = {} diff --git a/api/funkwhale_api/federation/webfinger.py b/api/funkwhale_api/federation/webfinger.py index 4e9753385..011dcf576 100644 --- a/api/funkwhale_api/federation/webfinger.py +++ b/api/funkwhale_api/federation/webfinger.py @@ -2,8 +2,11 @@ from django import forms from django.conf import settings from django.urls import reverse +from funkwhale_api.common import session + from . import actors from . import utils +from . import serializers VALID_RESOURCE_TYPES = ['acct'] @@ -23,13 +26,13 @@ def clean_resource(resource_string): return resource_type, resource -def clean_acct(acct_string): +def clean_acct(acct_string, ensure_local=True): try: username, hostname = acct_string.split('@') except ValueError: raise forms.ValidationError('Invalid format') - if hostname.lower() != settings.FEDERATION_HOSTNAME: + if ensure_local and hostname.lower() != settings.FEDERATION_HOSTNAME: raise forms.ValidationError( 'Invalid hostname {}'.format(hostname)) @@ -37,3 +40,15 @@ def clean_acct(acct_string): raise forms.ValidationError('Invalid username') return username, hostname + + +def get_resource(resource_string): + resource_type, resource = clean_resource(resource_string) + username, hostname = clean_acct(resource, ensure_local=False) + url = 'https://{}/.well-known/webfinger?resource={}'.format( + hostname, resource_string) + response = session.get_session().get(url) + response.raise_for_status() + serializer = serializers.ActorWebfingerSerializer(data=response.json()) + serializer.is_valid(raise_exception=True) + return serializer.validated_data diff --git a/api/tests/federation/test_webfinger.py b/api/tests/federation/test_webfinger.py index 96258455a..4b8dca207 100644 --- a/api/tests/federation/test_webfinger.py +++ b/api/tests/federation/test_webfinger.py @@ -40,3 +40,29 @@ def test_webfinger_clean_acct_errors(resource, message, settings): webfinger.clean_resource(resource) assert message == str(excinfo) + + +def test_webfinger_get_resource(r_mock): + resource = 'acct:test@test.webfinger' + payload = { + 'subject': resource, + 'aliases': ['https://test.webfinger'], + 'links': [ + { + 'rel': 'self', + 'type': 'application/activity+json', + 'href': 'https://test.webfinger/user/test' + } + ] + } + r_mock.get( + 'https://test.webfinger/.well-known/webfinger?resource={}'.format( + resource + ), + json=payload + ) + + data = webfinger.get_resource('acct:test@test.webfinger') + + assert data['actor_url'] == 'https://test.webfinger/user/test' + assert data['subject'] == resource From 206ae296b6d94203bbf1f475ebcc1b8997b32e33 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 8 Apr 2018 13:33:36 +0200 Subject: [PATCH 08/42] Ensure timeout in requests --- api/funkwhale_api/federation/activity.py | 1 + api/funkwhale_api/federation/actors.py | 2 ++ api/funkwhale_api/federation/webfinger.py | 2 +- api/funkwhale_api/music/views.py | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index 7d20d7f9c..a674c70e3 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -73,6 +73,7 @@ def deliver(activity, on_behalf_of, to=[]): auth=auth, json=activity, url=recipient_actor.inbox_url, + timeout=5, headers={ 'Content-Type': 'application/activity+json' } diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 2f6c04de0..d3a2093a9 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -31,6 +31,7 @@ def remove_tags(text): def get_actor_data(actor_url): response = session.get_session().get( actor_url, + timeout=5, headers={ 'Accept': 'application/activity+json', } @@ -42,6 +43,7 @@ def get_actor_data(actor_url): raise ValueError( 'Invalid actor payload: {}'.format(response.text)) + def get_actor(actor_url): data = get_actor_data(actor_url) serializer = serializers.ActorSerializer(data=data) diff --git a/api/funkwhale_api/federation/webfinger.py b/api/funkwhale_api/federation/webfinger.py index 011dcf576..444998b94 100644 --- a/api/funkwhale_api/federation/webfinger.py +++ b/api/funkwhale_api/federation/webfinger.py @@ -47,7 +47,7 @@ def get_resource(resource_string): username, hostname = clean_acct(resource, ensure_local=False) url = 'https://{}/.well-known/webfinger?resource={}'.format( hostname, resource_string) - response = session.get_session().get(url) + response = session.get_session().get(url, timeout=5) response.raise_for_status() serializer = serializers.ActorWebfingerSerializer(data=response.json()) serializer.is_valid(raise_exception=True) diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index 0870d9816..6bbc21db7 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -218,6 +218,7 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet): library_track.audio_url, auth=auth, stream=True, + timeout=20, headers={ 'Content-Type': 'application/activity+json' }) From 92fa348eacee7bd664e0915b5ca99fe5e7e52907 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 8 Apr 2018 13:33:46 +0200 Subject: [PATCH 09/42] Federation model admin --- api/funkwhale_api/federation/admin.py | 77 +++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 api/funkwhale_api/federation/admin.py diff --git a/api/funkwhale_api/federation/admin.py b/api/funkwhale_api/federation/admin.py new file mode 100644 index 000000000..ce636299f --- /dev/null +++ b/api/funkwhale_api/federation/admin.py @@ -0,0 +1,77 @@ +from django.contrib import admin + +from . import models + + +@admin.register(models.Actor) +class ActorAdmin(admin.ModelAdmin): + list_display = [ + 'url', + 'domain', + 'preferred_username', + 'type', + 'creation_date', + 'last_fetch_date'] + search_fields = ['url', 'domain', 'preferred_username'] + list_filter = [ + 'type' + ] + + +@admin.register(models.Follow) +class FollowAdmin(admin.ModelAdmin): + list_display = [ + 'actor', + 'target', + 'creation_date' + ] + search_fields = ['actor__url', 'target__url'] + list_select_related = True + + +@admin.register(models.FollowRequest) +class FollowRequestAdmin(admin.ModelAdmin): + list_display = [ + 'actor', + 'target', + 'creation_date', + 'approved' + ] + search_fields = ['actor__url', 'target__url'] + list_filter = [ + 'approved' + ] + list_select_related = True + + +@admin.register(models.Library) +class LibraryAdmin(admin.ModelAdmin): + list_display = [ + 'actor', + 'url', + 'creation_date', + 'fetched_date', + 'tracks_count'] + search_fields = ['actor__url', 'url'] + list_filter = [ + 'federation_enabled', + 'download_files', + 'autoimport', + ] + list_select_related = True + + +@admin.register(models.LibraryTrack) +class LibraryTrackAdmin(admin.ModelAdmin): + list_display = [ + 'title', + 'artist_name', + 'album_title', + 'url', + 'library', + 'creation_date', + 'published_date', + ] + search_fields = [ + 'library__url', 'url', 'artist_name', 'title', 'album_title'] + list_select_related = True From 836e8139558d20e7bbe6c07e0d588f616fba988a Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 8 Apr 2018 13:34:13 +0200 Subject: [PATCH 10/42] Typo in test name --- api/tests/federation/test_serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index 8c76bb443..e4fb88040 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -221,7 +221,7 @@ def test_paginated_collection_serializer_validation(): assert serializer.validated_data['items'] == [] -def test_collection_page_serializer_validdation(): +def test_collection_page_serializer_validation(): base = 'https://test.federation/test' data = { 'type': 'CollectionPage', From 097707dec45a5f023875a52f8f03c7f4ad36c417 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 8 Apr 2018 13:35:37 +0200 Subject: [PATCH 11/42] Added remote library scanning logic end endpoint --- api/config/api_urls.py | 4 + api/funkwhale_api/federation/api_urls.py | 11 +++ api/funkwhale_api/federation/library.py | 97 +++++++++++++++++++++ api/funkwhale_api/federation/serializers.py | 32 ++++++- api/funkwhale_api/federation/views.py | 22 ++++- api/funkwhale_api/federation/webfinger.py | 2 +- api/tests/federation/test_library.py | 66 ++++++++++++++ api/tests/federation/test_views.py | 15 ++++ 8 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 api/funkwhale_api/federation/api_urls.py create mode 100644 api/funkwhale_api/federation/library.py create mode 100644 api/tests/federation/test_library.py diff --git a/api/config/api_urls.py b/api/config/api_urls.py index cab6805b6..cf5b03744 100644 --- a/api/config/api_urls.py +++ b/api/config/api_urls.py @@ -32,6 +32,10 @@ v1_patterns += [ include( ('funkwhale_api.instance.urls', 'instance'), namespace='instance')), + url(r'^federation/', + include( + ('funkwhale_api.federation.api_urls', 'federation'), + namespace='federation')), url(r'^providers/', include( ('funkwhale_api.providers.urls', 'providers'), diff --git a/api/funkwhale_api/federation/api_urls.py b/api/funkwhale_api/federation/api_urls.py new file mode 100644 index 000000000..ecb5c38f1 --- /dev/null +++ b/api/funkwhale_api/federation/api_urls.py @@ -0,0 +1,11 @@ +from rest_framework import routers + +from . import views + +router = routers.SimpleRouter() +router.register( + r'libraries', + views.LibraryViewSet, + 'libraries') + +urlpatterns = router.urls diff --git a/api/funkwhale_api/federation/library.py b/api/funkwhale_api/federation/library.py new file mode 100644 index 000000000..13608098b --- /dev/null +++ b/api/funkwhale_api/federation/library.py @@ -0,0 +1,97 @@ +import requests + +from funkwhale_api.common import session + +from . import actors +from . import serializers +from . import signing +from . import webfinger + + +def scan_from_account_name(account_name): + """ + Given an account name such as library@test.library, will: + + 1. Perform the webfinger lookup + 2. Perform the actor lookup + 3. Perform the library's collection lookup + + and return corresponding data in a dictionary. + """ + + data = {} + try: + data['webfinger'] = webfinger.get_resource( + 'acct:{}'.format(account_name)) + except requests.ConnectionError: + return { + 'webfinger': { + 'errors': ['This webfinger resource is not reachable'] + } + } + except requests.HTTPError as e: + return { + 'webfinger': { + 'errors': [ + 'Error {} during webfinger request'.format( + e.response.status_code)] + } + } + + try: + data['actor'] = actors.get_actor_data(data['webfinger']['actor_url']) + except requests.ConnectionError: + data['actor'] = { + 'errors': ['This actor is not reachable'] + } + return data + except requests.HTTPError as e: + data['actor'] = { + 'errors': [ + 'Error {} during actor request'.format( + e.response.status_code)] + } + return data + + serializer = serializers.LibraryActorSerializer(data=data['actor']) + serializer.is_valid(raise_exception=True) + data['library'] = get_library_data( + serializer.validated_data['library_url']) + + return data + + +def get_library_data(library_url): + actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + auth = signing.get_auth(actor.private_key, actor.private_key_id) + try: + response = session.get_session().get( + library_url, + auth=auth, + timeout=5, + headers={ + 'Content-Type': 'application/activity+json' + } + ) + except requests.ConnectionError: + return { + 'errors': ['This library is not reachable'] + } + scode = response.status_code + if scode == 401: + return { + 'errors': ['This library requires authentication'] + } + elif scode == 403: + return { + 'errors': ['Permission denied while scanning library'] + } + elif scode >= 400: + return { + 'errors': ['Error {} while fetching the library'.format(scode)] + } + serializer = serializers.PaginatedCollectionSerializer( + data=response.json(), + ) + serializer.is_valid(raise_exception=True) + return serializer.validated_data diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 7e84e575a..704ad6364 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -27,8 +27,10 @@ class ActorSerializer(serializers.ModelSerializer): id = serializers.URLField(source='url') outbox = serializers.URLField(source='outbox_url') inbox = serializers.URLField(source='inbox_url') - following = serializers.URLField(source='following_url', required=False) - followers = serializers.URLField(source='followers_url', required=False) + following = serializers.URLField( + source='following_url', required=False, allow_null=True) + followers = serializers.URLField( + source='followers_url', required=False, allow_null=True) preferredUsername = serializers.CharField( source='preferred_username', required=False) publicKey = serializers.JSONField(source='public_key', required=False) @@ -94,6 +96,31 @@ class ActorSerializer(serializers.ModelSerializer): return value[:500] +class LibraryActorSerializer(ActorSerializer): + url = serializers.ListField( + child=serializers.JSONField()) + + class Meta(ActorSerializer.Meta): + fields = ActorSerializer.Meta.fields + ['url'] + + def validate(self, validated_data): + try: + urls = validated_data['url'] + except KeyError: + raise serializers.ValidationError('Missing URL field') + + for u in urls: + try: + if u['name'] != 'library': + continue + validated_data['library_url'] = u['href'] + break + except KeyError: + continue + + return validated_data + + class FollowSerializer(serializers.ModelSerializer): # left maps to activitypub fields, right to our internal models id = serializers.URLField(source='get_federation_url') @@ -226,7 +253,6 @@ OBJECT_SERIALIZERS = { class PaginatedCollectionSerializer(serializers.Serializer): type = serializers.ChoiceField(choices=['Collection']) totalItems = serializers.IntegerField(min_value=0) - items = serializers.ListField() actor = serializers.URLField() id = serializers.URLField() diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index da2b193a2..aaab343e4 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -4,15 +4,18 @@ from django.core import paginator from django.http import HttpResponse from django.urls import reverse -from rest_framework import viewsets -from rest_framework import views +from rest_framework import permissions as rest_permissions from rest_framework import response +from rest_framework import views +from rest_framework import viewsets from rest_framework.decorators import list_route, detail_route from funkwhale_api.music.models import TrackFile from . import actors from . import authentication +from . import library +from . import models from . import permissions from . import renderers from . import serializers @@ -154,3 +157,18 @@ class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet): return response.Response(status=404) return response.Response(data) + + +class LibraryViewSet(viewsets.GenericViewSet): + permission_classes = [rest_permissions.DjangoModelPermissions] + queryset = models.Library.objects.all() + + @list_route(methods=['get']) + def scan(self, request, *args, **kwargs): + account = request.GET.get('account') + if not account: + return response.Response( + {'account': 'This field is mandatory'}, status=400) + + data = library.scan_from_account_name(account) + return response.Response(data) diff --git a/api/funkwhale_api/federation/webfinger.py b/api/funkwhale_api/federation/webfinger.py index 444998b94..d4170a431 100644 --- a/api/funkwhale_api/federation/webfinger.py +++ b/api/funkwhale_api/federation/webfinger.py @@ -36,7 +36,7 @@ def clean_acct(acct_string, ensure_local=True): raise forms.ValidationError( 'Invalid hostname {}'.format(hostname)) - if username not in actors.SYSTEM_ACTORS: + if ensure_local and username not in actors.SYSTEM_ACTORS: raise forms.ValidationError('Invalid username') return username, hostname diff --git a/api/tests/federation/test_library.py b/api/tests/federation/test_library.py new file mode 100644 index 000000000..714a0c306 --- /dev/null +++ b/api/tests/federation/test_library.py @@ -0,0 +1,66 @@ +from funkwhale_api.federation import library +from funkwhale_api.federation import serializers + + +def test_library_scan_from_account_name(mocker, factories): + actor = factories['federation.Actor']( + preferred_username='library', + domain='test.library' + ) + get_resource_result = {'actor_url': actor.url} + get_resource = mocker.patch( + 'funkwhale_api.federation.webfinger.get_resource', + return_value=get_resource_result) + + actor_data = serializers.ActorSerializer(actor).data + actor_data['manuallyApprovesFollowers'] = False + actor_data['url'] = [{ + 'type': 'Link', + 'name': 'library', + 'mediaType': 'application/activity+json', + 'href': 'https://test.library' + }] + get_actor_data = mocker.patch( + 'funkwhale_api.federation.actors.get_actor_data', + return_value=actor_data) + + get_library_data_result = {'test': 'test'} + get_library_data = mocker.patch( + 'funkwhale_api.federation.library.get_library_data', + return_value=get_library_data_result) + + result = library.scan_from_account_name('library@test.actor') + + get_resource.assert_called_once_with('acct:library@test.actor') + get_actor_data.assert_called_once_with(actor.url) + get_library_data.assert_called_once_with(actor_data['url'][0]['href']) + + assert result == { + 'webfinger': get_resource_result, + 'actor': actor_data, + 'library': get_library_data_result, + } + + +def test_get_library_data(r_mock, factories): + actor = factories['federation.Actor']() + url = 'https://test.library' + conf = { + 'id': url, + 'items': [], + 'actor': actor, + 'page_size': 5, + } + data = serializers.PaginatedCollectionSerializer(conf).data + r_mock.get(url, json=data) + + result = library.get_library_data(url) + for f in ['totalItems', 'actor', 'id', 'type']: + assert result[f] == data[f] + + +def test_get_library_data_requires_authentication(r_mock, factories): + url = 'https://test.library' + r_mock.get(url, status_code=403) + result = library.get_library_data(url) + assert result['errors'] == ['This library requires authentication'] diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index b3fd85910..0b58e20f1 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -164,3 +164,18 @@ def test_library_actor_includes_library_link(db, settings, api_client): ] assert response.status_code == 200 assert response.data['url'] == expected_links + + +def test_can_scan_library(superuser_api_client, mocker): + result = {'test': 'test'} + scan = mocker.patch( + 'funkwhale_api.federation.library.scan_from_account_name', + return_value=result) + + url = reverse('api:v1:federation:libraries-scan') + response = superuser_api_client.get( + url, data={'account': 'test@test.library'}) + + assert response.status_code == 200 + assert response.data == result + scan.assert_called_once_with('test@test.library') From 6c0a43a0eaaf40bb9ab16db2775f706d02098ebb Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 8 Apr 2018 18:19:32 +0200 Subject: [PATCH 12/42] We can now work on federation locally thank to traefik --- .env.dev | 6 ++-- README.rst | 67 +++++++++++++++++++++++++++++++++++ api/config/settings/common.py | 27 ++++++++++++-- dev.yml | 45 ++++++++++++++++++++--- docker/nginx/entrypoint.sh | 15 +++++--- docker/ssl/test.crt | 22 ++++++++++++ docker/ssl/test.key | 28 +++++++++++++++ docker/traefik.toml | 26 ++++++++++++++ docker/traefik.yml | 22 ++++++++++++ 9 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 docker/ssl/test.crt create mode 100644 docker/ssl/test.key create mode 100644 docker/traefik.toml create mode 100644 docker/traefik.yml diff --git a/.env.dev b/.env.dev index 2e8834143..5a010cdf4 100644 --- a/.env.dev +++ b/.env.dev @@ -1,9 +1,11 @@ API_AUTHENTICATION_REQUIRED=True RAVEN_ENABLED=false RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5 -DJANGO_ALLOWED_HOSTS=localhost,nginx +DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx DJANGO_SETTINGS_MODULE=config.settings.local DJANGO_SECRET_KEY=dev C_FORCE_ROOT=true -FUNKWHALE_URL=http://localhost +FUNKWHALE_HOSTNAME=localhost +FUNKWHALE_PROTOCOL=http PYTHONDONTWRITEBYTECODE=true +WEBPACK_DEVSERVER_PORT=8080 diff --git a/README.rst b/README.rst index 2d5d2011d..2e4772adb 100644 --- a/README.rst +++ b/README.rst @@ -206,3 +206,70 @@ Typical workflow for a merge request 6. Push your branch 7. Create your merge request 8. Take a step back and enjoy, we're really grateful you did all of this and took the time to contribute! + + +Working with federation locally +------------------------------- + +To achieve that, you'll need: + +1. to update your dns resolver to resolve all your .dev hostnames locally +2. a reverse proxy (such as traefik) to catch those .dev requests and + and with https certificate +3. two instances (or more) running locally, following the regular dev setup + +Resolve .dev names locally +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you use dnsmasq, this is as simple as doing:: + + echo "address=/test/172.17.0.1" | sudo tee /etc/dnsmasq.d/test.conf + sudo systemctl restart dnsmasq + +If you use NetworkManager with dnsmasq integration, use this instead:: + + echo "address=/test/172.17.0.1" | sudo tee /etc/NetworkManager/dnsmasq.d/test.conf + sudo systemctl restart NetworkManager + +Add wildcard certificate to the trusted certificates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Simply copy bundled certificates:: + + sudo cp docker/ssl/test.crt /usr/local/share/ca-certificates/ + sudo update-ca-certificates + +This certificate is a wildcard for ``*.funkwhale.test`` + +Run a reverse proxy for your instances +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + +Crete docker network +^^^^^^^^^^^^^^^^^^^^ + +Create the federation network:: + + docker network create federation + +Launch everything +^^^^^^^^^^^^^^^^^ + +Launch the traefik proxy:: + + docker-compose -f docker/traefik.yml up -d + +Then, in separate terminals, you can setup as many different instances as you +need:: + + export COMPOSE_PROJECT_NAME=node2 + docker-compose -f dev.yml run --rm api python manage.py migrate + docker-compose -f dev.yml run --rm api python manage.py createsuperuser + docker-compose -f dev.yml up nginx api front + +Note that by default, if you don't export the COMPOSE_PROJECT_NAME, +we will default to node1 as the name of your instance. + +Assuming your project name is ``node1``, your server will be reachable +at ``https://node1.funkwhale.test/``. Not that you'll have to trust +the SSL Certificate as it's self signed. diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 7b7f6e64c..ebfd21dd6 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -25,8 +25,26 @@ try: except FileNotFoundError: pass -FUNKWHALE_URL = env('FUNKWHALE_URL') -FUNKWHALE_HOSTNAME = urlsplit(FUNKWHALE_URL).netloc +FUNKWHALE_HOSTNAME = None +FUNKWHALE_HOSTNAME_SUFFIX = env('FUNKWHALE_HOSTNAME_SUFFIX', default=None) +FUNKWHALE_HOSTNAME_PREFIX = env('FUNKWHALE_HOSTNAME_PREFIX', default=None) +if FUNKWHALE_HOSTNAME_PREFIX and FUNKWHALE_HOSTNAME_SUFFIX: + # We're in traefik case, in development + FUNKWHALE_HOSTNAME = '{}.{}'.format( + FUNKWHALE_HOSTNAME_PREFIX, FUNKWHALE_HOSTNAME_SUFFIX) + FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https') +else: + try: + FUNKWHALE_HOSTNAME = env('FUNKWHALE_HOSTNAME') + FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https') + except Exception: + FUNKWHALE_URL = env('FUNKWHALE_URL') + _parsed = urlsplit(FUNKWHALE_URL) + FUNKWHALE_HOSTNAME = _parsed.netloc + FUNKWHALE_PROTOCOL = _parsed.scheme + +FUNKWHALE_URL = '{}://{}'.format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME) + FEDERATION_ENABLED = env.bool('FEDERATION_ENABLED', default=True) FEDERATION_HOSTNAME = env('FEDERATION_HOSTNAME', default=FUNKWHALE_HOSTNAME) @@ -406,3 +424,8 @@ ACCOUNT_USERNAME_BLACKLIST = [ 'staff', 'service', ] + env.list('ACCOUNT_USERNAME_BLACKLIST', default=[]) + +EXTERNAL_REQUESTS_VERIFY_SSL = env.bool( + 'EXTERNAL_REQUESTS_VERIFY_SSL', + default=True +) diff --git a/dev.yml b/dev.yml index 409b6b4a7..9488d4a6f 100644 --- a/dev.yml +++ b/dev.yml @@ -10,22 +10,39 @@ services: - "HOST=0.0.0.0" - "WEBPACK_DEVSERVER_PORT=${WEBPACK_DEVSERVER_PORT-8080}" ports: - - "${WEBPACK_DEVSERVER_PORT-8080}:${WEBPACK_DEVSERVER_PORT-8080}" + - "${WEBPACK_DEVSERVER_PORT_BINDING-8080:}${WEBPACK_DEVSERVER_PORT-8080}" volumes: - './front:/app' - './po:/po' + networks: + - federation + - internal + labels: + traefik.backend: "${COMPOSE_PROJECT_NAME-node1}" + traefik.frontend.rule: "Host: ${COMPOSE_PROJECT_NAME-node1}.funkwhale.test" + traefik.enable: 'true' + traefik.federation.protocol: 'http' + traefik.federation.port: "${WEBPACK_DEVSERVER_PORT-8080}" postgres: env_file: - .env.dev - .env image: postgres + volumes: + - "./data/${COMPOSE_PROJECT_NAME-node1}/postgres:/var/lib/postgresql/data" + networks: + - internal redis: env_file: - .env.dev - .env image: redis:3.0 + volumes: + - "./data/${COMPOSE_PROJECT_NAME-node1}/redis:/data" + networks: + - internal celeryworker: env_file: @@ -39,11 +56,17 @@ services: - redis command: celery -A funkwhale_api.taskapp worker -l debug environment: + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}" + - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test" + - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}" + - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL-http}" - "DATABASE_URL=postgresql://postgres@postgres/postgres" - "CACHE_URL=redis://redis:6379/0" volumes: - ./api:/app - ./data/music:/music + networks: + - internal api: env_file: - .env.dev @@ -56,12 +79,17 @@ services: - ./api:/app - ./data/music:/music environment: + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}" + - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test" + - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}" + - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL-http}" - "DATABASE_URL=postgresql://postgres@postgres/postgres" - "CACHE_URL=redis://redis:6379/0" links: - postgres - redis - + networks: + - internal nginx: command: /entrypoint.sh env_file: @@ -70,6 +98,8 @@ services: image: nginx environment: - "WEBPACK_DEVSERVER_PORT=${WEBPACK_DEVSERVER_PORT-8080}" + - "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME- }" + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}" links: - api - front @@ -79,8 +109,9 @@ services: - ./deploy/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf.template:ro - ./api/funkwhale_api/media:/protected/media ports: - - "0.0.0.0:6001:6001" - + - "6001" + networks: + - internal docs: build: docs command: python serve.py @@ -89,3 +120,9 @@ services: ports: - '35730:35730' - '8001:8001' + +networks: + internal: + federation: + external: + name: federation diff --git a/docker/nginx/entrypoint.sh b/docker/nginx/entrypoint.sh index 1819acf1c..14e072a7e 100755 --- a/docker/nginx/entrypoint.sh +++ b/docker/nginx/entrypoint.sh @@ -1,10 +1,17 @@ #!/bin/bash -eux -FIRST_HOST=$(echo ${DJANGO_ALLOWED_HOSTS} | cut -d, -f1) + +FORWARDED_PORT="$WEBPACK_DEVSERVER_PORT" +COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME// /}" +if [ -n "$COMPOSE_PROJECT_NAME" ]; then + echo + FUNKWHALE_HOSTNAME="$COMPOSE_PROJECT_NAME.funkwhale.test" + FORWARDED_PORT="443" +fi echo "Copying template file..." cp /etc/nginx/funkwhale_proxy.conf{.template,} -sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host ${FIRST_HOST}:${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf -sed -i "s/proxy_set_header Host \$host/proxy_set_header Host ${FIRST_HOST}/" /etc/nginx/funkwhale_proxy.conf -sed -i "s/proxy_set_header X-Forwarded-Port \$server_port/proxy_set_header X-Forwarded-Port ${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host ${FUNKWHALE_HOSTNAME}:${FORWARDED_PORT}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/proxy_set_header Host \$host/proxy_set_header Host ${FUNKWHALE_HOSTNAME}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/proxy_set_header X-Forwarded-Port \$server_port/proxy_set_header X-Forwarded-Port ${FORWARDED_PORT}/" /etc/nginx/funkwhale_proxy.conf cat /etc/nginx/funkwhale_proxy.conf nginx -g "daemon off;" diff --git a/docker/ssl/test.crt b/docker/ssl/test.crt new file mode 100644 index 000000000..e4d3eefb8 --- /dev/null +++ b/docker/ssl/test.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDljCCAn6gAwIBAgIJAOA/w9NwL3aMMA0GCSqGSIb3DQEBCwUAMGAxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQxGTAXBgNVBAMMECouZnVua3doYWxlLnRlc3QwHhcNMTgw +NDA4MTMwNDAzWhcNMjgwNDA1MTMwNDAzWjBgMQswCQYDVQQGEwJBVTETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRkwFwYDVQQDDBAqLmZ1bmt3aGFsZS50ZXN0MIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEAyGqRLEMFs1mpRwauTicIRj2zwBUe6JMNRbIvOUkaj2KY6avA +7tiNti/ygBoTyJl2JK3mmLqxElqedpMhjVvYde/PyjXoZ+0Vq4FWv89LV6ZM/Scf +TCIYwWF1ppi6GYFmU3WCIMISkKiPBtMArB0oZxiUWLmkyd8jih2wnQOpkQ20FfG0 +CtlrKlQKyAe7X3zPuqGfaMUN7J4w9g3/SC66YulbAtI1/Z4tuG8J4m2RC6jH1hVy +364l3ifEC+m9Kax/ystfu/mkLdyQgRfOZTNf2JhS3BL8zpoWMXFK+4+7TYisrV1h +0pzIAsoQeBB+cFOOFEwRAv0FxSWnZ+/shjnwbwIDAQABo1MwUTAdBgNVHQ4EFgQU +sULmofttRyWUMM93IsD8jBvyCd4wHwYDVR0jBBgwFoAUsULmofttRyWUMM93IsD8 +jBvyCd4wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAUg/fiXut +hW6fDx9f0JdB4uLiLnv8tDP35ackLLapFJhXtflIXcqCzxStQ46nMs1wjaZPb+ws +pLULzvTKTxJbu+JYc2nvis4m2oSFczJ3S9tgug4Ppv8yS7N1pp7kfjOvBjgh6sYW +p+Ctb5r8qvgvT9yDTeCnsqktb/OkRHlHwhRYfnuxh+96s4mzifqFUP4uCCcFYPTc +RE0Ag3oI5sHOdDk/cdYE5PGQPjSP6gzn0lsrz1Q3x1C8+txSHzsJnvS3Ost+dwcy +JSjDBXauy9cZv93Voevcl16Ioo7trtkp4dwAoep52vOT/KMkJ4zm19msV3BP4wMa +BUqrV2F7twD5zw== +-----END CERTIFICATE----- diff --git a/docker/ssl/test.key b/docker/ssl/test.key new file mode 100644 index 000000000..669e5f700 --- /dev/null +++ b/docker/ssl/test.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDIapEsQwWzWalH +Bq5OJwhGPbPAFR7okw1Fsi85SRqPYpjpq8Du2I22L/KAGhPImXYkreaYurESWp52 +kyGNW9h178/KNehn7RWrgVa/z0tXpkz9Jx9MIhjBYXWmmLoZgWZTdYIgwhKQqI8G +0wCsHShnGJRYuaTJ3yOKHbCdA6mRDbQV8bQK2WsqVArIB7tffM+6oZ9oxQ3snjD2 +Df9ILrpi6VsC0jX9ni24bwnibZELqMfWFXLfriXeJ8QL6b0prH/Ky1+7+aQt3JCB +F85lM1/YmFLcEvzOmhYxcUr7j7tNiKytXWHSnMgCyhB4EH5wU44UTBEC/QXFJadn +7+yGOfBvAgMBAAECggEBAMVB3lEqRloYTbxSnwzc7g/0ew77usg+tDl8/23qvfGS +od6b5fEvw4sl9hCPmhk+skG3x9dbKR1fg8hBWCzB0XOC7YmhNXXUrBd53eA8L3O9 +gtlHwE424Ra0zg+DEug3rHdImSOU4KDwxpV46Jh+ul1+m8QYNFFdBqXSQxrHmAXj +MQ6++rjoJ+bhucmjBouzMYXHTGhdae3kjDFrFJ4cUsH6F03NcDwS+AmZxa/DWQ/H +SoBQBeLoE6I1aKhLgY91yO1e7CtSzS2GFCODReN4b3cylaR7jE7Mg87TZcga6Wfa +Xcd120VVlVq6HmZc/Xob7aUim3AuY2er8bcvmg1XOsECgYEA5EMM5UlpLdNWv1hp +5IMvkeCbXtLJ3IOHO0xLkFdx0CxaR9TyAAqIrSh1t9rFhYqLUNiOdMc2TqrvdgEU +B/QZrAevWRc5sjPvFXmYeWSCi/tjRgQh4jClWDX/TlfAlP55z2BFyMPMX6//WbBQ +5aL9xymTymzFFcaE8EytT5Jz8rUCgYEA4MVF3IkaQepl6H1gf2T6ev+MtGk9AGg9 +DSJpio7hfMcY5X3NrTJJFF9DJFXqfo3ILOMyUpIUHqkCGKXil0n9ypLp4vq7l+6c +m1gtKFXh7uKAV4XtSnR0nuK/N10JJp2HbbFYGlziRaa1iEPAFvLDQHu4jyf5sXyV +HvreuQgGWRMCgYEAlUaQKWaP5UsfoPUGE04DjwfvM9zv7EkL6CimBhhZswU+aVmG +haZd6bfa/EiTAhkvsMheqVoaVuoMvgRIgEcPfuRrtPyuW68A/O9PWpvzj+3v5zsO +maisiPqPI0HaDNY6/PZ9zKTXhABKIvJehT7JbjTvlOL7JJl2GNxcPvyM3T0CgYEA +tnVtUKi69+ce8qtUOhXufwoTXiBPtJTpelAE/MUfpfq46xJEc+PuDuuFxWk5AaJ2 +bHnBz+VlD76CRR/j4IvfySGZWvfOcHbyCeh6P9P3o8OaC3JcPaRrRs8qCfcsBny6 +AwGDU2MzCvdZRVQ6CmbmuOG13//DYaCQLKXZRrqM7KECgYEAxDsqtyHA/a38UhS8 +iQ8HqrZp8CuzJoJw/QILvzjojD1cvmwF73RrPEpRfEaLWVQGQ5F1IlHk/009C5zy +eUT4ZaPxLem6khBf7pn3xXaVBGZsYoltek5sUBsu/jA+4Sw6bcUmhBRBCs98JGpR +DVJtvOTk9aGW8M8UbgqwW+e/6ng= +-----END PRIVATE KEY----- diff --git a/docker/traefik.toml b/docker/traefik.toml new file mode 100644 index 000000000..85da2ea72 --- /dev/null +++ b/docker/traefik.toml @@ -0,0 +1,26 @@ +defaultEntryPoints = ["http", "https"] + +################################################################ +# Web configuration backend +################################################################ +[web] +address = ":8040" +################################################################ +# Docker configuration backend +################################################################ +[docker] +domain = "funkwhale.test" +watch = true +exposedbydefault = false + +[entryPoints] + [entryPoints.http] + address = ":80" + [entryPoints.http.redirect] + entryPoint = "https" + [entryPoints.https] + address = ":443" + [entryPoints.https.tls] + [[entryPoints.https.tls.certificates]] + certFile = "/ssl/traefik.crt" + keyFile = "/ssl/traefik.key" diff --git a/docker/traefik.yml b/docker/traefik.yml new file mode 100644 index 000000000..0b15b3290 --- /dev/null +++ b/docker/traefik.yml @@ -0,0 +1,22 @@ +version: '2.1' + +services: + traefik: + image: traefik:alpine + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./traefik.toml:/traefik.toml + - ./ssl/test.key:/ssl/traefik.key + - ./ssl/test.crt:/ssl/traefik.crt + ports: + - '80:80' + - '443:443' + - '8040:8040' + networks: + federation: + + +networks: + federation: + external: + name: federation From 238d849298735af124c812969fa8020c0ff396b8 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 8 Apr 2018 18:24:07 +0200 Subject: [PATCH 13/42] Can now disable SSL cerification for external requests --- README.rst | 8 ++++++++ api/funkwhale_api/federation/activity.py | 3 +++ api/funkwhale_api/federation/actors.py | 1 + api/funkwhale_api/federation/library.py | 3 +++ api/funkwhale_api/federation/webfinger.py | 5 ++++- api/funkwhale_api/music/views.py | 1 + 6 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2e4772adb..f39baead6 100644 --- a/README.rst +++ b/README.rst @@ -273,3 +273,11 @@ we will default to node1 as the name of your instance. Assuming your project name is ``node1``, your server will be reachable at ``https://node1.funkwhale.test/``. Not that you'll have to trust the SSL Certificate as it's self signed. + +When working on federation with traefik, ensure you have this in your ``env``:: + + # This will ensure we don't bind any port on the host, and thus enable + # multiple instances of funkwhale to be spawned concurrently. + WEBPACK_DEVSERVER_PORT_BINDING= + # This disable certificate verification + EXTERNAL_REQUESTS_VERIFY_SSL=false diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index a674c70e3..24a1f782e 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -3,6 +3,8 @@ import json import requests_http_signature import uuid +from django.conf import settings + from funkwhale_api.common import session from . import models @@ -74,6 +76,7 @@ def deliver(activity, on_behalf_of, to=[]): json=activity, url=recipient_actor.inbox_url, timeout=5, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={ 'Content-Type': 'application/activity+json' } diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index d3a2093a9..bb0b99cc2 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -32,6 +32,7 @@ def get_actor_data(actor_url): response = session.get_session().get( actor_url, timeout=5, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={ 'Accept': 'application/activity+json', } diff --git a/api/funkwhale_api/federation/library.py b/api/funkwhale_api/federation/library.py index 13608098b..f9a1de8f7 100644 --- a/api/funkwhale_api/federation/library.py +++ b/api/funkwhale_api/federation/library.py @@ -1,5 +1,7 @@ import requests +from django.conf import settings + from funkwhale_api.common import session from . import actors @@ -69,6 +71,7 @@ def get_library_data(library_url): library_url, auth=auth, timeout=5, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={ 'Content-Type': 'application/activity+json' } diff --git a/api/funkwhale_api/federation/webfinger.py b/api/funkwhale_api/federation/webfinger.py index d4170a431..f5cb99635 100644 --- a/api/funkwhale_api/federation/webfinger.py +++ b/api/funkwhale_api/federation/webfinger.py @@ -47,7 +47,10 @@ def get_resource(resource_string): username, hostname = clean_acct(resource, ensure_local=False) url = 'https://{}/.well-known/webfinger?resource={}'.format( hostname, resource_string) - response = session.get_session().get(url, timeout=5) + response = session.get_session().get( + url, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, + timeout=5) response.raise_for_status() serializer = serializers.ActorWebfingerSerializer(data=response.json()) serializer.is_valid(raise_exception=True) diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index 6bbc21db7..98048b41d 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -219,6 +219,7 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet): auth=auth, stream=True, timeout=20, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={ 'Content-Type': 'application/activity+json' }) From ea27dd917f147b26de00440a7ea8fe986cde21bd Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 8 Apr 2018 20:26:48 +0200 Subject: [PATCH 14/42] Latest (hopefully) traefik tweaks --- .env.dev | 2 +- README.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.env.dev b/.env.dev index 5a010cdf4..3f904078c 100644 --- a/.env.dev +++ b/.env.dev @@ -1,7 +1,7 @@ API_AUTHENTICATION_REQUIRED=True RAVEN_ENABLED=false RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5 -DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx +DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx,0.0.0.0,127.0.0.1 DJANGO_SETTINGS_MODULE=config.settings.local DJANGO_SECRET_KEY=dev C_FORCE_ROOT=true diff --git a/README.rst b/README.rst index f39baead6..747a1e220 100644 --- a/README.rst +++ b/README.rst @@ -281,3 +281,5 @@ When working on federation with traefik, ensure you have this in your ``env``:: WEBPACK_DEVSERVER_PORT_BINDING= # This disable certificate verification EXTERNAL_REQUESTS_VERIFY_SSL=false + # this ensure you don't have incorrect urls pointing to http resources + FUNKWHALE_PROTOCOL=https From f0ef9ea561d9db589e2558f9e8065b75f9edc064 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 8 Apr 2018 20:27:10 +0200 Subject: [PATCH 15/42] Better error handling during scan --- api/funkwhale_api/federation/library.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/funkwhale_api/federation/library.py b/api/funkwhale_api/federation/library.py index f9a1de8f7..6fa3c7183 100644 --- a/api/funkwhale_api/federation/library.py +++ b/api/funkwhale_api/federation/library.py @@ -96,5 +96,10 @@ def get_library_data(library_url): serializer = serializers.PaginatedCollectionSerializer( data=response.json(), ) - serializer.is_valid(raise_exception=True) + if not serializer.is_valid(): + return { + 'errors': [ + 'Invalid ActivityPub response from remote library'] + } + return serializer.validated_data From c97db31cb129e03248e59a2ffc7a99c0b1924998 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Mon, 9 Apr 2018 19:00:11 +0200 Subject: [PATCH 16/42] Include following state in scan payload --- api/funkwhale_api/federation/library.py | 47 +++++++++++++++++++++++- api/tests/federation/test_library.py | 6 ++- api/tests/federation/test_serializers.py | 1 - 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/api/funkwhale_api/federation/library.py b/api/funkwhale_api/federation/library.py index 6fa3c7183..f19a7a291 100644 --- a/api/funkwhale_api/federation/library.py +++ b/api/funkwhale_api/federation/library.py @@ -1,3 +1,4 @@ +import json import requests from django.conf import settings @@ -5,6 +6,7 @@ from django.conf import settings from funkwhale_api.common import session from . import actors +from . import models from . import serializers from . import signing from . import webfinger @@ -22,6 +24,39 @@ def scan_from_account_name(account_name): """ data = {} + try: + username, domain = webfinger.clean_acct( + account_name, ensure_local=False) + except serializers.ValidationError: + return { + 'webfinger': { + 'errors': ['Invalid account string'] + } + } + system_library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + library = models.Library.objects.filter( + actor__domain=domain, + actor__preferred_username=username + ).select_related('actor').first() + follow_request = None + if library: + data['local']['following'] = True + data['local']['awaiting_approval'] = True + + else: + follow_request = models.FollowRequest.objects.filter( + target__preferred_username=username, + target__domain=username, + actor=system_library, + ).first() + data['local'] = { + 'following': False, + 'awaiting_approval': False, + } + if follow_request: + data['awaiting_approval'] = follow_request.approved is None + + follow_request = models.Follow try: data['webfinger'] = webfinger.get_resource( 'acct:{}'.format(account_name)) @@ -39,6 +74,12 @@ def scan_from_account_name(account_name): e.response.status_code)] } } + except json.JSONDecodeError as e: + return { + 'webfinger': { + 'errors': ['Could not process webfinger response'] + } + } try: data['actor'] = actors.get_actor_data(data['webfinger']['actor_url']) @@ -56,7 +97,11 @@ def scan_from_account_name(account_name): return data serializer = serializers.LibraryActorSerializer(data=data['actor']) - serializer.is_valid(raise_exception=True) + if not serializer.is_valid(): + data['actor'] = { + 'errors': ['Invalid ActivityPub actor'] + } + return data data['library'] = get_library_data( serializer.validated_data['library_url']) diff --git a/api/tests/federation/test_library.py b/api/tests/federation/test_library.py index 714a0c306..7a3abf5d8 100644 --- a/api/tests/federation/test_library.py +++ b/api/tests/federation/test_library.py @@ -39,6 +39,10 @@ def test_library_scan_from_account_name(mocker, factories): 'webfinger': get_resource_result, 'actor': actor_data, 'library': get_library_data_result, + 'local': { + 'following': False, + 'awaiting_approval': False, + }, } @@ -63,4 +67,4 @@ def test_get_library_data_requires_authentication(r_mock, factories): url = 'https://test.library' r_mock.get(url, status_code=403) result = library.get_library_data(url) - assert result['errors'] == ['This library requires authentication'] + assert result['errors'] == ['Permission denied while scanning library'] diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index e4fb88040..7b7dda33c 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -218,7 +218,6 @@ def test_paginated_collection_serializer_validation(): assert serializer.validated_data['totalItems'] == 5 assert serializer.validated_data['id'] == data['id'] assert serializer.validated_data['actor'] == data['actor'] - assert serializer.validated_data['items'] == [] def test_collection_page_serializer_validation(): From 0b2fe8439ae17bef680499e7f62054bc18e8dd98 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Tue, 10 Apr 2018 21:25:35 +0200 Subject: [PATCH 17/42] Removed too complex FollowRequest model, we now use an aproved field on Follow --- api/funkwhale_api/federation/activity.py | 69 +------ api/funkwhale_api/federation/actors.py | 74 ++++---- api/funkwhale_api/federation/admin.py | 15 +- api/funkwhale_api/federation/factories.py | 9 - api/funkwhale_api/federation/library.py | 26 ++- .../migrations/0004_auto_20180410_1624.py | 29 +++ api/funkwhale_api/federation/models.py | 44 +---- api/funkwhale_api/federation/serializers.py | 136 +++++++++++-- api/funkwhale_api/federation/views.py | 30 +++ api/tests/federation/test_activity.py | 37 +--- api/tests/federation/test_actors.py | 61 +++--- api/tests/federation/test_models.py | 44 ----- api/tests/federation/test_serializers.py | 179 ++++++++++++++++++ api/tests/federation/test_views.py | 33 ++++ 14 files changed, 480 insertions(+), 306 deletions(-) create mode 100644 api/funkwhale_api/federation/migrations/0004_auto_20180410_1624.py diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index 24a1f782e..af31b8c5a 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -6,8 +6,10 @@ import uuid from django.conf import settings from funkwhale_api.common import session +from funkwhale_api.common import utils as funkwhale_utils from . import models +from . import serializers from . import signing logger = logging.getLogger(__name__) @@ -85,66 +87,9 @@ def deliver(activity, on_behalf_of, to=[]): logger.debug('Remote answered with %s', response.status_code) -def get_follow(follow_id, follower, followed): - return { - '@context': [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - {} - ], - 'actor': follower.url, - 'id': follower.url + '#follows/{}'.format(follow_id), - 'object': followed.url, - 'type': 'Follow' - } - - -def get_undo(id, actor, object): - return { - '@context': [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - {} - ], - 'type': 'Undo', - 'id': id + '/undo', - 'actor': actor.url, - 'object': object, - } - - -def get_accept_follow(accept_id, accept_actor, follow, follow_actor): - return { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {} - ], - "id": accept_actor.url + '#accepts/follows/{}'.format( - accept_id), - "type": "Accept", - "actor": accept_actor.url, - "object": { - "id": follow['id'], - "type": "Follow", - "actor": follow_actor.url, - "object": accept_actor.url - }, - } - - -def accept_follow(target, follow, actor): - accept_uuid = uuid.uuid4() - accept = get_accept_follow( - accept_id=accept_uuid, - accept_actor=target, - follow=follow, - follow_actor=actor) +def accept_follow(follow): + serializer = serializers.AcceptFollowSerializer(follow) deliver( - accept, - to=[actor.url], - on_behalf_of=target) - return models.Follow.objects.get_or_create( - actor=actor, - target=target, - ) + serializer.data, + to=[follow.actor.url], + on_behalf_of=follow.target) diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index bb0b99cc2..5a4e917bd 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -153,24 +153,32 @@ class SystemActor(object): def handle_follow(self, ac, sender): system_actor = self.get_actor_instance() - if self.manually_approves_followers: - fr, created = models.FollowRequest.objects.get_or_create( - actor=sender, - target=system_actor, - approved=None, - ) - return fr + serializer = serializers.FollowSerializer( + data=ac, context={'follow_actor': sender}) + if not serializer.is_valid(): + return logger.info('Invalid follow payload') + approved = True if not self.manually_approves_followers else None + follow = serializer.save(approved=approved) + if follow.approved: + return activity.accept_follow(follow) - return activity.accept_follow( - system_actor, ac, sender - ) + def handle_accept(self, ac, sender): + system_actor = self.get_actor_instance() + serializer = serializers.AcceptFollowSerializer( + data=ac, + context={'follow_target': sender, 'follow_actor': system_actor}) + if not serializer.is_valid(raise_exception=True): + return logger.info('Received invalid payload') + + serializer.save() def handle_undo_follow(self, ac, sender): - actor = self.get_actor_instance() - models.Follow.objects.filter( - actor=sender, - target=actor, - ).delete() + system_actor = self.get_actor_instance() + serializer = serializers.UndoFollowSerializer( + data=ac, context={'actor': sender, 'target': system_actor}) + if not serializer.is_valid(): + return logger.info('Received invalid payload') + serializer.save() def handle_undo(self, ac, sender): if ac['object']['type'] != 'Follow': @@ -206,20 +214,6 @@ class LibraryActor(SystemActor): def manually_approves_followers(self): return settings.FEDERATION_MUSIC_NEEDS_APPROVAL - def handle_follow(self, ac, sender): - system_actor = self.get_actor_instance() - if self.manually_approves_followers: - fr, created = models.FollowRequest.objects.get_or_create( - actor=sender, - target=system_actor, - approved=None, - ) - return fr - - return activity.accept_follow( - system_actor, ac, sender - ) - @transaction.atomic def handle_create(self, ac, sender): try: @@ -360,15 +354,15 @@ class TestActor(SystemActor): super().handle_follow(ac, sender) # also, we follow back test_actor = self.get_actor_instance() - follow_uuid = uuid.uuid4() - follow = activity.get_follow( - follow_id=follow_uuid, - follower=test_actor, - followed=sender) + follow_back = models.Follow.objects.get_or_create( + actor=test_actor, + target=sender, + approved=None, + )[0] activity.deliver( - follow, - to=[ac['actor']], - on_behalf_of=test_actor) + serializers.FollowSerializer(follow_back).data, + to=[follow_back.target.url], + on_behalf_of=follow_back.actor) def handle_undo_follow(self, ac, sender): super().handle_undo_follow(ac, sender) @@ -381,11 +375,7 @@ class TestActor(SystemActor): ) except models.Follow.DoesNotExist: return - undo = activity.get_undo( - id=follow.get_federation_url(), - actor=actor, - object=serializers.FollowSerializer(follow).data, - ) + undo = serializers.UndoFollowSerializer(follow).data follow.delete() activity.deliver( undo, diff --git a/api/funkwhale_api/federation/admin.py b/api/funkwhale_api/federation/admin.py index ce636299f..6a097174b 100644 --- a/api/funkwhale_api/federation/admin.py +++ b/api/funkwhale_api/federation/admin.py @@ -23,24 +23,13 @@ class FollowAdmin(admin.ModelAdmin): list_display = [ 'actor', 'target', + 'approved', 'creation_date' ] - search_fields = ['actor__url', 'target__url'] - list_select_related = True - - -@admin.register(models.FollowRequest) -class FollowRequestAdmin(admin.ModelAdmin): - list_display = [ - 'actor', - 'target', - 'creation_date', - 'approved' - ] - search_fields = ['actor__url', 'target__url'] list_filter = [ 'approved' ] + search_fields = ['actor__url', 'target__url'] list_select_related = True diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index b3ac72039..1aeb733c8 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -113,15 +113,6 @@ class FollowFactory(factory.DjangoModelFactory): ) -@registry.register -class FollowRequestFactory(factory.DjangoModelFactory): - target = factory.SubFactory(ActorFactory) - actor = factory.SubFactory(ActorFactory) - - class Meta: - model = models.FollowRequest - - @registry.register class LibraryFactory(factory.DjangoModelFactory): actor = factory.SubFactory(ActorFactory) diff --git a/api/funkwhale_api/federation/library.py b/api/funkwhale_api/federation/library.py index f19a7a291..177f14754 100644 --- a/api/funkwhale_api/federation/library.py +++ b/api/funkwhale_api/federation/library.py @@ -38,25 +38,21 @@ def scan_from_account_name(account_name): actor__domain=domain, actor__preferred_username=username ).select_related('actor').first() - follow_request = None - if library: - data['local']['following'] = True - data['local']['awaiting_approval'] = True - - else: - follow_request = models.FollowRequest.objects.filter( + data['local'] = { + 'following': False, + 'awaiting_approval': False, + } + try: + follow = models.Follow.objects.get( target__preferred_username=username, target__domain=username, actor=system_library, - ).first() - data['local'] = { - 'following': False, - 'awaiting_approval': False, - } - if follow_request: - data['awaiting_approval'] = follow_request.approved is None + ) + data['local']['awaiting_approval'] = not bool(follow.approved) + data['local']['following'] = True + except models.Follow.DoesNotExist: + pass - follow_request = models.Follow try: data['webfinger'] = webfinger.get_resource( 'acct:{}'.format(account_name)) diff --git a/api/funkwhale_api/federation/migrations/0004_auto_20180410_1624.py b/api/funkwhale_api/federation/migrations/0004_auto_20180410_1624.py new file mode 100644 index 000000000..b199706aa --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0004_auto_20180410_1624.py @@ -0,0 +1,29 @@ +# Generated by Django 2.0.3 on 2018-04-10 16:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0003_auto_20180407_1010'), + ] + + operations = [ + migrations.RemoveField( + model_name='followrequest', + name='actor', + ), + migrations.RemoveField( + model_name='followrequest', + name='target', + ), + migrations.AddField( + model_name='follow', + name='approved', + field=models.NullBooleanField(default=None), + ), + migrations.DeleteModel( + name='FollowRequest', + ), + ] diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index bf1e5d830..201463066 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -109,6 +109,7 @@ class Follow(models.Model): creation_date = models.DateTimeField(default=timezone.now) modification_date = models.DateTimeField( auto_now=True) + approved = models.NullBooleanField(default=None) class Meta: unique_together = ['actor', 'target'] @@ -117,49 +118,6 @@ class Follow(models.Model): return '{}#follows/{}'.format(self.actor.url, self.uuid) -class FollowRequest(models.Model): - uuid = models.UUIDField(default=uuid.uuid4, unique=True) - actor = models.ForeignKey( - Actor, - related_name='emmited_follow_requests', - on_delete=models.CASCADE, - ) - target = models.ForeignKey( - Actor, - related_name='received_follow_requests', - on_delete=models.CASCADE, - ) - creation_date = models.DateTimeField(default=timezone.now) - modification_date = models.DateTimeField( - auto_now=True) - approved = models.NullBooleanField(default=None) - - def approve(self): - from . import activity - from . import serializers - self.approved = True - self.save(update_fields=['approved']) - Follow.objects.get_or_create( - target=self.target, - actor=self.actor - ) - if self.target.is_local: - follow = { - '@context': serializers.AP_CONTEXT, - 'actor': self.actor.url, - 'id': self.actor.url + '#follows/{}'.format(uuid.uuid4()), - 'object': self.target.url, - 'type': 'Follow' - } - activity.accept_follow( - self.target, follow, self.actor - ) - - def refuse(self): - self.approved = False - self.save(update_fields=['approved']) - - class Library(models.Model): creation_date = models.DateTimeField(default=timezone.now) modification_date = models.DateTimeField( diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 704ad6364..f0d1e35fd 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -121,28 +121,132 @@ class LibraryActorSerializer(ActorSerializer): return validated_data -class FollowSerializer(serializers.ModelSerializer): - # left maps to activitypub fields, right to our internal models - id = serializers.URLField(source='get_federation_url') - object = serializers.URLField(source='target.url') - actor = serializers.URLField(source='actor.url') - type = serializers.CharField(source='ap_type') +class FollowSerializer(serializers.Serializer): + id = serializers.URLField() + object = serializers.URLField() + actor = serializers.URLField() + type = serializers.ChoiceField(choices=['Follow']) - class Meta: - model = models.Actor - fields = [ - 'id', - 'object', - 'actor', - 'type' - ] + def validate_object(self, v): + expected = self.context.get('follow_target') + if expected and expected.url != v: + raise serializers.ValidationError('Invalid target') + try: + return models.Actor.objects.get(url=v) + except models.Actor.DoesNotExist: + raise serializers.ValidationError('Target not found') + + def validate_actor(self, v): + expected = self.context.get('follow_actor') + if expected and expected.url != v: + raise serializers.ValidationError('Invalid actor') + try: + return models.Actor.objects.get(url=v) + except models.Actor.DoesNotExist: + raise serializers.ValidationError('Actor not found') + + def save(self, **kwargs): + return models.Follow.objects.get_or_create( + actor=self.validated_data['actor'], + target=self.validated_data['object'], + **kwargs, + )[0] def to_representation(self, instance): - ret = super().to_representation(instance) - ret['@context'] = AP_CONTEXT + return { + '@context': AP_CONTEXT, + 'actor': instance.actor.url, + 'id': instance.get_federation_url(), + 'object': instance.target.url, + 'type': 'Follow' + } return ret +class AcceptFollowSerializer(serializers.Serializer): + id = serializers.URLField() + actor = serializers.URLField() + object = FollowSerializer() + type = serializers.ChoiceField(choices=['Accept']) + + def validate_actor(self, v): + expected = self.context.get('follow_target') + if expected and expected.url != v: + raise serializers.ValidationError('Invalid actor') + try: + return models.Actor.objects.get(url=v) + except models.Actor.DoesNotExist: + raise serializers.ValidationError('Actor not found') + + def validate(self, validated_data): + # we ensure the accept actor actually match the follow target + if validated_data['actor'] != validated_data['object']['object']: + raise serializers.ValidationError('Actor mismatch') + try: + validated_data['follow'] = models.Follow.objects.filter( + target=validated_data['actor'], + actor=validated_data['object']['actor'] + ).exclude(approved=True).get() + except models.Follow.DoesNotExist: + raise serializers.ValidationError('No follow to accept') + return validated_data + + def to_representation(self, instance): + return { + "@context": AP_CONTEXT, + "id": instance.get_federation_url() + '/accept', + "type": "Accept", + "actor": instance.target.url, + "object": FollowSerializer(instance).data + } + + def save(self): + self.validated_data['follow'].approved = True + self.validated_data['follow'].save() + return self.validated_data['follow'] + + +class UndoFollowSerializer(serializers.Serializer): + id = serializers.URLField() + actor = serializers.URLField() + object = FollowSerializer() + type = serializers.ChoiceField(choices=['Undo']) + + def validate_actor(self, v): + expected = self.context.get('follow_target') + if expected and expected.url != v: + raise serializers.ValidationError('Invalid actor') + try: + return models.Actor.objects.get(url=v) + except models.Actor.DoesNotExist: + raise serializers.ValidationError('Actor not found') + + def validate(self, validated_data): + # we ensure the accept actor actually match the follow actor + if validated_data['actor'] != validated_data['object']['actor']: + raise serializers.ValidationError('Actor mismatch') + try: + validated_data['follow'] = models.Follow.objects.filter( + actor=validated_data['actor'], + target=validated_data['object']['object'] + ).get() + except models.Follow.DoesNotExist: + raise serializers.ValidationError('No follow to remove') + return validated_data + + def to_representation(self, instance): + return { + "@context": AP_CONTEXT, + "id": instance.get_federation_url() + '/undo', + "type": "Undo", + "actor": instance.actor.url, + "object": FollowSerializer(instance).data + } + + def save(self): + self.validated_data['follow'].delete() + + class ActorWebfingerSerializer(serializers.Serializer): subject = serializers.CharField() aliases = serializers.ListField(child=serializers.URLField()) diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index aaab343e4..2d4220472 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -1,6 +1,7 @@ from django import forms from django.conf import settings from django.core import paginator +from django.db import transaction from django.http import HttpResponse from django.urls import reverse @@ -9,9 +10,12 @@ from rest_framework import response from rest_framework import views from rest_framework import viewsets from rest_framework.decorators import list_route, detail_route +from rest_framework.serializers import ValidationError +from funkwhale_api.common import utils as funkwhale_utils from funkwhale_api.music.models import TrackFile +from . import activity from . import actors from . import authentication from . import library @@ -172,3 +176,29 @@ class LibraryViewSet(viewsets.GenericViewSet): data = library.scan_from_account_name(account) return response.Response(data) + + @transaction.atomic + def create(self, request, *args, **kwargs): + try: + actor_url = request.data['actor_url'] + except KeyError: + raise ValidationError('Missing actor_url') + + try: + actor = actors.get_actor(actor_url) + library_data = library.get_library_data(actor.url) + except Exception as e: + raise ValidationError('Error while fetching actor and library') + + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow, created = models.Follow.objects.get_or_create( + actor=library_actor, + target=actor, + ) + serializer = serializers.FollowSerializer(follow) + activity.deliver( + serializer.data, + on_behalf_of=library_actor, + to=[actor.url] + ) + return response.Response({}, status=201) diff --git a/api/tests/federation/test_activity.py b/api/tests/federation/test_activity.py index 09c5e3bf7..dbd60bbd7 100644 --- a/api/tests/federation/test_activity.py +++ b/api/tests/federation/test_activity.py @@ -1,6 +1,7 @@ import uuid from funkwhale_api.federation import activity +from funkwhale_api.federation import serializers def test_deliver(nodb_factories, r_mock, mocker): @@ -38,37 +39,9 @@ def test_deliver(nodb_factories, r_mock, mocker): def test_accept_follow(mocker, factories): deliver = mocker.patch( 'funkwhale_api.federation.activity.deliver') - actor = factories['federation.Actor']() - target = factories['federation.Actor'](local=True) - follow = { - 'actor': actor.url, - 'type': 'Follow', - 'id': 'http://test.federation/user#follows/267', - 'object': target.url, - } - uid = uuid.uuid4() - mocker.patch('uuid.uuid4', return_value=uid) - expected_accept = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {} - ], - "id": target.url + '#accepts/follows/{}'.format(uid), - "type": "Accept", - "actor": target.url, - "object": { - "id": follow['id'], - "type": "Follow", - "actor": actor.url, - "object": target.url - }, - } - activity.accept_follow( - target, follow, actor - ) + follow = factories['federation.Follow'](approved=None) + expected_accept = serializers.AcceptFollowSerializer(follow).data + activity.accept_follow(follow) deliver.assert_called_once_with( - expected_accept, to=[actor.url], on_behalf_of=target + expected_accept, to=[follow.actor.url], on_behalf_of=follow.target ) - follow_instance = actor.emitted_follows.first() - assert follow_instance.target == target diff --git a/api/tests/federation/test_actors.py b/api/tests/federation/test_actors.py index 090d9b03f..fe70cc6e5 100644 --- a/api/tests/federation/test_actors.py +++ b/api/tests/federation/test_actors.py @@ -7,6 +7,7 @@ from django.utils import timezone from rest_framework import exceptions +from funkwhale_api.federation import activity from funkwhale_api.federation import actors from funkwhale_api.federation import models from funkwhale_api.federation import serializers @@ -261,8 +262,6 @@ def test_test_actor_handles_follow( deliver = mocker.patch( 'funkwhale_api.federation.activity.deliver') actor = factories['federation.Actor']() - now = timezone.now() - mocker.patch('django.utils.timezone.now', return_value=now) accept_follow = mocker.patch( 'funkwhale_api.federation.activity.accept_follow') test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() @@ -272,28 +271,15 @@ def test_test_actor_handles_follow( 'id': 'http://test.federation/user#follows/267', 'object': test_actor.url, } - uid = uuid.uuid4() - mocker.patch('uuid.uuid4', return_value=uid) - expected_follow = { - '@context': serializers.AP_CONTEXT, - 'actor': test_actor.url, - 'id': test_actor.url + '#follows/{}'.format(uid), - 'object': actor.url, - 'type': 'Follow' - } - actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor) - accept_follow.assert_called_once_with( - test_actor, data, actor + follow = models.Follow.objects.get(target=test_actor, approved=True) + follow_back = models.Follow.objects.get(actor=test_actor, approved=None) + accept_follow.assert_called_once_with(follow) + deliver.assert_called_once_with( + serializers.FollowSerializer(follow_back).data, + on_behalf_of=test_actor, + to=[actor.url] ) - expected_calls = [ - mocker.call( - expected_follow, - to=[actor.url], - on_behalf_of=test_actor, - ) - ] - deliver.assert_has_calls(expected_calls) def test_test_actor_handles_undo_follow( @@ -346,12 +332,10 @@ def test_library_actor_handles_follow_manual_approval( } library_actor.system_conf.post_inbox(data, actor=actor) - fr = library_actor.received_follow_requests.first() + follow = library_actor.received_follows.first() - assert library_actor.received_follow_requests.count() == 1 - assert fr.target == library_actor - assert fr.actor == actor - assert fr.approved is None + assert follow.actor == actor + assert follow.approved is None def test_library_actor_handles_follow_auto_approval( @@ -369,10 +353,27 @@ def test_library_actor_handles_follow_auto_approval( } library_actor.system_conf.post_inbox(data, actor=actor) - assert library_actor.received_follow_requests.count() == 0 - accept_follow.assert_called_once_with( - library_actor, data, actor + follow = library_actor.received_follows.first() + + assert follow.actor == actor + assert follow.approved is True + + +def test_library_actor_handles_accept( + mocker, factories): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + actor = factories['federation.Actor']() + pending_follow = factories['federation.Follow']( + actor=library_actor, + target=actor, + approved=None, ) + serializer = serializers.AcceptFollowSerializer(pending_follow) + library_actor.system_conf.post_inbox(serializer.data, actor=actor) + + pending_follow.refresh_from_db() + + assert pending_follow.approved is True def test_library_actor_handle_create_audio_no_library(mocker, factories): diff --git a/api/tests/federation/test_models.py b/api/tests/federation/test_models.py index b17b6eb65..ae158e659 100644 --- a/api/tests/federation/test_models.py +++ b/api/tests/federation/test_models.py @@ -35,50 +35,6 @@ def test_follow_federation_url(factories): assert follow.get_federation_url() == expected -def test_follow_request_approve(mocker, factories): - uid = uuid.uuid4() - mocker.patch('uuid.uuid4', return_value=uid) - accept_follow = mocker.patch( - 'funkwhale_api.federation.activity.accept_follow') - fr = factories['federation.FollowRequest'](target__local=True) - fr.approve() - - follow = { - '@context': serializers.AP_CONTEXT, - 'actor': fr.actor.url, - 'id': fr.actor.url + '#follows/{}'.format(uid), - 'object': fr.target.url, - 'type': 'Follow' - } - - assert fr.approved is True - assert list(fr.target.followers.all()) == [fr.actor] - accept_follow.assert_called_once_with( - fr.target, follow, fr.actor - ) - - -def test_follow_request_approve_non_local(mocker, factories): - uid = uuid.uuid4() - mocker.patch('uuid.uuid4', return_value=uid) - accept_follow = mocker.patch( - 'funkwhale_api.federation.activity.accept_follow') - fr = factories['federation.FollowRequest']() - fr.approve() - - assert fr.approved is True - assert list(fr.target.followers.all()) == [fr.actor] - accept_follow.assert_not_called() - - -def test_follow_request_refused(mocker, factories): - fr = factories['federation.FollowRequest']() - fr.refuse() - - assert fr.approved is False - assert fr.target.followers.count() == 0 - - def test_library_model_unique_per_actor(factories): library = factories['federation.Library']() with pytest.raises(db.IntegrityError): diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index 7b7dda33c..e6eca0a42 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -1,4 +1,5 @@ import arrow +import pytest from django.urls import reverse from django.core.paginator import Paginator @@ -170,6 +171,184 @@ def test_follow_serializer_to_ap(factories): assert serializer.data == expected +def test_follow_serializer_save(factories): + actor = factories['federation.Actor']() + target = factories['federation.Actor']() + + data = expected = { + 'id': 'https://test.follow', + 'type': 'Follow', + 'actor': actor.url, + 'object': target.url, + } + serializer = serializers.FollowSerializer(data=data) + + assert serializer.is_valid(raise_exception=True) + + follow = serializer.save() + + assert follow.pk is not None + assert follow.actor == actor + assert follow.target == target + assert follow.approved is None + + +def test_follow_serializer_save_validates_on_context(factories): + actor = factories['federation.Actor']() + target = factories['federation.Actor']() + impostor = factories['federation.Actor']() + + data = expected = { + 'id': 'https://test.follow', + 'type': 'Follow', + 'actor': actor.url, + 'object': target.url, + } + serializer = serializers.FollowSerializer( + data=data, + context={'follow_actor': impostor, 'follow_target': impostor}) + + assert serializer.is_valid() is False + + assert 'actor' in serializer.errors + assert 'object' in serializer.errors + + +def test_accept_follow_serializer_representation(factories): + follow = factories['federation.Follow'](approved=None) + + expected = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/accept', + 'type': 'Accept', + 'actor': follow.target.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.AcceptFollowSerializer(follow) + + assert serializer.data == expected + + +def test_accept_follow_serializer_save(factories): + follow = factories['federation.Follow'](approved=None) + + data = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/accept', + 'type': 'Accept', + 'actor': follow.target.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.AcceptFollowSerializer(data=data) + assert serializer.is_valid(raise_exception=True) + serializer.save() + + follow.refresh_from_db() + + assert follow.approved is True + + +def test_accept_follow_serializer_validates_on_context(factories): + follow = factories['federation.Follow'](approved=None) + impostor = factories['federation.Actor']() + data = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/accept', + 'type': 'Accept', + 'actor': impostor.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.AcceptFollowSerializer( + data=data, + context={'follow_actor': impostor, 'follow_target': impostor}) + + assert serializer.is_valid() is False + assert 'actor' in serializer.errors['object'] + assert 'object' in serializer.errors['object'] + + +def test_undo_follow_serializer_representation(factories): + follow = factories['federation.Follow'](approved=True) + + expected = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/undo', + 'type': 'Undo', + 'actor': follow.actor.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.UndoFollowSerializer(follow) + + assert serializer.data == expected + + +def test_undo_follow_serializer_save(factories): + follow = factories['federation.Follow'](approved=True) + + data = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/undo', + 'type': 'Undo', + 'actor': follow.actor.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.UndoFollowSerializer(data=data) + assert serializer.is_valid(raise_exception=True) + serializer.save() + + with pytest.raises(models.Follow.DoesNotExist): + follow.refresh_from_db() + + +def test_undo_follow_serializer_validates_on_context(factories): + follow = factories['federation.Follow'](approved=True) + impostor = factories['federation.Actor']() + data = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/undo', + 'type': 'Undo', + 'actor': impostor.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.UndoFollowSerializer( + data=data, + context={'follow_actor': impostor, 'follow_target': impostor}) + + assert serializer.is_valid() is False + assert 'actor' in serializer.errors['object'] + assert 'object' in serializer.errors['object'] + + def test_paginated_collection_serializer(factories): tfs = factories['music.TrackFile'].create_batch(size=5) actor = factories['federation.Actor'](local=True) diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index 0b58e20f1..bd174f721 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -4,6 +4,7 @@ from django.core.paginator import Paginator import pytest from funkwhale_api.federation import actors +from funkwhale_api.federation import models from funkwhale_api.federation import serializers from funkwhale_api.federation import utils from funkwhale_api.federation import webfinger @@ -179,3 +180,35 @@ def test_can_scan_library(superuser_api_client, mocker): assert response.status_code == 200 assert response.data == result scan.assert_called_once_with('test@test.library') + + +def test_follow_library_manually(superuser_api_client, mocker, factories): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + actor = factories['federation.Actor'](manually_approves_followers=True) + follow = {'test': 'follow'} + deliver = mocker.patch( + 'funkwhale_api.federation.activity.deliver') + actor_get = mocker.patch( + 'funkwhale_api.federation.actors.get_actor', + return_value=actor) + library_get = mocker.patch( + 'funkwhale_api.federation.library.get_library_data', + return_value={}) + + url = reverse('api:v1:federation:libraries-list') + response = superuser_api_client.post( + url, {'actor_url': actor.url}) + + assert response.status_code == 201 + + follow = models.Follow.objects.get( + actor=library_actor, + target=actor, + approved=None, + ) + + deliver.assert_called_once_with( + serializers.FollowSerializer(follow).data, + on_behalf_of=library_actor, + to=[actor.url] + ) From 3caa03aedfe89656f7cbd9fafca8c10bd6490cc5 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Tue, 10 Apr 2018 22:47:13 +0200 Subject: [PATCH 18/42] use a dedicated serializer to handle library creation --- api/funkwhale_api/federation/actors.py | 2 +- ...410_1624.py => 0004_auto_20180410_2025.py} | 8 +- api/funkwhale_api/federation/models.py | 7 ++ api/funkwhale_api/federation/serializers.py | 96 +++++++++++++++++-- api/funkwhale_api/federation/views.py | 27 +----- api/tests/federation/test_serializers.py | 43 +++++++++ api/tests/federation/test_views.py | 45 ++++++--- 7 files changed, 182 insertions(+), 46 deletions(-) rename api/funkwhale_api/federation/migrations/{0004_auto_20180410_1624.py => 0004_auto_20180410_2025.py} (65%) diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 5a4e917bd..27a418c7d 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -170,7 +170,7 @@ class SystemActor(object): if not serializer.is_valid(raise_exception=True): return logger.info('Received invalid payload') - serializer.save() + return serializer.save() def handle_undo_follow(self, ac, sender): system_actor = self.get_actor_instance() diff --git a/api/funkwhale_api/federation/migrations/0004_auto_20180410_1624.py b/api/funkwhale_api/federation/migrations/0004_auto_20180410_2025.py similarity index 65% rename from api/funkwhale_api/federation/migrations/0004_auto_20180410_1624.py rename to api/funkwhale_api/federation/migrations/0004_auto_20180410_2025.py index b199706aa..bea4d14ae 100644 --- a/api/funkwhale_api/federation/migrations/0004_auto_20180410_1624.py +++ b/api/funkwhale_api/federation/migrations/0004_auto_20180410_2025.py @@ -1,6 +1,7 @@ -# Generated by Django 2.0.3 on 2018-04-10 16:24 +# Generated by Django 2.0.3 on 2018-04-10 20:25 from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): @@ -23,6 +24,11 @@ class Migration(migrations.Migration): name='approved', field=models.NullBooleanField(default=None), ), + migrations.AddField( + model_name='library', + name='follow', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='library', to='federation.Follow'), + ), migrations.DeleteModel( name='FollowRequest', ), diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index 201463066..7dc9c46e4 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -137,6 +137,13 @@ class Library(models.Model): # should we automatically import new files from this library? autoimport = models.BooleanField() tracks_count = models.PositiveIntegerField(null=True, blank=True) + follow = models.OneToOneField( + Follow, + related_name='library', + null=True, + blank=True, + on_delete=models.SET_NULL, + ) class LibraryTrack(models.Model): diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index f0d1e35fd..68eef1355 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -8,7 +8,7 @@ from django.db import transaction from rest_framework import serializers from dynamic_preferences.registries import global_preferences_registry -from funkwhale_api.common.utils import set_query_parameter +from funkwhale_api.common import utils as funkwhale_utils from . import activity from . import models @@ -121,6 +121,66 @@ class LibraryActorSerializer(ActorSerializer): return validated_data +class APILibraryCreateSerializer(serializers.ModelSerializer): + actor = serializers.URLField() + + class Meta: + model = models.Library + fields = [ + 'actor', + 'autoimport', + 'federation_enabled', + 'download_files', + ] + + def validate(self, validated_data): + from . import actors + from . import library + + actor_url = validated_data['actor'] + actor_data = actors.get_actor_data(actor_url) + acs = LibraryActorSerializer(data=actor_data) + acs.is_valid(raise_exception=True) + try: + actor = models.Actor.objects.get(url=actor_url) + except models.Actor.DoesNotExist: + actor = acs.save() + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + validated_data['follow'] = models.Follow.objects.get_or_create( + actor=library_actor, + target=actor, + )[0] + if validated_data['follow'].approved is None: + funkwhale_utils.on_commit( + activity.deliver, + FollowSerializer(validated_data['follow']).data, + on_behalf_of=validated_data['follow'].actor, + to=[validated_data['follow'].target.url], + ) + + library_data = library.get_library_data( + acs.validated_data['library_url']) + if 'errors' in library_data: + raise serializers.ValidationError(str(library_data['errors'])) + validated_data['library'] = library_data + validated_data['actor'] = actor + return validated_data + + def create(self, validated_data): + library = models.Library.objects.get_or_create( + url=validated_data['library']['id'], + defaults={ + 'actor': validated_data['actor'], + 'follow': validated_data['follow'], + 'tracks_count': validated_data['library']['totalItems'], + 'federation_enabled': validated_data['federation_enabled'], + 'autoimport': validated_data['autoimport'], + 'download_files': validated_data['download_files'], + } + )[0] + return library + + class FollowSerializer(serializers.Serializer): id = serializers.URLField() object = serializers.URLField() @@ -163,6 +223,20 @@ class FollowSerializer(serializers.Serializer): return ret +class APIFollowSerializer(serializers.ModelSerializer): + class Meta: + model = models.Follow + fields = [ + 'uuid', + 'id', + 'approved', + 'creation_date', + 'modification_date', + 'actor', + 'target', + ] + + class AcceptFollowSerializer(serializers.Serializer): id = serializers.URLField() actor = serializers.URLField() @@ -244,7 +318,7 @@ class UndoFollowSerializer(serializers.Serializer): } def save(self): - self.validated_data['follow'].delete() + return self.validated_data['follow'].delete() class ActorWebfingerSerializer(serializers.Serializer): @@ -365,9 +439,10 @@ class PaginatedCollectionSerializer(serializers.Serializer): conf['items'], conf.get('page_size', 20) ) - first = set_query_parameter(conf['id'], page=1) + first = funkwhale_utils.set_query_parameter(conf['id'], page=1) current = first - last = set_query_parameter(conf['id'], page=paginator.num_pages) + last = funkwhale_utils.set_query_parameter( + conf['id'], page=paginator.num_pages) d = { 'id': conf['id'], 'actor': conf['actor'].url, @@ -394,9 +469,12 @@ class CollectionPageSerializer(serializers.Serializer): def to_representation(self, conf): page = conf['page'] - first = set_query_parameter(conf['id'], page=1) - last = set_query_parameter(conf['id'], page=page.paginator.num_pages) - id = set_query_parameter(conf['id'], page=page.number) + first = funkwhale_utils.set_query_parameter( + conf['id'], page=1) + last = funkwhale_utils.set_query_parameter( + conf['id'], page=page.paginator.num_pages) + id = funkwhale_utils.set_query_parameter( + conf['id'], page=page.number) d = { 'id': id, 'partOf': conf['id'], @@ -417,11 +495,11 @@ class CollectionPageSerializer(serializers.Serializer): } if page.has_previous(): - d['prev'] = set_query_parameter( + d['prev'] = funkwhale_utils.set_query_parameter( conf['id'], page=page.previous_page_number()) if page.has_next(): - d['next'] = set_query_parameter( + d['next'] = funkwhale_utils.set_query_parameter( conf['id'], page=page.next_page_number()) if self.context.get('include_ap_context', True): diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 2d4220472..623fd574f 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -179,26 +179,7 @@ class LibraryViewSet(viewsets.GenericViewSet): @transaction.atomic def create(self, request, *args, **kwargs): - try: - actor_url = request.data['actor_url'] - except KeyError: - raise ValidationError('Missing actor_url') - - try: - actor = actors.get_actor(actor_url) - library_data = library.get_library_data(actor.url) - except Exception as e: - raise ValidationError('Error while fetching actor and library') - - library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() - follow, created = models.Follow.objects.get_or_create( - actor=library_actor, - target=actor, - ) - serializer = serializers.FollowSerializer(follow) - activity.deliver( - serializer.data, - on_behalf_of=library_actor, - to=[actor.url] - ) - return response.Response({}, status=201) + serializer = serializers.APILibraryCreateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + library = serializer.save() + return response.Response(serializer.data, status=201) diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index e6eca0a42..8086a0059 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -619,3 +619,46 @@ def test_collection_serializer_to_ap(factories): collection, context={'actor': library, 'id': 'https://test.id'}) assert serializer.data == expected + + +def test_api_library_create_serializer_save(factories, r_mock): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + actor = factories['federation.Actor']() + follow = factories['federation.Follow']( + target=actor, + actor=library_actor, + ) + actor_data = serializers.ActorSerializer(actor).data + actor_data['url'] = [{ + 'href': 'https://test.library', + 'name': 'library', + 'type': 'Link', + }] + library_conf = { + 'id': 'https://test.library', + 'items': range(10), + 'actor': actor, + 'page_size': 5, + } + library_data = serializers.PaginatedCollectionSerializer(library_conf).data + r_mock.get(actor.url, json=actor_data) + r_mock.get('https://test.library', json=library_data) + data = { + 'actor': actor.url, + 'autoimport': False, + 'federation_enabled': True, + 'download_files': False, + } + + serializer = serializers.APILibraryCreateSerializer(data=data) + assert serializer.is_valid(raise_exception=True) is True + library = serializer.save() + follow = models.Follow.objects.get( + target=actor, actor=library_actor, approved=None) + + assert library.autoimport is data['autoimport'] + assert library.federation_enabled is data['federation_enabled'] + assert library.download_files is data['download_files'] + assert library.tracks_count == 10 + assert library.actor == actor + assert library.follow == follow diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index bd174f721..99d425661 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -4,6 +4,7 @@ from django.core.paginator import Paginator import pytest from funkwhale_api.federation import actors +from funkwhale_api.federation import activity from funkwhale_api.federation import models from funkwhale_api.federation import serializers from funkwhale_api.federation import utils @@ -182,22 +183,37 @@ def test_can_scan_library(superuser_api_client, mocker): scan.assert_called_once_with('test@test.library') -def test_follow_library_manually(superuser_api_client, mocker, factories): +def test_follow_library(superuser_api_client, mocker, factories, r_mock): library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() - actor = factories['federation.Actor'](manually_approves_followers=True) + actor = factories['federation.Actor']() follow = {'test': 'follow'} - deliver = mocker.patch( - 'funkwhale_api.federation.activity.deliver') - actor_get = mocker.patch( - 'funkwhale_api.federation.actors.get_actor', - return_value=actor) - library_get = mocker.patch( - 'funkwhale_api.federation.library.get_library_data', - return_value={}) + on_commit = mocker.patch( + 'funkwhale_api.common.utils.on_commit') + actor_data = serializers.ActorSerializer(actor).data + actor_data['url'] = [{ + 'href': 'https://test.library', + 'name': 'library', + 'type': 'Link', + }] + library_conf = { + 'id': 'https://test.library', + 'items': range(10), + 'actor': actor, + 'page_size': 5, + } + library_data = serializers.PaginatedCollectionSerializer(library_conf).data + r_mock.get(actor.url, json=actor_data) + r_mock.get('https://test.library', json=library_data) + data = { + 'actor': actor.url, + 'autoimport': False, + 'federation_enabled': True, + 'download_files': False, + } url = reverse('api:v1:federation:libraries-list') response = superuser_api_client.post( - url, {'actor_url': actor.url}) + url, data) assert response.status_code == 201 @@ -206,8 +222,13 @@ def test_follow_library_manually(superuser_api_client, mocker, factories): target=actor, approved=None, ) + library = follow.library - deliver.assert_called_once_with( + assert response.data == serializers.APILibraryCreateSerializer( + library).data + + on_commit.assert_called_once_with( + activity.deliver, serializers.FollowSerializer(follow).data, on_behalf_of=library_actor, to=[actor.url] From e82a53da3570128d493d85c13762dd8493f7a975 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Tue, 10 Apr 2018 23:17:51 +0200 Subject: [PATCH 19/42] Added API endpoints to list library followees and followers --- api/funkwhale_api/federation/filters.py | 19 +++++++++++ api/funkwhale_api/federation/serializers.py | 19 +++++++++++ api/funkwhale_api/federation/views.py | 35 +++++++++++++++++++++ api/tests/federation/test_views.py | 28 +++++++++++++++++ 4 files changed, 101 insertions(+) create mode 100644 api/funkwhale_api/federation/filters.py diff --git a/api/funkwhale_api/federation/filters.py b/api/funkwhale_api/federation/filters.py new file mode 100644 index 000000000..a43c2dc0c --- /dev/null +++ b/api/funkwhale_api/federation/filters.py @@ -0,0 +1,19 @@ +import django_filters + +from . import models + + +class FollowFilter(django_filters.FilterSet): + ordering = django_filters.OrderingFilter( + # tuple-mapping retains order + fields=( + ('creation_date', 'creation_date'), + ('modification_date', 'modification_date'), + ), + ) + + class Meta: + model = models.Follow + fields = { + 'approved': ['exact'], + } diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 68eef1355..0fe52e1f5 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -96,6 +96,22 @@ class ActorSerializer(serializers.ModelSerializer): return value[:500] +class APIActorSerializer(serializers.ModelSerializer): + class Meta: + model = models.Actor + fields = [ + 'id', + 'url', + 'creation_date', + 'summary', + 'preferred_username', + 'name', + 'last_fetch_date', + 'domain', + 'type', + 'manually_approves_followers', + + ] class LibraryActorSerializer(ActorSerializer): url = serializers.ListField( child=serializers.JSONField()) @@ -224,6 +240,9 @@ class FollowSerializer(serializers.Serializer): class APIFollowSerializer(serializers.ModelSerializer): + actor = APIActorSerializer() + target = APIActorSerializer() + class Meta: model = models.Follow fields = [ diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 623fd574f..5b11942f2 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -18,6 +18,7 @@ from funkwhale_api.music.models import TrackFile from . import activity from . import actors from . import authentication +from . import filters from . import library from . import models from . import permissions @@ -177,6 +178,40 @@ class LibraryViewSet(viewsets.GenericViewSet): data = library.scan_from_account_name(account) return response.Response(data) + @list_route(methods=['get']) + def following(self, request, *args, **kwargs): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + queryset = models.Follow.objects.filter( + actor=library_actor + ).select_related( + 'target', + 'target', + ).order_by('-creation_date') + filterset = filters.FollowFilter(request.GET, queryset=queryset) + serializer = serializers.APIFollowSerializer(filterset.qs, many=True) + data = { + 'results': serializer.data, + 'count': len(filterset.qs), + } + return response.Response(data) + + @list_route(methods=['get']) + def followers(self, request, *args, **kwargs): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + queryset = models.Follow.objects.filter( + target=library_actor + ).select_related( + 'target', + 'target', + ).order_by('-creation_date') + filterset = filters.FollowFilter(request.GET, queryset=queryset) + serializer = serializers.APIFollowSerializer(filterset.qs, many=True) + data = { + 'results': serializer.data, + 'count': len(filterset.qs), + } + return response.Response(data) + @transaction.atomic def create(self, request, *args, **kwargs): serializer = serializers.APILibraryCreateSerializer(data=request.data) diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index 99d425661..3bc9fa88f 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -233,3 +233,31 @@ def test_follow_library(superuser_api_client, mocker, factories, r_mock): on_behalf_of=library_actor, to=[actor.url] ) + + +def test_can_list_system_actor_following(factories, superuser_api_client): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow1 = factories['federation.Follow'](actor=library_actor) + follow2 = factories['federation.Follow']() + + url = reverse('api:v1:federation:libraries-following') + response = superuser_api_client.get(url) + + assert response.status_code == 200 + assert response.data['results'] == [ + serializers.APIFollowSerializer(follow1).data + ] + + +def test_can_list_system_actor_followers(factories, superuser_api_client): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow1 = factories['federation.Follow'](actor=library_actor) + follow2 = factories['federation.Follow'](target=library_actor) + + url = reverse('api:v1:federation:libraries-followers') + response = superuser_api_client.get(url) + + assert response.status_code == 200 + assert response.data['results'] == [ + serializers.APIFollowSerializer(follow2).data + ] From fe7ca088c58fcf43911e3c102fef0454b820596e Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Tue, 10 Apr 2018 23:34:57 +0200 Subject: [PATCH 20/42] Library list endpoint --- api/funkwhale_api/federation/filters.py | 14 ++++++++++++++ api/funkwhale_api/federation/serializers.py | 20 ++++++++++++++++++++ api/funkwhale_api/federation/views.py | 18 ++++++++++++++++-- api/tests/federation/test_views.py | 14 ++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/api/funkwhale_api/federation/filters.py b/api/funkwhale_api/federation/filters.py index a43c2dc0c..d76fe7ad0 100644 --- a/api/funkwhale_api/federation/filters.py +++ b/api/funkwhale_api/federation/filters.py @@ -3,6 +3,20 @@ import django_filters from . import models +class LibraryFilter(django_filters.FilterSet): + approved = django_filters.BooleanFilter('following__approved') + + class Meta: + model = models.Library + fields = { + 'approved': ['exact'], + 'federation_enabled': ['exact'], + 'download_files': ['exact'], + 'autoimport': ['exact'], + 'tracks_count': ['exact'], + } + + class FollowFilter(django_filters.FilterSet): ordering = django_filters.OrderingFilter( # tuple-mapping retains order diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 0fe52e1f5..cf59eaa63 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -137,6 +137,26 @@ class LibraryActorSerializer(ActorSerializer): return validated_data +class APILibrarySerializer(serializers.ModelSerializer): + actor = APIActorSerializer + + class Meta: + model = models.Library + fields = [ + 'actor', + 'autoimport', + 'federation_enabled', + 'download_files', + 'tracks_count', + 'url', + 'uuid', + 'creation_date', + 'follow', + 'fetched_date', + 'modification_date', + ] + + class APILibraryCreateSerializer(serializers.ModelSerializer): actor = serializers.URLField() diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 5b11942f2..bceac4243 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -5,6 +5,7 @@ from django.db import transaction from django.http import HttpResponse from django.urls import reverse +from rest_framework import mixins from rest_framework import permissions as rest_permissions from rest_framework import response from rest_framework import views @@ -164,9 +165,22 @@ class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet): return response.Response(data) -class LibraryViewSet(viewsets.GenericViewSet): +class LibraryViewSet( + mixins.ListModelMixin, + viewsets.GenericViewSet): permission_classes = [rest_permissions.DjangoModelPermissions] - queryset = models.Library.objects.all() + queryset = models.Library.objects.all().select_related( + 'actor', + 'follow', + ) + filter_class = filters.LibraryFilter + serializer_class = serializers.APILibrarySerializer + ordering_fields = ( + 'id', + 'creation_date', + 'fetched_date', + 'actor__domain', + ) @list_route(methods=['get']) def scan(self, request, *args, **kwargs): diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index 3bc9fa88f..a2786be09 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -261,3 +261,17 @@ def test_can_list_system_actor_followers(factories, superuser_api_client): assert response.data['results'] == [ serializers.APIFollowSerializer(follow2).data ] + + +def test_can_list_libraries(factories, superuser_api_client): + library1 = factories['federation.Library']() + library2 = factories['federation.Library']() + + url = reverse('api:v1:federation:libraries-list') + response = superuser_api_client.get(url) + + assert response.status_code == 200 + assert response.data['results'] == [ + serializers.APILibrarySerializer(library1).data, + serializers.APILibrarySerializer(library2).data, + ] From f4f75dcb4f4c6f04d60f0f2c0c14cfe140ccf5e1 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Wed, 11 Apr 2018 20:40:47 +0200 Subject: [PATCH 21/42] Can now scan and follow library from front-end --- api/funkwhale_api/federation/serializers.py | 1 + api/funkwhale_api/users/models.py | 3 + front/src/components/Sidebar.vue | 3 + .../src/components/federation/LibraryCard.vue | 82 +++++++++++++ .../src/components/federation/LibraryForm.vue | 110 ++++++++++++++++++ front/src/router/index.js | 5 + front/src/views/federation/Home.vue | 40 +++++++ 7 files changed, 244 insertions(+) create mode 100644 front/src/components/federation/LibraryCard.vue create mode 100644 front/src/components/federation/LibraryForm.vue create mode 100644 front/src/views/federation/Home.vue diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index cf59eaa63..bca35902d 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -159,6 +159,7 @@ class APILibrarySerializer(serializers.ModelSerializer): class APILibraryCreateSerializer(serializers.ModelSerializer): actor = serializers.URLField() + federation_enabled = serializers.BooleanField() class Meta: model = models.Library diff --git a/api/funkwhale_api/users/models.py b/api/funkwhale_api/users/models.py index 9516c108f..572fa9ddc 100644 --- a/api/funkwhale_api/users/models.py +++ b/api/funkwhale_api/users/models.py @@ -31,6 +31,9 @@ class User(AbstractUser): 'dynamic_preferences.change_globalpreferencemodel': { 'external_codename': 'settings.change', }, + 'federation.change_library': { + 'external_codename': 'federation.manage', + }, } privacy_level = fields.get_privacy_field() diff --git a/front/src/components/Sidebar.vue b/front/src/components/Sidebar.vue index 42a923b6b..c04ebe5a8 100644 --- a/front/src/components/Sidebar.vue +++ b/front/src/components/Sidebar.vue @@ -45,6 +45,9 @@ Activity + Federation diff --git a/front/src/components/federation/LibraryCard.vue b/front/src/components/federation/LibraryCard.vue new file mode 100644 index 000000000..9676f2de5 --- /dev/null +++ b/front/src/components/federation/LibraryCard.vue @@ -0,0 +1,82 @@ + + + diff --git a/front/src/components/federation/LibraryForm.vue b/front/src/components/federation/LibraryForm.vue new file mode 100644 index 000000000..5cf6dabb2 --- /dev/null +++ b/front/src/components/federation/LibraryForm.vue @@ -0,0 +1,110 @@ + + + diff --git a/front/src/router/index.js b/front/src/router/index.js index d41764227..0981c37f9 100644 --- a/front/src/router/index.js +++ b/front/src/router/index.js @@ -25,6 +25,7 @@ import RequestsList from '@/components/requests/RequestsList' import PlaylistDetail from '@/views/playlists/Detail' import PlaylistList from '@/views/playlists/List' import Favorites from '@/components/favorites/List' +import Federation from '@/views/federation/Home' Vue.use(Router) @@ -83,6 +84,10 @@ export default new Router({ defaultPaginateBy: route.query.paginateBy }) }, + { + path: '/manage/federation', + component: Federation + }, { path: '/library', component: Library, diff --git a/front/src/views/federation/Home.vue b/front/src/views/federation/Home.vue new file mode 100644 index 000000000..c9e3693d6 --- /dev/null +++ b/front/src/views/federation/Home.vue @@ -0,0 +1,40 @@ + + + From 472cc7e26a231e06ecb00f3666a4d9a08f863bef Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Wed, 11 Apr 2018 21:58:41 +0200 Subject: [PATCH 22/42] Detail library view with settings update --- api/funkwhale_api/federation/serializers.py | 37 ++++- api/funkwhale_api/federation/views.py | 3 + api/tests/federation/test_views.py | 31 ++++ front/src/router/index.js | 10 +- front/src/views/federation/Base.vue | 7 + front/src/views/federation/Home.vue | 2 +- front/src/views/federation/LibraryDetail.vue | 149 +++++++++++++++++++ 7 files changed, 228 insertions(+), 11 deletions(-) create mode 100644 front/src/views/federation/Base.vue create mode 100644 front/src/views/federation/LibraryDetail.vue diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index bca35902d..c717e679b 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -112,6 +112,8 @@ class APIActorSerializer(serializers.ModelSerializer): 'manually_approves_followers', ] + + class LibraryActorSerializer(ActorSerializer): url = serializers.ListField( child=serializers.JSONField()) @@ -137,33 +139,52 @@ class LibraryActorSerializer(ActorSerializer): return validated_data +class APIFollowSerializer(serializers.ModelSerializer): + class Meta: + model = models.Follow + fields = [ + 'uuid', + 'actor', + 'target', + 'approved', + 'creation_date', + 'modification_date', + ] + + class APILibrarySerializer(serializers.ModelSerializer): - actor = APIActorSerializer + actor = APIActorSerializer() + follow = APIFollowSerializer() class Meta: model = models.Library - fields = [ + + read_only_fields = [ 'actor', - 'autoimport', - 'federation_enabled', - 'download_files', - 'tracks_count', - 'url', 'uuid', - 'creation_date', + 'url', + 'tracks_count', 'follow', 'fetched_date', 'modification_date', + 'creation_date', ] + fields = [ + 'autoimport', + 'federation_enabled', + 'download_files', + ] + read_only_fields class APILibraryCreateSerializer(serializers.ModelSerializer): actor = serializers.URLField() federation_enabled = serializers.BooleanField() + uuid = serializers.UUIDField(read_only=True) class Meta: model = models.Library fields = [ + 'uuid', 'actor', 'autoimport', 'federation_enabled', diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index bceac4243..cac6c4163 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -166,6 +166,8 @@ class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet): class LibraryViewSet( + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, mixins.ListModelMixin, viewsets.GenericViewSet): permission_classes = [rest_permissions.DjangoModelPermissions] @@ -173,6 +175,7 @@ class LibraryViewSet( 'actor', 'follow', ) + lookup_field = 'uuid' filter_class = filters.LibraryFilter serializer_class = serializers.APILibrarySerializer ordering_fields = ( diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index a2786be09..72feaabfd 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -275,3 +275,34 @@ def test_can_list_libraries(factories, superuser_api_client): serializers.APILibrarySerializer(library1).data, serializers.APILibrarySerializer(library2).data, ] + + +def test_can_detail_library(factories, superuser_api_client): + library = factories['federation.Library']() + + url = reverse( + 'api:v1:federation:libraries-detail', + kwargs={'uuid': str(library.uuid)}) + response = superuser_api_client.get(url) + + assert response.status_code == 200 + assert response.data == serializers.APILibrarySerializer(library).data + + +def test_can_patch_library(factories, superuser_api_client): + library = factories['federation.Library']() + data = { + 'federation_enabled': not library.federation_enabled, + 'download_files': not library.download_files, + 'autoimport': not library.autoimport, + } + url = reverse( + 'api:v1:federation:libraries-detail', + kwargs={'uuid': str(library.uuid)}) + response = superuser_api_client.patch(url, data) + + assert response.status_code == 200 + library.refresh_from_db() + + for k, v in data.items(): + assert getattr(library, k) == v diff --git a/front/src/router/index.js b/front/src/router/index.js index 0981c37f9..394a12849 100644 --- a/front/src/router/index.js +++ b/front/src/router/index.js @@ -25,7 +25,9 @@ import RequestsList from '@/components/requests/RequestsList' import PlaylistDetail from '@/views/playlists/Detail' import PlaylistList from '@/views/playlists/List' import Favorites from '@/components/favorites/List' -import Federation from '@/views/federation/Home' +import FederationBase from '@/views/federation/Base' +import FederationHome from '@/views/federation/Home' +import FederationLibraryDetail from '@/views/federation/LibraryDetail' Vue.use(Router) @@ -86,7 +88,11 @@ export default new Router({ }, { path: '/manage/federation', - component: Federation + component: FederationBase, + children: [ + { path: '', component: FederationHome }, + { path: 'library/:id', name: 'federation.libraries.detail', component: FederationLibraryDetail, props: true } + ] }, { path: '/library', diff --git a/front/src/views/federation/Base.vue b/front/src/views/federation/Base.vue new file mode 100644 index 000000000..6add8e5e4 --- /dev/null +++ b/front/src/views/federation/Base.vue @@ -0,0 +1,7 @@ + diff --git a/front/src/views/federation/Home.vue b/front/src/views/federation/Home.vue index c9e3693d6..89048aac5 100644 --- a/front/src/views/federation/Home.vue +++ b/front/src/views/federation/Home.vue @@ -1,5 +1,5 @@