From 520fb9d078011c89d2ff4d1ad6f45f990efc2db7 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Wed, 11 Apr 2018 23:13:33 +0200 Subject: [PATCH] Started work on library scanning --- api/funkwhale_api/federation/library.py | 21 +++++++ api/funkwhale_api/federation/models.py | 4 +- api/funkwhale_api/federation/serializers.py | 16 ++++- api/funkwhale_api/federation/tasks.py | 27 ++++++++ api/funkwhale_api/radios/models.py | 3 +- api/tests/federation/test_serializers.py | 19 ++++++ api/tests/federation/test_tasks.py | 61 +++++++++++++++++++ .../src/components/federation/LibraryForm.vue | 2 +- front/src/views/federation/LibraryDetail.vue | 8 +++ 9 files changed, 157 insertions(+), 4 deletions(-) create mode 100644 api/funkwhale_api/federation/tasks.py create mode 100644 api/tests/federation/test_tasks.py diff --git a/api/funkwhale_api/federation/library.py b/api/funkwhale_api/federation/library.py index 177f14754..d7c2f817c 100644 --- a/api/funkwhale_api/federation/library.py +++ b/api/funkwhale_api/federation/library.py @@ -144,3 +144,24 @@ def get_library_data(library_url): } return serializer.validated_data + + +def get_library_page(library, page_url): + actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + auth = signing.get_auth(actor.private_key, actor.private_key_id) + response = session.get_session().get( + page_url, + auth=auth, + timeout=5, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, + headers={ + 'Content-Type': 'application/activity+json' + } + ) + serializer = serializers.CollectionPageSerializer( + data=response.json(), + context={ + 'library': library, + 'item_serializer': serializers.AudioSerializer}) + serializer.is_valid(raise_exception=True) + return serializer.validated_data diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index 7dc9c46e4..76dbfd1ad 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -2,6 +2,7 @@ import uuid from django.conf import settings from django.contrib.postgres.fields import JSONField +from django.core.serializers.json import DjangoJSONEncoder from django.db import models from django.utils import timezone @@ -160,4 +161,5 @@ class LibraryTrack(models.Model): artist_name = models.CharField(max_length=500) album_title = models.CharField(max_length=500) title = models.CharField(max_length=500) - metadata = JSONField(default={}, max_length=10000) + metadata = JSONField( + default={}, max_length=10000, encoder=DjangoJSONEncoder) diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index c717e679b..42054c7c4 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -494,6 +494,8 @@ class PaginatedCollectionSerializer(serializers.Serializer): totalItems = serializers.IntegerField(min_value=0) actor = serializers.URLField() id = serializers.URLField() + first = serializers.URLField() + last = serializers.URLField() def to_representation(self, conf): paginator = Paginator( @@ -524,10 +526,22 @@ class CollectionPageSerializer(serializers.Serializer): items = serializers.ListField() actor = serializers.URLField() id = serializers.URLField() - prev = serializers.URLField(required=False) + first = serializers.URLField() + last = serializers.URLField() next = serializers.URLField(required=False) + prev = serializers.URLField(required=False) partOf = serializers.URLField() + def validate_items(self, v): + item_serializer = self.context.get('item_serializer') + if not item_serializer: + return v + raw_items = [item_serializer(data=i, context=self.context) for i in v] + for i in raw_items: + i.is_valid(raise_exception=True) + + return raw_items + def to_representation(self, conf): page = conf['page'] first = funkwhale_utils.set_query_parameter( diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py new file mode 100644 index 000000000..fd03d3290 --- /dev/null +++ b/api/funkwhale_api/federation/tasks.py @@ -0,0 +1,27 @@ +from funkwhale_api.taskapp import celery + +from . import library as lb +from . import models + + +@celery.app.task(name='federation.scan_library') +@celery.require_instance(models.Library, 'library') +def scan_library(library): + if not library.federation_enabled: + return + + data = lb.get_library_data(library.url) + scan_library_page.delay( + library_id=library.id, page_url=data['first']) + + +@celery.app.task(name='federation.scan_library_page') +@celery.require_instance(models.Library, 'library') +def scan_library_page(library, page_url): + if not library.federation_enabled: + return + + data = lb.get_library_page(library, page_url) + lts = [] + for item_serializer in data['items']: + lts.append(item_serializer.save()) diff --git a/api/funkwhale_api/radios/models.py b/api/funkwhale_api/radios/models.py index d9c12534c..0273b5387 100644 --- a/api/funkwhale_api/radios/models.py +++ b/api/funkwhale_api/radios/models.py @@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError from django.contrib.postgres.fields import JSONField from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType +from django.core.serializers.json import DjangoJSONEncoder from funkwhale_api.music.models import Track @@ -23,7 +24,7 @@ class Radio(models.Model): creation_date = models.DateTimeField(default=timezone.now) is_public = models.BooleanField(default=False) version = models.PositiveIntegerField(default=0) - config = JSONField() + config = JSONField(encoder=DjangoJSONEncoder) def get_candidates(self): return filters.run(self.config) diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index 8086a0059..6d33a529d 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -386,6 +386,8 @@ def test_paginated_collection_serializer_validation(): 'id': 'https://test.federation/test', 'totalItems': 5, 'actor': 'http://test.actor', + 'first': 'https://test.federation/test?page=1', + 'last': 'https://test.federation/test?page=1', 'items': [] } @@ -407,6 +409,8 @@ def test_collection_page_serializer_validation(): 'totalItems': 5, 'actor': 'https://test.actor', 'items': [], + 'first': 'https://test.federation/test?page=1', + 'last': 'https://test.federation/test?page=3', 'prev': base + '?page=1', 'next': base + '?page=3', 'partOf': base, @@ -426,6 +430,21 @@ def test_collection_page_serializer_validation(): assert serializer.validated_data['partOf'] == data['partOf'] +def test_collection_page_serializer_can_validate_child(): + base = 'https://test.federation/test' + data = { + 'items': [{'in': 'valid'}], + } + + serializer = serializers.CollectionPageSerializer( + data=data, + context={'item_serializer': serializers.AudioSerializer} + ) + + assert serializer.is_valid() is False + assert 'items' in serializer.errors + + def test_collection_page_serializer(factories): tfs = factories['music.TrackFile'].create_batch(size=5) actor = factories['federation.Actor'](local=True) diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py new file mode 100644 index 000000000..85684c1a3 --- /dev/null +++ b/api/tests/federation/test_tasks.py @@ -0,0 +1,61 @@ +from django.core.paginator import Paginator + +from funkwhale_api.federation import serializers +from funkwhale_api.federation import tasks + + +def test_scan_library_does_nothing_if_federation_disabled(mocker, factories): + library = factories['federation.Library'](federation_enabled=False) + tasks.scan_library(library_id=library.pk) + + assert library.tracks.count() == 0 + + +def test_scan_library_page_does_nothing_if_federation_disabled( + mocker, factories): + library = factories['federation.Library'](federation_enabled=False) + tasks.scan_library_page(library_id=library.pk, page_url=None) + + assert library.tracks.count() == 0 + + +def test_scan_library_fetches_page_and_calls_scan_page( + mocker, factories, r_mock): + library = factories['federation.Library'](federation_enabled=True) + collection_conf = { + 'actor': library.actor, + 'id': library.url, + 'page_size': 10, + 'items': range(10), + } + collection = serializers.PaginatedCollectionSerializer(collection_conf) + scan_page = mocker.patch( + 'funkwhale_api.federation.tasks.scan_library_page.delay') + r_mock.get(collection_conf['id'], json=collection.data) + tasks.scan_library(library_id=library.pk) + + scan_page.assert_called_once_with( + library_id=library.id, + page_url=collection.data['first'], + ) + + +def test_scan_page_fetches_page_and_creates_tracks( + mocker, factories, r_mock): + library = factories['federation.Library'](federation_enabled=True) + tfs = factories['music.TrackFile'].create_batch(size=5) + page_conf = { + 'actor': library.actor, + 'id': library.url, + 'page': Paginator(tfs, 5).page(1), + 'item_serializer': serializers.AudioSerializer, + } + page = serializers.CollectionPageSerializer(page_conf) + #scan_page = mocker.patch( + # 'funkwhale_api.federation.tasks.scan_library_page.delay') + r_mock.get(page.data['id'], json=page.data) + + tasks.scan_library_page(library_id=library.pk, page_url=page.data['id']) + + lts = list(library.tracks.all().order_by('-published_date')) + assert len(lts) == 5 diff --git a/front/src/components/federation/LibraryForm.vue b/front/src/components/federation/LibraryForm.vue index 5cf6dabb2..5da46dc17 100644 --- a/front/src/components/federation/LibraryForm.vue +++ b/front/src/components/federation/LibraryForm.vue @@ -43,7 +43,7 @@ export default { data () { return { isLoading: false, - libraryUsername: 'library@node2.funkwhale.test', + libraryUsername: '', result: null, errors: [] } diff --git a/front/src/views/federation/LibraryDetail.vue b/front/src/views/federation/LibraryDetail.vue index d2430bda5..d33fcc212 100644 --- a/front/src/views/federation/LibraryDetail.vue +++ b/front/src/views/federation/LibraryDetail.vue @@ -77,6 +77,14 @@ + + Last fetched + + + + + +