MirrorMirror
/
funkwhale
mirror of https://dev.funkwhale.audio/funkwhale/funkwhale.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implemented followers notification on import and autoimport

This commit is contained in:
Eliot Berriot 2018-04-16 21:59:13 +02:00
parent adcbe8852d
commit f31874edf5
No known key found for this signature in database
GPG Key ID: DD6965E2476E5C27
10 changed files with 253 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
api/funkwhale_api/federation/actors.py
View File

@ -12,6 +12,9 @@ from rest_framework.exceptions import PermissionDenied
from dynamic_preferences.registries import global_preferences_registry
from funkwhale_api.common import session
from funkwhale_api.common import utils as funkwhale_utils
from funkwhale_api.music import models as music_models
from funkwhale_api.music import tasks as music_tasks
from . import activity
from . import keys
@ -243,7 +246,7 @@ class LibraryActor(SystemActor):
data=i, context={'library': remote_library})
for i in items
]
now = timezone.now()
valid_serializers = []
for s in item_serializers:
if s.is_valid():
@ -252,8 +255,30 @@ class LibraryActor(SystemActor):
logger.debug(
'Skipping invalid item %s, %s', s.initial_data, s.errors)
lts = []
for s in valid_serializers:
s.save()
lts.append(s.save())
if remote_library.autoimport:
batch = music_models.ImportBatch.objects.create(
source='federation',
)
for lt in lts:
if lt.creation_date < now:
# track was already in the library, we do not trigger
# an import
continue
job = music_models.ImportJob.objects.create(
batch=batch,
library_track=lt,
mbid=lt.mbid,
source=lt.url,
)
funkwhale_utils.on_commit(
music_tasks.import_job_run.delay,
import_job_id=job.pk,
use_acoustid=False,
)
class TestActor(SystemActor):

5
api/funkwhale_api/federation/models.py
View File

@ -97,6 +97,11 @@ class Actor(models.Model):
if self.is_system:
return actors.SYSTEM_ACTORS[self.preferred_username]
def get_approved_followers(self):
follows = self.received_follows.filter(approved=True)
return self.followers.filter(
pk__in=follows.values_list('actor', flat=True))
class Follow(models.Model):
ap_type = 'Follow'

10
api/funkwhale_api/federation/serializers.py
View File

@ -493,7 +493,7 @@ class ActorWebfingerSerializer(serializers.Serializer):
class ActivitySerializer(serializers.Serializer):
actor = serializers.URLField()
id = serializers.URLField()
id = serializers.URLField(required=False)
type = serializers.ChoiceField(
choices=[(c, c) for c in activity.ACTIVITY_TYPES])
object = serializers.JSONField()
@ -525,6 +525,14 @@ class ActivitySerializer(serializers.Serializer):
)
return value
def to_representation(self, conf):
d = {}
d.update(conf)
if self.context.get('include_ap_context', True):
d['@context'] = AP_CONTEXT
return d
class ObjectSerializer(serializers.Serializer):
id = serializers.URLField()

7
api/funkwhale_api/music/factories.py
View File

@ -81,6 +81,9 @@ class ImportBatchFactory(factory.django.DjangoModelFactory):
submitted_by=None,
source='federation',
)
finished = factory.Trait(
status='finished',
)
@registry.register
@ -98,6 +101,10 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
library_track=factory.SubFactory(LibraryTrackFactory),
batch=factory.SubFactory(ImportBatchFactory, federation=True),
)
finished = factory.Trait(
status='finished',
track_file=factory.SubFactory(TrackFileFactory),
)
@registry.register(name='music.FileImportJob')

9
api/funkwhale_api/music/models.py
View File

@ -505,8 +505,17 @@ class ImportBatch(models.Model):
return str(self.pk)
def update_status(self):
old_status = self.status
self.status = utils.compute_status(self.jobs.all())
self.save(update_fields=['status'])
if self.status != old_status and self.status == 'finished':
from . import tasks
tasks.import_batch_notify_followers.delay(import_batch_id=self.pk)
def get_federation_url(self):
return federation_utils.full_url(
'/federation/music/import/batch/{}'.format(self.uuid)
)
class ImportJob(models.Model):

46
api/funkwhale_api/music/tasks.py
View File

@ -2,6 +2,10 @@ from django.core.files.base import ContentFile
from dynamic_preferences.registries import global_preferences_registry
from funkwhale_api.federation import activity
from funkwhale_api.federation import actors
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.taskapp import celery
from funkwhale_api.providers.acoustid import get_acoustid_client
from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path
@ -128,6 +132,7 @@ def _do_import(import_job, replace, use_acoustid=True):
# it's imported on the track, we don't need it anymore
import_job.audio_file.delete()
import_job.save()
return track.pk
@ -162,3 +167,44 @@ def fetch_content(lyrics):
cleaned_content = lyrics_utils.clean_content(content)
lyrics.content = cleaned_content
lyrics.save(update_fields=['content'])
@celery.app.task(name='music.import_batch_notify_followers')
@celery.require_instance(
models.ImportBatch.objects.filter(status='finished'), 'import_batch')
def import_batch_notify_followers(import_batch):
if not settings.FEDERATION_ENABLED:
return
if import_batch.source == 'federation':
return
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
followers = library_actor.get_approved_followers()
jobs = import_batch.jobs.filter(
status='finished',
library_track__isnull=True,
track_file__isnull=False,
).select_related(
'track_file__track__artist',
'track_file__track__album__artist',
)
track_files = [job.track_file for job in jobs]
collection = federation_serializers.CollectionSerializer({
'actor': library_actor,
'id': import_batch.get_federation_url(),
'items': track_files,
'item_serializer': federation_serializers.AudioSerializer
}).data
for f in followers:
create = federation_serializers.ActivitySerializer(
{
'type': 'Create',
'id': collection['id'],
'object': collection,
'actor': library_actor.url,
'to': [f.url],
}
).data
activity.deliver(create, on_behalf_of=library_actor, to=[f.url])

61
api/tests/federation/test_actors.py
View File

@ -12,6 +12,8 @@ from funkwhale_api.federation import actors
from funkwhale_api.federation import models
from funkwhale_api.federation import serializers
from funkwhale_api.federation import utils
from funkwhale_api.music import models as music_models
from funkwhale_api.music import tasks as music_tasks
def test_actor_fetching(r_mock):
@ -465,3 +467,62 @@ def test_library_actor_handle_create_audio(mocker, factories):
assert lt.artist_name == a['metadata']['artist']['name']
assert lt.album_title == a['metadata']['release']['title']
assert lt.published_date == arrow.get(a['published'])
def test_library_actor_handle_create_audio_autoimport(mocker, factories):
mocked_import = mocker.patch(
'funkwhale_api.common.utils.on_commit')
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
remote_library = factories['federation.Library'](
federation_enabled=True,
autoimport=True,
)
data = {
'actor': remote_library.actor.url,
'type': 'Create',
'id': 'http://test.federation/audio/create',
'object': {
'id': 'https://batch.import',
'type': 'Collection',
'totalItems': 2,
'items': factories['federation.Audio'].create_batch(size=2)
},
}
library_actor.system_conf.post_inbox(data, actor=remote_library.actor)
lts = list(remote_library.tracks.order_by('id'))
assert len(lts) == 2
for i, a in enumerate(data['object']['items']):
lt = lts[i]
assert lt.pk is not None
assert lt.url == a['id']
assert lt.library == remote_library
assert lt.audio_url == a['url']['href']
assert lt.audio_mimetype == a['url']['mediaType']
assert lt.metadata == a['metadata']
assert lt.title == a['metadata']['recording']['title']
assert lt.artist_name == a['metadata']['artist']['name']
assert lt.album_title == a['metadata']['release']['title']
assert lt.published_date == arrow.get(a['published'])
batch = music_models.ImportBatch.objects.latest('id')
assert batch.jobs.count() == len(lts)
assert batch.source == 'federation'
assert batch.submitted_by is None
for i, job in enumerate(batch.jobs.order_by('id')):
lt = lts[i]
assert job.library_track == lt
assert job.mbid == lt.mbid
assert job.source == lt.url
mocked_import.assert_any_call(
music_tasks.import_job_run.delay,
import_job_id=job.pk,
use_acoustid=False,
)

87
api/tests/music/test_import.py
View File

@ -3,6 +3,8 @@ import pytest
from django.urls import reverse
from funkwhale_api.federation import actors
from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.music import tasks
@ -144,3 +146,88 @@ def test_import_job_from_federation_musicbrainz_artist(factories, mocker):
artist_from_api.assert_called_once_with(
mbid=lt.metadata['artist']['musicbrainz_id'])
def test_import_job_run_triggers_notifies_followers(
factories, mocker, tmpfile):
mocker.patch(
'funkwhale_api.downloader.download',
return_value={'audio_file_path': tmpfile.name})
mocked_notify = mocker.patch(
'funkwhale_api.music.tasks.import_batch_notify_followers.delay')
batch = factories['music.ImportBatch']()
job = factories['music.ImportJob'](
finished=True, batch=batch)
track = factories['music.Track'](mbid=job.mbid)
batch.update_status()
batch.refresh_from_db()
assert batch.status == 'finished'
mocked_notify.assert_called_once_with(import_batch_id=batch.pk)
def test_import_batch_notifies_followers_skip_on_disabled_federation(
settings, factories, mocker):
mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver')
batch = factories['music.ImportBatch'](finished=True)
settings.FEDERATION_ENABLED = False
tasks.import_batch_notify_followers(import_batch_id=batch.pk)
mocked_deliver.assert_not_called()
def test_import_batch_notifies_followers_skip_on_federation_import(
factories, mocker):
mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver')
batch = factories['music.ImportBatch'](finished=True, federation=True)
tasks.import_batch_notify_followers(import_batch_id=batch.pk)
mocked_deliver.assert_not_called()
def test_import_batch_notifies_followers(
factories, mocker):
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
f1 = factories['federation.Follow'](approved=True, target=library_actor)
f2 = factories['federation.Follow'](approved=False, target=library_actor)
f3 = factories['federation.Follow']()
mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver')
batch = factories['music.ImportBatch']()
job1 = factories['music.ImportJob'](
finished=True, batch=batch)
job2 = factories['music.ImportJob'](
finished=True, federation=True, batch=batch)
job3 = factories['music.ImportJob'](
status='pending', batch=batch)
batch.status = 'finished'
batch.save()
tasks.import_batch_notify_followers(import_batch_id=batch.pk)
# only f1 match the requirements to be notified
# and only job1 is a non federated track with finished import
expected = {
'@context': federation_serializers.AP_CONTEXT,
'actor': library_actor.url,
'type': 'Create',
'id': batch.get_federation_url(),
'to': [f1.actor.url],
'object': federation_serializers.CollectionSerializer(
{
'id': batch.get_federation_url(),
'items': [job1.track_file],
'actor': library_actor,
'item_serializer': federation_serializers.AudioSerializer
}
).data
}
mocked_deliver.assert_called_once_with(
expected,
on_behalf_of=library_actor,
to=[f1.actor.url]
)

2
front/build/dev-server.js
View File

@ -14,7 +14,7 @@ var webpackConfig = process.env.NODE_ENV === 'testing'
? require('./webpack.prod.conf')
: require('./webpack.dev.conf')
// require('./i18n')
require('./i18n')
// default port where dev server listens for incoming traffic
var port = process.env.PORT || config.dev.port

2
front/src/views/federation/LibraryDetail.vue
View File

@ -46,7 +46,6 @@
<td>
</td>
</tr>
<!-- Disabled until properly implemented on the backend
<tr>
<td>Auto importing</td>
<td>
@ -59,6 +58,7 @@
</td>
<td></td>
</tr>
<!-- Disabled until properly implemented on the backend
<tr>
<td>File mirroring</td>
<td>