AudioCollection to import job and track file creation

api/funkwhale_api/music/serializers.py

@@ -1,3 +1,4 @@
+from django.db import transaction
 from rest_framework import serializers
 from taggit.models import Tag
 
@@ -150,3 +151,83 @@ class TrackActivitySerializer(activity_serializers.ModelSerializer):
 
     def get_type(self, obj):
         return 'Audio'
+
+
+class AudioMetadataSerializer(serializers.Serializer):
+    artist = serializers.CharField(required=False)
+    release = serializers.CharField(required=False)
+    recording = serializers.CharField(required=False)
+
+
+class AudioSerializer(serializers.Serializer):
+    type = serializers.CharField()
+    id = serializers.URLField()
+    url = serializers.JSONField()
+    metadata = AudioMetadataSerializer()
+
+    def validate_type(self, v):
+        if v != 'Audio':
+            raise serializers.ValidationError('Invalid type for audio')
+        return v
+
+    def validate_url(self, v):
+        try:
+            url = v['href']
+        except (KeyError, TypeError):
+            raise serializers.ValidationError('Missing href')
+
+        try:
+            media_type = v['mediaType']
+        except (KeyError, TypeError):
+            raise serializers.ValidationError('Missing mediaType')
+
+        if not media_type.startswith('audio/'):
+            raise serializers.ValidationError('Invalid mediaType')
+
+        return url
+
+    def validate_url(self, v):
+        try:
+            url = v['href']
+        except (KeyError, TypeError):
+            raise serializers.ValidationError('Missing href')
+
+        try:
+            media_type = v['mediaType']
+        except (KeyError, TypeError):
+            raise serializers.ValidationError('Missing mediaType')
+
+        if not media_type.startswith('audio/'):
+            raise serializers.ValidationError('Invalid mediaType')
+
+        return v
+
+    def create(self, validated_data, batch):
+        metadata = validated_data['metadata'].copy()
+        metadata['mediaType'] = validated_data['url']['mediaType']
+        return models.ImportJob.objects.create(
+            batch=batch,
+            source=validated_data['url']['href'],
+            federation_source=validated_data['id'],
+            metadata=metadata,
+        )
+
+
+class AudioCollectionImportSerializer(serializers.Serializer):
+    id = serializers.URLField()
+    items = serializers.ListField(
+        child=AudioSerializer(),
+        min_length=1,
+    )
+
+    @transaction.atomic
+    def create(self, validated_data):
+        batch = models.ImportBatch.objects.create(
+            federation_actor=self.context['sender'],
+            federation_source=validated_data['id'],
+            source='federation',
+        )
+        for i in validated_data['items']:
+            s = AudioSerializer(data=i)
+            job = s.create(i, batch)
+        return batch

api/funkwhale_api/music/tasks.py

@@ -25,6 +25,44 @@ def set_acoustid_on_track_file(track_file):
         return update(result['id'])
 
 
+def get_mbid(url, type):
+    prefix = 'https://musicbrainz.org/{}/'.format(type)
+    if url.startswith(prefix):
+        return url.replace(prefix, '')
+
+
+def import_track_from_metadata(metadata):
+    raw_track = metadata['recording']
+    if isinstance(raw_track, str):
+        track_mbid = get_mbid(raw_track, 'recording')
+        return models.Track.get_or_create_from_api(mbid=track_mbid)
+
+    raw_album = metadata['release']
+    if isinstance(raw_album, str):
+        album_mbid = get_mbid(raw_album, 'release')
+        album = models.Album.get_or_create_from_api(mbid=album_mbid)
+        return models.Track.get_or_create_from_title(
+            raw_track['title'], artist=album.artist, album=album)
+
+    raw_artist = metadata['artist']
+    if isinstance(raw_artist, str):
+        artist_mbid = get_mbid(raw_artist, 'artist')
+        artist = models.Artist.get_or_create_from_api(mbid=artist_mbid)
+        album = models.Album.get_or_create_from_title(
+            raw_album['title'], artist=artist)
+        return models.Track.get_or_create_from_title(
+            raw_track['title'], artist=artist, album=album)
+
+    # worst case scenario, we have absolutely no way to link to a
+    # musicbrainz resource, we rely on the name/titles
+    artist = models.Artist.get_or_create_from_name(
+        raw_artist['name'])
+    album = models.Album.get_or_create_from_title(
+        raw_album['title'], artist=artist)
+    return models.Track.get_or_create_from_title(
+        raw_track['title'], artist=artist, album=album)
+
+
 def _do_import(import_job, replace, use_acoustid=True):
     from_file = bool(import_job.audio_file)
     mbid = import_job.mbid
@@ -43,9 +81,14 @@ def _do_import(import_job, replace, use_acoustid=True):
             acoustid_track_id = match['id']
     if mbid:
         track, _ = models.Track.get_or_create_from_api(mbid=mbid)
-    else:
+    elif import_job.audio_file:
         track = import_track_data_from_path(import_job.audio_file.path)
+    else:
+        # probably federation, we use metadata stored on the job itself
+        if not import_job.metadata:
+            raise ValueError('We cannot import without at least metadatas')
 
+        track = import_track_from_metadata(import_job.metadata)
     track_file = None
     if replace:
         track_file = track.files.first()
@@ -63,6 +106,9 @@ def _do_import(import_job, replace, use_acoustid=True):
         track_file.audio_file = ContentFile(import_job.audio_file.read())
         track_file.audio_file.name = import_job.audio_file.name
         track_file.duration = duration
+    elif import_job.federation_source:
+        # no downloading, we hotlink
+        pass
     else:
         track_file.download_file()
     track_file.save()

api/tests/music/test_import.py

@@ -1,7 +1,10 @@
 import json
+import pytest
 
 from django.urls import reverse
 
+from funkwhale_api.music import tasks
+
 
 def test_create_import_can_bind_to_request(
         artists, albums, mocker, factories, superuser_api_client):
@@ -33,3 +36,101 @@ def test_create_import_can_bind_to_request(
     batch = request.import_batches.latest('id')
 
     assert batch.import_request == request
+
+
+@pytest.mark.parametrize('url,type,expected', [
+    ('https://musicbrainz.org/artist/test', 'artist', 'test'),
+    ('https://musicbrainz.org/release/test', 'release', 'test'),
+    ('https://musicbrainz.org/recording/test', 'recording', 'test'),
+    ('https://musicbrainz.org/recording/test', 'artist', None),
+])
+def test_get_mbid(url, type, expected):
+    assert tasks.get_mbid(url, type) == expected
+
+
+def test_import_job_from_federation_no_musicbrainz(factories):
+    job = factories['music.ImportJob'](
+        federation=True,
+        metadata__artist={'name': 'Hello'},
+        metadata__release={'title': 'World'},
+        metadata__recording={'title': 'Ping'},
+        mbid=None,
+    )
+
+    tasks.import_job_run(import_job_id=job.pk)
+    job.refresh_from_db()
+
+    tf = job.track_file
+    assert tf.track.title == 'Ping'
+    assert tf.track.artist.name == 'Hello'
+    assert tf.track.album.title == 'World'
+
+
+def test_import_job_from_federation_musicbrainz_recording(factories, mocker):
+    t = factories['music.Track']()
+    track_from_api = mocker.patch(
+        'funkwhale_api.music.models.Track.get_or_create_from_api',
+        return_value=t)
+    job = factories['music.ImportJob'](
+        federation=True,
+        metadata__artist={'name': 'Hello'},
+        metadata__release={'title': 'World'},
+        mbid=None,
+    )
+
+    tasks.import_job_run(import_job_id=job.pk)
+    job.refresh_from_db()
+
+    tf = job.track_file
+    assert tf.track == t
+    track_from_api.assert_called_once_with(
+        mbid=tasks.get_mbid(job.metadata['recording'], 'recording'))
+
+
+def test_import_job_from_federation_musicbrainz_release(factories, mocker):
+    a = factories['music.Album']()
+    album_from_api = mocker.patch(
+        'funkwhale_api.music.models.Album.get_or_create_from_api',
+        return_value=a)
+    job = factories['music.ImportJob'](
+        federation=True,
+        metadata__artist={'name': 'Hello'},
+        metadata__recording={'title': 'Ping'},
+        mbid=None,
+    )
+
+    tasks.import_job_run(import_job_id=job.pk)
+    job.refresh_from_db()
+
+    tf = job.track_file
+    assert tf.track.title == 'Ping'
+    assert tf.track.artist == a.artist
+    assert tf.track.album == a
+
+    album_from_api.assert_called_once_with(
+        mbid=tasks.get_mbid(job.metadata['release'], 'release'))
+
+
+def test_import_job_from_federation_musicbrainz_artist(factories, mocker):
+    a = factories['music.Artist']()
+    artist_from_api = mocker.patch(
+        'funkwhale_api.music.models.Artist.get_or_create_from_api',
+        return_value=a)
+    job = factories['music.ImportJob'](
+        federation=True,
+        metadata__release={'title': 'World'},
+        metadata__recording={'title': 'Ping'},
+        mbid=None,
+    )
+
+    tasks.import_job_run(import_job_id=job.pk)
+    job.refresh_from_db()
+
+    tf = job.track_file
+    assert tf.track.title == 'Ping'
+    assert tf.track.artist == a
+    assert tf.track.album.artist == a
+    assert tf.track.album.title == 'World'
+
+    artist_from_api.assert_called_once_with(
+        mbid=tasks.get_mbid(job.metadata['artist'], 'artist'))

api/tests/music/test_serializers.py

@@ -0,0 +1,32 @@
+from funkwhale_api.music import serializers
+
+
+def test_activity_pub_audio_collection_serializer(factories):
+    sender = factories['federation.Actor']()
+
+    collection = {
+        'id': 'https://batch.import',
+        'type': 'Collection',
+        'totalItems': 2,
+        'items': factories['federation.Audio'].create_batch(size=2)
+    }
+
+    serializer = serializers.AudioCollectionImportSerializer(
+        data=collection, context={'sender': sender})
+
+    assert serializer.is_valid(raise_exception=True)
+
+    batch = serializer.save()
+    jobs = list(batch.jobs.all())
+
+    assert batch.source == 'federation'
+    assert batch.federation_source == collection['id']
+    assert batch.federation_actor == sender
+    assert len(jobs) == 2
+
+    for i, a in enumerate(collection['items']):
+        job = jobs[i]
+        assert job.federation_source == a['id']
+        assert job.source == a['url']['href']
+        a['metadata']['mediaType'] = a['url']['mediaType']
+        assert job.metadata == a['metadata']