Will now fetch and cache federated tracks

api/funkwhale_api/federation/migrations/0005_auto_20180413_1723.py

@@ -0,0 +1,26 @@
+# Generated by Django 2.0.3 on 2018-04-13 17:23
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0004_auto_20180410_2025'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='librarytrack',
+            name='audio_file',
+            field=models.FileField(blank=True, null=True, upload_to=funkwhale_api.federation.models.get_file_path),
+        ),
+        migrations.AlterField(
+            model_name='librarytrack',
+            name='metadata',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default={}, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=10000),
+        ),
+    ]

api/funkwhale_api/federation/models.py

@@ -1,4 +1,6 @@
+import os
 import uuid
+import tempfile
 
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
@@ -6,6 +8,9 @@ from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.utils import timezone
 
+from funkwhale_api.common import session
+from funkwhale_api.music import utils as music_utils
+
 TYPE_CHOICES = [
     ('Person', 'Person'),
     ('Application', 'Application'),
@@ -147,10 +152,23 @@ class Library(models.Model):
     )
 
 
+def get_file_path(instance, filename):
+    uid = str(uuid.uuid4())
+    chunk_size = 2
+    chunks = [uid[i:i+chunk_size] for i in range(0, len(uid), chunk_size)]
+    parts = chunks[:3] + [filename]
+    return os.path.join('federation_cache', *parts)
+
+
 class LibraryTrack(models.Model):
     url = models.URLField(unique=True)
     audio_url = models.URLField()
     audio_mimetype = models.CharField(max_length=200)
+    audio_file = models.FileField(
+        upload_to=get_file_path,
+        null=True,
+        blank=True)
+
     creation_date = models.DateTimeField(default=timezone.now)
     modification_date = models.DateTimeField(
         auto_now=True)
@@ -170,3 +188,26 @@ class LibraryTrack(models.Model):
             return self.metadata['recording']['musicbrainz_id']
         except KeyError:
             pass
+
+    def download_audio(self):
+        from . import actors
+        auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
+        remote_response = session.get_session().get(
+            self.audio_url,
+            auth=auth,
+            stream=True,
+            timeout=20,
+            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+            headers={
+                'Content-Type': 'application/activity+json'
+            }
+        )
+        with remote_response as r:
+            remote_response.raise_for_status()
+            extension = music_utils.get_ext_from_type(self.audio_mimetype)
+            title = ' - '.join([self.title, self.album_title, self.artist_name])
+            filename = '{}.{}'.format(title, extension)
+            tmp_file = tempfile.TemporaryFile()
+            for chunk in r.iter_content(chunk_size=512):
+                tmp_file.write(chunk)
+            self.audio_file.save(filename, tmp_file)

api/funkwhale_api/music/views.py

@@ -23,7 +23,6 @@ from rest_framework import permissions
 from musicbrainzngs import ResponseError
 
 from funkwhale_api.common import utils as funkwhale_utils
-from funkwhale_api.common import session
 from funkwhale_api.federation import actors
 from funkwhale_api.requests.models import ImportRequest
 from funkwhale_api.musicbrainz import api
@@ -206,35 +205,22 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
             return Response(status=404)
 
         mt = f.mimetype
+        audio_file = f.audio_file
         try:
             library_track = f.library_track
         except ObjectDoesNotExist:
             library_track = None
-        if library_track and not f.audio_file:
+        if library_track and not audio_file:
-            # we proxy the response to the remote library
+            if not library_track.audio_file:
-            # since we did not mirror the file locally
+                # we need to populate from cache
+                library_track.download_audio()
+            audio_file = library_track.audio_file
             mt = library_track.audio_mimetype
-            file_extension = utils.get_ext_from_type(mt)
+        response = Response()
-            filename = '{}.{}'.format(f.track.full_name, file_extension)
+        filename = f.filename
-            auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
+        response['X-Accel-Redirect'] = "{}{}".format(
-            remote_response = session.get_session().get(
+            settings.PROTECT_FILES_PATH,
-                library_track.audio_url,
+            audio_file.url)
-                auth=auth,
-                stream=True,
-                timeout=20,
-                verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
-                headers={
-                    'Content-Type': 'application/activity+json'
-                })
-            logger.debug(
-                'Proxying media request to %s', library_track.audio_url)
-            response = StreamingHttpResponse(remote_response.iter_content())
-        else:
-            response = Response()
-            filename = f.filename
-            response['X-Accel-Redirect'] = "{}{}".format(
-                settings.PROTECT_FILES_PATH,
-                f.audio_file.url)
         filename = "filename*=UTF-8''{}".format(
             urllib.parse.quote(filename))
         response["Content-Disposition"] = "attachment; {}".format(filename)

api/tests/music/test_views.py

@@ -79,12 +79,16 @@ def test_can_proxy_remote_track(
     settings.PROTECT_AUDIO_FILES = False
     track_file = factories['music.TrackFile'](federation=True)
 
-    r_mock.get(track_file.library_track.audio_url, body=io.StringIO('test'))
+    r_mock.get(track_file.library_track.audio_url, body=io.BytesIO(b'test'))
     response = api_client.get(track_file.path)
 
+    library_track = track_file.library_track
+    library_track.refresh_from_db()
     assert response.status_code == 200
-    assert list(response.streaming_content) == [b't', b'e', b's', b't']
+    assert response['X-Accel-Redirect'] == "{}{}".format(
-    assert response['Content-Type'] == track_file.library_track.audio_mimetype
+        settings.PROTECT_FILES_PATH,
+        library_track.audio_file.url)
+    assert library_track.audio_file.read() == b'test'
 
 
 def test_can_create_import_from_federation_tracks(