Merge branch '183-inplace-federation' into 'develop'

Resolve "In place imported files not visible on federated collections"

Closes #183

See merge request funkwhale/funkwhale!168

api/funkwhale_api/federation/serializers.py

@@ -1,3 +1,4 @@
+import logging
 import urllib.parse
 
 from django.urls import reverse
@@ -21,6 +22,8 @@ AP_CONTEXT = [
     {},
 ]
 
+logger = logging.getLogger(__name__)
+
 
 class ActorSerializer(serializers.Serializer):
     id = serializers.URLField()
@@ -620,6 +623,8 @@ class CollectionPageSerializer(serializers.Serializer):
         for i in raw_items:
             if i.is_valid():
                 valid_items.append(i)
+            else:
+                logger.debug('Invalid item %s: %s', i.data, i.errors)
 
         return valid_items
 

api/funkwhale_api/music/management/commands/fix_track_files.py

@@ -0,0 +1,45 @@
+import cacheops
+import os
+
+from django.db import transaction
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+from funkwhale_api.music import models, utils
+
+
+class Command(BaseCommand):
+    help = 'Run common checks and fix against imported tracks'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            dest='dry_run',
+            default=False,
+            help='Do not execute anything'
+        )
+
+    def handle(self, *args, **options):
+        if options['dry_run']:
+            self.stdout.write('Dry-run on, will not commit anything')
+        self.fix_mimetypes(**options)
+        cacheops.invalidate_model(models.TrackFile)
+
+    @transaction.atomic
+    def fix_mimetypes(self, dry_run, **kwargs):
+        self.stdout.write('Fixing missing mimetypes...')
+        matching = models.TrackFile.objects.filter(
+            source__startswith='file://', mimetype=None)
+        self.stdout.write(
+            '[mimetypes] {} entries found with no mimetype'.format(
+                matching.count()))
+        for extension, mimetype in utils.EXTENSION_TO_MIMETYPE.items():
+            qs = matching.filter(source__endswith='.{}'.format(extension))
+            self.stdout.write(
+                '[mimetypes] setting {} {} files to {}'.format(
+                    qs.count(), extension, mimetype
+                ))
+            if not dry_run:
+                self.stdout.write('[mimetypes] commiting...')
+                qs.update(mimetype=mimetype)

api/funkwhale_api/music/tasks.py

@@ -1,3 +1,5 @@
+import os
+
 from django.core.files.base import ContentFile
 
 from dynamic_preferences.registries import global_preferences_registry
@@ -13,6 +15,7 @@ from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path
 from django.conf import settings
 from . import models
 from . import lyrics as lyrics_utils
+from . import utils as music_utils
 
 
 @celery.app.task(name='acoustid.set_on_track_file')
@@ -129,6 +132,10 @@ def _do_import(import_job, replace=False, use_acoustid=True):
     elif not import_job.audio_file and not import_job.source.startswith('file://'):
         # not an implace import, and we have a source, so let's download it
         track_file.download_file()
+    elif not import_job.audio_file and import_job.source.startswith('file://'):
+        # in place import, we set mimetype from extension
+        path, ext = os.path.splitext(import_job.source)
+        track_file.mimetype = music_utils.get_type_from_ext(ext)
     track_file.save()
     import_job.status = 'finished'
     import_job.track_file = track_file

api/funkwhale_api/music/utils.py

@@ -63,8 +63,21 @@ def compute_status(jobs):
     return 'finished'
 
 
+AUDIO_EXTENSIONS_AND_MIMETYPE = [
+    ('ogg', 'audio/ogg'),
+    ('mp3', 'audio/mpeg'),
+]
+
+EXTENSION_TO_MIMETYPE = {ext: mt for ext, mt in AUDIO_EXTENSIONS_AND_MIMETYPE}
+MIMETYPE_TO_EXTENSION = {mt: ext for ext, mt in AUDIO_EXTENSIONS_AND_MIMETYPE}
+
+
 def get_ext_from_type(mimetype):
-    mapping = {
-        'audio/ogg': 'ogg',
-        'audio/mpeg': 'mp3',
-    }
+    return MIMETYPE_TO_EXTENSION.get(mimetype)
+
+
+def get_type_from_ext(extension):
+    if extension.startswith('.'):
+        # we remove leading dot
+        extension = extension[1:]
+    return EXTENSION_TO_MIMETYPE.get(extension)

api/tests/music/test_import.py

@@ -243,3 +243,4 @@ def test__do_import_in_place_mbid(factories, tmpfile):
 
     assert bool(tf.audio_file) is False
     assert tf.source == 'file:///test.ogg'
+    assert tf.mimetype == 'audio/ogg'

changes/changelog.d/183.bugfix

@@ -0,0 +1 @@
+Ensure in place imported files get a proper mimetype (#183)

changes/changelog.d/183.enhancement

@@ -0,0 +1 @@
+Added a fix_track_files command to run checks and fixes against library (#183)