Merge branch 'performance-improvements' into 'develop'

Improved performance when listing playable tracks, albums and artists

See merge request funkwhale/funkwhale!514

api/funkwhale_api/music/migrations/0037_auto_20190103_1757.py

@@ -0,0 +1,43 @@
+# Generated by Django 2.0.9 on 2019-01-03 17:57
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0036_track_disc_number'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='track',
+            options={'ordering': ['album', 'disc_number', 'position']},
+        ),
+        migrations.AlterField(
+            model_name='album',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='artist',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='track',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='upload',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='work',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -44,7 +44,7 @@ class APIModelMixin(models.Model):
         "federation.Activity", null=True, blank=True, on_delete=models.SET_NULL
     )
     api_includes = []
-    creation_date = models.DateTimeField(default=timezone.now)
+    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
     import_hooks = []
 
     class Meta:
@@ -161,10 +161,11 @@ class ArtistQuerySet(models.QuerySet):
 
     def playable_by(self, actor, include=True):
         tracks = Track.objects.playable_by(actor, include)
+        matches = self.filter(tracks__in=tracks).values_list("pk")
         if include:
-            return self.filter(tracks__in=tracks).distinct()
+            return self.filter(pk__in=matches)
         else:
-            return self.exclude(tracks__in=tracks).distinct()
+            return self.exclude(pk__in=matches)
 
 
 class Artist(APIModelMixin):
@@ -229,10 +230,11 @@ class AlbumQuerySet(models.QuerySet):
 
     def playable_by(self, actor, include=True):
         tracks = Track.objects.playable_by(actor, include)
+        matches = self.filter(tracks__in=tracks).values_list("pk")
         if include:
-            return self.filter(tracks__in=tracks).distinct()
+            return self.filter(pk__in=matches)
         else:
-            return self.exclude(tracks__in=tracks).distinct()
+            return self.exclude(pk__in=matches)
 
     def with_prefetched_tracks_and_playable_uploads(self, actor):
         tracks = Track.objects.with_playable_uploads(actor)
@@ -429,10 +431,11 @@ class TrackQuerySet(models.QuerySet):
 
     def playable_by(self, actor, include=True):
         files = Upload.objects.playable_by(actor, include)
+        matches = self.filter(uploads__in=files).values_list("pk")
         if include:
-            return self.filter(uploads__in=files).distinct()
+            return self.filter(pk__in=matches)
         else:
-            return self.exclude(uploads__in=files).distinct()
+            return self.exclude(pk__in=matches)
 
     def with_playable_uploads(self, actor):
         uploads = Upload.objects.playable_by(actor).select_related("track")
@@ -606,10 +609,8 @@ class UploadQuerySet(models.QuerySet):
         libraries = Library.objects.viewable_by(actor)
 
         if include:
-            return self.filter(
+            return self.filter(library__in=libraries, import_status="finished")
-                library__in=libraries, import_status="finished"
+        return self.exclude(library__in=libraries, import_status="finished")
-            ).distinct()
-        return self.exclude(library__in=libraries, import_status="finished").distinct()
 
     def local(self, include=True):
         return self.exclude(library__actor__user__isnull=include)
@@ -657,7 +658,7 @@ class Upload(models.Model):
         blank=True,
         max_length=500,
     )
-    creation_date = models.DateTimeField(default=timezone.now)
+    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
     modification_date = models.DateTimeField(default=timezone.now, null=True)
     accessed_date = models.DateTimeField(null=True, blank=True)
     duration = models.IntegerField(null=True, blank=True)

api/funkwhale_api/music/views.py

@@ -71,7 +71,7 @@ class ArtistViewSet(viewsets.ReadOnlyModelViewSet):
         albums = albums.annotate_playable_by_actor(
             utils.get_actor_from_request(self.request)
         )
-        return queryset.prefetch_related(Prefetch("albums", queryset=albums)).distinct()
+        return queryset.prefetch_related(Prefetch("albums", queryset=albums))
 
     libraries = detail_route(methods=["get"])(
         get_libraries(
@@ -99,7 +99,7 @@ class AlbumViewSet(viewsets.ReadOnlyModelViewSet):
             .order_for_album()
         )
         qs = queryset.prefetch_related(Prefetch("tracks", queryset=tracks))
-        return qs.distinct()
+        return qs
 
     libraries = detail_route(methods=["get"])(
         get_libraries(filter_uploads=lambda o, uploads: uploads.filter(track__album=o))

api/funkwhale_api/subsonic/views.py

@@ -57,7 +57,7 @@ def find_object(
 
             if filter_playable:
                 actor = utils.get_actor_from_request(request)
-                qs = qs.playable_by(actor).distinct()
+                qs = qs.playable_by(actor)
 
             try:
                 obj = qs.get(**{model_field: value})

changes/changelog.d/db-query-optimization.enhancement

@@ -0,0 +1 @@
+Improved performance when listing playable tracks, albums and artists