diff --git a/api/funkwhale_api/music/filters.py b/api/funkwhale_api/music/filters.py index fbea3735a..752422e75 100644 --- a/api/funkwhale_api/music/filters.py +++ b/api/funkwhale_api/music/filters.py @@ -2,6 +2,7 @@ from django.db.models import Count from django_filters import rest_framework as filters +from funkwhale_api.common import fields from . import models @@ -28,6 +29,39 @@ class ArtistFilter(ListenableMixin): } +class ImportBatchFilter(filters.FilterSet): + q = fields.SearchFilter(search_fields=[ + 'submitted_by__username', + 'source', + ]) + + class Meta: + model = models.ImportBatch + fields = { + 'status': ['exact'], + 'source': ['exact'], + 'submitted_by': ['exact'], + } + + +class ImportJobFilter(filters.FilterSet): + q = fields.SearchFilter(search_fields=[ + 'batch__submitted_by__username', + 'source', + ]) + + class Meta: + model = models.ImportJob + fields = { + 'batch': ['exact'], + 'batch__status': ['exact'], + 'batch__source': ['exact'], + 'batch__submitted_by': ['exact'], + 'status': ['exact'], + 'source': ['exact'], + } + + class AlbumFilter(ListenableMixin): listenable = filters.BooleanFilter(name='_', method='filter_listenable') diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index b5f69eb1d..b9ecfc50d 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -6,6 +6,7 @@ from funkwhale_api.activity import serializers as activity_serializers from funkwhale_api.federation import utils as federation_utils from funkwhale_api.federation.models import LibraryTrack from funkwhale_api.federation.serializers import AP_CONTEXT +from funkwhale_api.users.serializers import UserBasicSerializer from . import models @@ -90,6 +91,7 @@ class TrackSerializerNested(LyricsMixin): files = TrackFileSerializer(many=True, read_only=True) album = SimpleAlbumSerializer(read_only=True) tags = TagSerializer(many=True, read_only=True) + class Meta: model = models.Track fields = ('id', 'mbid', 'title', 'artist', 'files', 'album', 'tags', 'lyrics') @@ -108,6 +110,7 @@ class AlbumSerializerNested(serializers.ModelSerializer): class ArtistSerializerNested(serializers.ModelSerializer): albums = AlbumSerializerNested(many=True, read_only=True) tags = TagSerializer(many=True, read_only=True) + class Meta: model = models.Artist fields = ('id', 'mbid', 'name', 'albums', 'tags') @@ -121,18 +124,43 @@ class LyricsSerializer(serializers.ModelSerializer): class ImportJobSerializer(serializers.ModelSerializer): track_file = TrackFileSerializer(read_only=True) + class Meta: model = models.ImportJob - fields = ('id', 'mbid', 'batch', 'source', 'status', 'track_file', 'audio_file') + fields = ( + 'id', + 'mbid', + 'batch', + 'source', + 'status', + 'track_file', + 'audio_file') read_only_fields = ('status', 'track_file') class ImportBatchSerializer(serializers.ModelSerializer): - jobs = ImportJobSerializer(many=True, read_only=True) + submitted_by = UserBasicSerializer(read_only=True) + class Meta: model = models.ImportBatch - fields = ('id', 'jobs', 'status', 'creation_date', 'import_request') - read_only_fields = ('creation_date',) + fields = ( + 'id', + 'submitted_by', + 'source', + 'status', + 'creation_date', + 'import_request') + read_only_fields = ( + 'creation_date', 'submitted_by', 'source') + + def to_representation(self, instance): + repr = super().to_representation(instance) + try: + repr['job_count'] = instance.job_count + except AttributeError: + # Queryset was not annotated + pass + return repr class TrackActivitySerializer(activity_serializers.ModelSerializer): diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index 224d085b6..890cc2578 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -11,6 +11,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.conf import settings from django.db import models, transaction from django.db.models.functions import Length +from django.db.models import Count from django.http import StreamingHttpResponse from django.urls import reverse from django.utils.decorators import method_decorator @@ -99,14 +100,14 @@ class ImportBatchViewSet( mixins.RetrieveModelMixin, viewsets.GenericViewSet): queryset = ( - models.ImportBatch.objects.all() - .prefetch_related('jobs__track_file') - .order_by('-creation_date')) + models.ImportBatch.objects + .select_related() + .order_by('-creation_date') + .annotate(job_count=Count('jobs')) + ) serializer_class = serializers.ImportBatchSerializer permission_classes = (permissions.DjangoModelPermissions, ) - - def get_queryset(self): - return super().get_queryset().filter(submitted_by=self.request.user) + filter_class = filters.ImportBatchFilter def perform_create(self, serializer): serializer.save(submitted_by=self.request.user) @@ -119,13 +120,30 @@ class ImportJobPermission(HasModelPermission): class ImportJobViewSet( mixins.CreateModelMixin, + mixins.ListModelMixin, viewsets.GenericViewSet): - queryset = (models.ImportJob.objects.all()) + queryset = (models.ImportJob.objects.all().select_related()) serializer_class = serializers.ImportJobSerializer permission_classes = (ImportJobPermission, ) + filter_class = filters.ImportJobFilter - def get_queryset(self): - return super().get_queryset().filter(batch__submitted_by=self.request.user) + @list_route(methods=['get']) + def stats(self, request, *args, **kwargs): + qs = models.ImportJob.objects.all() + filterset = filters.ImportJobFilter(request.GET, queryset=qs) + qs = filterset.qs + qs = qs.values('status').order_by('status') + qs = qs.annotate(status_count=Count('status')) + + data = {} + for row in qs: + data[row['status']] = row['status_count'] + + for s, _ in models.IMPORT_STATUS_CHOICES: + data.setdefault(s, 0) + + data['count'] = sum([v for v in data.values()]) + return Response(data) def perform_create(self, serializer): source = 'file://' + serializer.validated_data['audio_file'].name @@ -136,7 +154,8 @@ class ImportJobViewSet( ) -class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet): +class TrackViewSet( + TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet): """ A simple ViewSet for viewing and editing accounts. """ diff --git a/api/tests/music/test_api.py b/api/tests/music/test_api.py index 606720e13..cc6fe644b 100644 --- a/api/tests/music/test_api.py +++ b/api/tests/music/test_api.py @@ -181,30 +181,6 @@ def test_can_import_whole_artist( assert job.source == row['source'] -def test_user_can_query_api_for_his_own_batches( - superuser_api_client, factories): - factories['music.ImportJob']() - job = factories['music.ImportJob']( - batch__submitted_by=superuser_api_client.user) - url = reverse('api:v1:import-batches-list') - - response = superuser_api_client.get(url) - results = response.data - assert results['count'] == 1 - assert results['results'][0]['jobs'][0]['mbid'] == job.mbid - - -def test_user_cannnot_access_other_batches( - superuser_api_client, factories): - factories['music.ImportJob']() - job = factories['music.ImportJob']() - url = reverse('api:v1:import-batches-list') - - response = superuser_api_client.get(url) - results = response.data - assert results['count'] == 0 - - def test_user_can_create_an_empty_batch(superuser_api_client, factories): url = reverse('api:v1:import-batches-list') response = superuser_api_client.post(url) diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py index 95c56d914..5863e7b07 100644 --- a/api/tests/music/test_views.py +++ b/api/tests/music/test_views.py @@ -128,3 +128,46 @@ def test_can_create_import_from_federation_tracks( assert batch.jobs.count() == 5 for i, job in enumerate(batch.jobs.all()): assert job.library_track == lts[i] + + +def test_can_list_import_jobs(factories, superuser_api_client): + job = factories['music.ImportJob']() + url = reverse('api:v1:import-jobs-list') + response = superuser_api_client.get(url) + + assert response.status_code == 200 + assert response.data['results'][0]['id'] == job.pk + + +def test_import_job_stats(factories, superuser_api_client): + job1 = factories['music.ImportJob'](status='pending') + job2 = factories['music.ImportJob'](status='errored') + + url = reverse('api:v1:import-jobs-stats') + response = superuser_api_client.get(url) + expected = { + 'errored': 1, + 'pending': 1, + 'finished': 0, + 'skipped': 0, + 'count': 2, + } + assert response.status_code == 200 + assert response.data == expected + + +def test_import_job_stats_filter(factories, superuser_api_client): + job1 = factories['music.ImportJob'](status='pending') + job2 = factories['music.ImportJob'](status='errored') + + url = reverse('api:v1:import-jobs-stats') + response = superuser_api_client.get(url, {'batch': job1.batch.pk}) + expected = { + 'errored': 0, + 'pending': 1, + 'finished': 0, + 'skipped': 0, + 'count': 1, + } + assert response.status_code == 200 + assert response.data == expected diff --git a/changes/changelog.d/171.enhancement b/changes/changelog.d/171.enhancement new file mode 100644 index 000000000..14e9f6d9b --- /dev/null +++ b/changes/changelog.d/171.enhancement @@ -0,0 +1,2 @@ +Import job and batch API and front-end have been improved with better performance, +pagination and additional filters (#171) diff --git a/front/src/components/library/import/BatchDetail.vue b/front/src/components/library/import/BatchDetail.vue index c7894fcc0..b73c8cf82 100644 --- a/front/src/components/library/import/BatchDetail.vue +++ b/front/src/components/library/import/BatchDetail.vue @@ -4,31 +4,80 @@
-
-
-
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ {{ $t('Import batch') }} + + #{{ batch.id }} +
+ {{ $t('Launch date') }} + + +
+ {{ $t('Submitted by') }} + + +
{{ $t('Pending') }}{{ stats.pending }}
{{ $t('Skipped') }}{{ stats.skipped }}
{{ $t('Errored') }}{{ stats.errored }}
{{ $t('Finished') }}{{ stats.finished }}/{{ stats.count}}
+
+
+
+ + +
+
+ + +
-
Importing {{ batch.jobs.length }} tracks...
-
Imported {{ batch.jobs.length }} tracks!
- +
- - - - - + + + + + - + + + + + + + + +
{{ $t('Job ID') }}{{ $t('Recording MusicBrainz ID') }}{{ $t('Source') }}{{ $t('Status') }}{{ $t('Track') }}
{{ job.id }} {{ job.mbid }} @@ -45,29 +94,64 @@
+ + + {{ $t('Showing results {%start%}-{%end%} on {%total%}', {start: ((jobFilters.page-1) * jobFilters.paginateBy) + 1 , end: ((jobFilters.page-1) * jobFilters.paginateBy) + jobResult.results.length, total: jobResult.count})}} + +
-