From 2569f136b78a77cd10ee39155c8320f780d520fc Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Mon, 28 May 2018 22:17:36 +0200 Subject: [PATCH 1/8] Fix #241: autoremove media files on model instance deletion --- api/config/settings/common.py | 1 + api/requirements/base.txt | 1 + changes/changelog.d/241.enhancement | 1 + 3 files changed, 3 insertions(+) create mode 100644 changes/changelog.d/241.enhancement diff --git a/api/config/settings/common.py b/api/config/settings/common.py index f376781b0..50c62e9d5 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -97,6 +97,7 @@ THIRD_PARTY_APPS = ( 'dynamic_preferences', 'django_filters', 'cacheops', + 'django_cleanup', ) diff --git a/api/requirements/base.txt b/api/requirements/base.txt index d88483de4..13c0efdbc 100644 --- a/api/requirements/base.txt +++ b/api/requirements/base.txt @@ -65,3 +65,4 @@ cryptography>=2,<3 # requests-http-signature==0.0.3 # clone until the branch is merged and released upstream git+https://github.com/EliotBerriot/requests-http-signature.git@signature-header-support +django-cleanup==2.1.0 diff --git a/changes/changelog.d/241.enhancement b/changes/changelog.d/241.enhancement new file mode 100644 index 000000000..00c84c497 --- /dev/null +++ b/changes/changelog.d/241.enhancement @@ -0,0 +1 @@ +Autoremove media files on model instance deletion (#241) From c7782693bca3aeddcceeba932dd30a4e8bd79233 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Tue, 29 May 2018 00:07:20 +0200 Subject: [PATCH 2/8] See #223: api for listing/managing library files --- api/config/api_urls.py | 4 ++ api/funkwhale_api/manage/__init__.py | 3 + api/funkwhale_api/manage/filters.py | 25 ++++++++ api/funkwhale_api/manage/serializers.py | 81 +++++++++++++++++++++++++ api/funkwhale_api/manage/urls.py | 11 ++++ api/funkwhale_api/manage/views.py | 49 +++++++++++++++ api/funkwhale_api/music/factories.py | 5 ++ api/tests/manage/__init__.py | 0 api/tests/manage/test_serializers.py | 10 +++ api/tests/manage/test_views.py | 25 ++++++++ 10 files changed, 213 insertions(+) create mode 100644 api/funkwhale_api/manage/__init__.py create mode 100644 api/funkwhale_api/manage/filters.py create mode 100644 api/funkwhale_api/manage/serializers.py create mode 100644 api/funkwhale_api/manage/urls.py create mode 100644 api/funkwhale_api/manage/views.py create mode 100644 api/tests/manage/__init__.py create mode 100644 api/tests/manage/test_serializers.py create mode 100644 api/tests/manage/test_views.py diff --git a/api/config/api_urls.py b/api/config/api_urls.py index e75781d14..98b863a93 100644 --- a/api/config/api_urls.py +++ b/api/config/api_urls.py @@ -38,6 +38,10 @@ v1_patterns += [ include( ('funkwhale_api.instance.urls', 'instance'), namespace='instance')), + url(r'^manage/', + include( + ('funkwhale_api.manage.urls', 'manage'), + namespace='manage')), url(r'^federation/', include( ('funkwhale_api.federation.api_urls', 'federation'), diff --git a/api/funkwhale_api/manage/__init__.py b/api/funkwhale_api/manage/__init__.py new file mode 100644 index 000000000..03e091e5c --- /dev/null +++ b/api/funkwhale_api/manage/__init__.py @@ -0,0 +1,3 @@ +""" +App that includes all views/serializers and stuff for management API +""" diff --git a/api/funkwhale_api/manage/filters.py b/api/funkwhale_api/manage/filters.py new file mode 100644 index 000000000..9853b7a61 --- /dev/null +++ b/api/funkwhale_api/manage/filters.py @@ -0,0 +1,25 @@ +from django.db.models import Count + +from django_filters import rest_framework as filters + +from funkwhale_api.common import fields +from funkwhale_api.music import models as music_models + + +class ManageTrackFileFilterSet(filters.FilterSet): + q = fields.SearchFilter(search_fields=[ + 'track__title', + 'track__album__title', + 'track__artist__name', + 'source', + ]) + + class Meta: + model = music_models.TrackFile + fields = [ + 'q', + 'track__album', + 'track__artist', + 'track', + 'library_track' + ] diff --git a/api/funkwhale_api/manage/serializers.py b/api/funkwhale_api/manage/serializers.py new file mode 100644 index 000000000..bbd393236 --- /dev/null +++ b/api/funkwhale_api/manage/serializers.py @@ -0,0 +1,81 @@ +from django.db import transaction +from rest_framework import serializers + +from funkwhale_api.common import serializers as common_serializers +from funkwhale_api.music import models as music_models + +from . import filters + + +class ManageTrackFileArtistSerializer(serializers.ModelSerializer): + class Meta: + model = music_models.Artist + fields = [ + 'id', + 'mbid', + 'creation_date', + 'name', + ] + + +class ManageTrackFileAlbumSerializer(serializers.ModelSerializer): + artist = ManageTrackFileArtistSerializer() + + class Meta: + model = music_models.Album + fields = ( + 'id', + 'mbid', + 'title', + 'artist', + 'release_date', + 'cover', + 'creation_date', + ) + + +class ManageTrackFileTrackSerializer(serializers.ModelSerializer): + artist = ManageTrackFileArtistSerializer() + album = ManageTrackFileAlbumSerializer() + + class Meta: + model = music_models.Track + fields = ( + 'id', + 'mbid', + 'title', + 'album', + 'artist', + 'creation_date', + 'position', + ) + + +class ManageTrackFileSerializer(serializers.ModelSerializer): + track = ManageTrackFileTrackSerializer() + + class Meta: + model = music_models.TrackFile + fields = ( + 'id', + 'path', + 'source', + 'filename', + 'mimetype', + 'track', + 'duration', + 'mimetype', + 'bitrate', + 'size', + 'path', + 'library_track', + ) + + +class ManageTrackFileActionSerializer(common_serializers.ActionSerializer): + actions = ['delete'] + filterset_class = filters.ManageTrackFileFilterSet + + @transaction.atomic + def handle_delete(self, objects): + return objects.delete() diff --git a/api/funkwhale_api/manage/urls.py b/api/funkwhale_api/manage/urls.py new file mode 100644 index 000000000..c434581ec --- /dev/null +++ b/api/funkwhale_api/manage/urls.py @@ -0,0 +1,11 @@ +from django.conf.urls import include, url +from . import views + +from rest_framework import routers +library_router = routers.SimpleRouter() +library_router.register(r'track-files', views.ManageTrackFileViewSet, 'track-files') + +urlpatterns = [ + url(r'^library/', + include((library_router.urls, 'instance'), namespace='library')), +] diff --git a/api/funkwhale_api/manage/views.py b/api/funkwhale_api/manage/views.py new file mode 100644 index 000000000..74059caa1 --- /dev/null +++ b/api/funkwhale_api/manage/views.py @@ -0,0 +1,49 @@ +from rest_framework import mixins +from rest_framework import response +from rest_framework import viewsets +from rest_framework.decorators import list_route + +from funkwhale_api.music import models as music_models +from funkwhale_api.users.permissions import HasUserPermission + +from . import filters +from . import serializers + + +class ManageTrackFileViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.DestroyModelMixin, + viewsets.GenericViewSet): + queryset = ( + music_models.TrackFile.objects.all() + .select_related( + 'track__artist', + 'track__album__artist', + 'library_track') + .order_by('-id') + ) + serializer_class = serializers.ManageTrackFileSerializer + filter_class = filters.ManageTrackFileFilterSet + permission_classes = (HasUserPermission,) + required_permissions = ['library'] + ordering_fields = [ + 'accessed_date', + 'modification_date', + 'creation_date', + 'track__artist__name', + 'bitrate', + 'size', + 'duration', + ] + + @list_route(methods=['post']) + def action(self, request, *args, **kwargs): + queryset = self.get_queryset() + serializer = serializers.ManageTrackFileActionSerializer( + request.data, + queryset=queryset, + ) + serializer.is_valid(raise_exception=True) + result = serializer.save() + return response.Response(result, status=200) diff --git a/api/funkwhale_api/music/factories.py b/api/funkwhale_api/music/factories.py index 412e2f798..11423f5b0 100644 --- a/api/funkwhale_api/music/factories.py +++ b/api/funkwhale_api/music/factories.py @@ -117,6 +117,11 @@ class ImportJobFactory(factory.django.DjangoModelFactory): status='finished', audio_file=None, ) + with_audio_file = factory.Trait( + status='finished', + audio_file=factory.django.FileField( + from_path=os.path.join(SAMPLES_PATH, 'test.ogg')), + ) @registry.register(name='music.FileImportJob') diff --git a/api/tests/manage/__init__.py b/api/tests/manage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/tests/manage/test_serializers.py b/api/tests/manage/test_serializers.py new file mode 100644 index 000000000..45167722c --- /dev/null +++ b/api/tests/manage/test_serializers.py @@ -0,0 +1,10 @@ +from funkwhale_api.manage import serializers + + +def test_manage_track_file_action_delete(factories): + tfs = factories['music.TrackFile'](size=5) + s = serializers.ManageTrackFileActionSerializer(queryset=None) + + s.handle_delete(tfs.__class__.objects.all()) + + assert tfs.__class__.objects.count() == 0 diff --git a/api/tests/manage/test_views.py b/api/tests/manage/test_views.py new file mode 100644 index 000000000..0507e6c11 --- /dev/null +++ b/api/tests/manage/test_views.py @@ -0,0 +1,25 @@ +import pytest + +from django.urls import reverse + +from funkwhale_api.manage import serializers +from funkwhale_api.manage import views + + +@pytest.mark.parametrize('view,permissions,operator', [ + (views.ManageTrackFileViewSet, ['library'], 'and'), +]) +def test_permissions(assert_user_permission, view, permissions, operator): + assert_user_permission(view, permissions, operator) + + +def test_track_file_view(factories, superuser_api_client): + tfs = factories['music.TrackFile'].create_batch(size=5) + qs = tfs[0].__class__.objects.order_by('-creation_date') + url = reverse('api:v1:manage:library:track-files-list') + + response = superuser_api_client.get(url, {'sort': '-creation_date'}) + expected = serializers.ManageTrackFileSerializer(qs, many=True).data + + assert response.data['count'] == len(tfs) + assert response.data['results'] == expected From 7df9112d551a77776a281755a73abc1c1f44fbc4 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Tue, 29 May 2018 00:07:38 +0200 Subject: [PATCH 3/8] See #223: front-end to browse/manage library files --- front/src/components/Sidebar.vue | 6 + .../components/manage/library/FilesTable.vue | 205 ++++++++++++++++++ front/src/router/index.js | 13 ++ front/src/views/admin/library/Base.vue | 28 +++ front/src/views/admin/library/FilesList.vue | 23 ++ 5 files changed, 275 insertions(+) create mode 100644 front/src/components/manage/library/FilesTable.vue create mode 100644 front/src/views/admin/library/Base.vue create mode 100644 front/src/views/admin/library/FilesList.vue diff --git a/front/src/components/Sidebar.vue b/front/src/components/Sidebar.vue index e8f330c38..72c55847f 100644 --- a/front/src/components/Sidebar.vue +++ b/front/src/components/Sidebar.vue @@ -68,6 +68,12 @@ :title="$t('Pending import requests')"> {{ notifications.importRequests }} + + {{ $t('Library') }} + +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+
+
+
+
+
+ + + + +
+
+ + + + {{ $t('Showing results {%start%}-{%end%} on {%total%}', {start: ((page-1) * paginateBy) + 1 , end: ((page-1) * paginateBy) + result.results.length, total: result.count})}} + +
+
+ + + diff --git a/front/src/router/index.js b/front/src/router/index.js index f71dab7f9..a52070e35 100644 --- a/front/src/router/index.js +++ b/front/src/router/index.js @@ -29,6 +29,8 @@ import PlaylistDetail from '@/views/playlists/Detail' import PlaylistList from '@/views/playlists/List' import Favorites from '@/components/favorites/List' import AdminSettings from '@/views/admin/Settings' +import AdminLibraryBase from '@/views/admin/library/Base' +import AdminLibraryFilesList from '@/views/admin/library/FilesList' import FederationBase from '@/views/federation/Base' import FederationScan from '@/views/federation/Scan' import FederationLibraryDetail from '@/views/federation/LibraryDetail' @@ -167,6 +169,17 @@ export default new Router({ { path: 'libraries/:id', name: 'federation.libraries.detail', component: FederationLibraryDetail, props: true } ] }, + { + path: '/manage/library', + component: AdminLibraryBase, + children: [ + { + path: 'files', + name: 'manage.library.files', + component: AdminLibraryFilesList + } + ] + }, { path: '/library', component: Library, diff --git a/front/src/views/admin/library/Base.vue b/front/src/views/admin/library/Base.vue new file mode 100644 index 000000000..834fca920 --- /dev/null +++ b/front/src/views/admin/library/Base.vue @@ -0,0 +1,28 @@ + + + + + diff --git a/front/src/views/admin/library/FilesList.vue b/front/src/views/admin/library/FilesList.vue new file mode 100644 index 000000000..9c52de576 --- /dev/null +++ b/front/src/views/admin/library/FilesList.vue @@ -0,0 +1,23 @@ + + + + + + From 7b84a988fdeee4a5002ccecf91a9b569cbc4a649 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Tue, 29 May 2018 21:55:00 +0200 Subject: [PATCH 4/8] See #223: dangerous actions can now prevent executing an action on all objects --- api/funkwhale_api/common/serializers.py | 7 +++++ api/tests/common/test_serializers.py | 36 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/api/funkwhale_api/common/serializers.py b/api/funkwhale_api/common/serializers.py index 62d9c567e..a995cc360 100644 --- a/api/funkwhale_api/common/serializers.py +++ b/api/funkwhale_api/common/serializers.py @@ -12,6 +12,9 @@ class ActionSerializer(serializers.Serializer): filters = serializers.DictField(required=False) actions = None filterset_class = None + # those are actions identifier where we don't want to allow the "all" + # selector because it's to dangerous. Like object deletion. + dangerous_actions = [] def __init__(self, *args, **kwargs): self.queryset = kwargs.pop('queryset') @@ -49,6 +52,10 @@ class ActionSerializer(serializers.Serializer): 'list of identifiers or the string "all".'.format(value)) def validate(self, data): + dangerous = data['action'] in self.dangerous_actions + if dangerous and self.initial_data['objects'] == 'all': + raise serializers.ValidationError( + 'This action is to dangerous to be applied to all objects') if self.filterset_class and 'filters' in data: qs_filterset = self.filterset_class( data['filters'], queryset=data['objects']) diff --git a/api/tests/common/test_serializers.py b/api/tests/common/test_serializers.py index 563676556..f0f5fb7e6 100644 --- a/api/tests/common/test_serializers.py +++ b/api/tests/common/test_serializers.py @@ -18,6 +18,17 @@ class TestSerializer(serializers.ActionSerializer): return {'hello': 'world'} +class TestDangerousSerializer(serializers.ActionSerializer): + actions = ['test', 'test_dangerous'] + dangerous_actions = ['test_dangerous'] + + def handle_test(self, objects): + pass + + def handle_test_dangerous(self, objects): + pass + + def test_action_serializer_validates_action(): data = {'objects': 'all', 'action': 'nope'} serializer = TestSerializer(data, queryset=models.User.objects.none()) @@ -98,3 +109,28 @@ def test_action_serializers_validates_at_least_one_object(): assert serializer.is_valid() is False assert 'non_field_errors' in serializer.errors + + +def test_dangerous_actions_refuses_all(factories): + factories['users.User']() + data = { + 'objects': 'all', + 'action': 'test_dangerous', + } + serializer = TestDangerousSerializer( + data, queryset=models.User.objects.all()) + + assert serializer.is_valid() is False + assert 'non_field_errors' in serializer.errors + + +def test_dangerous_actions_refuses_not_listed(factories): + factories['users.User']() + data = { + 'objects': 'all', + 'action': 'test', + } + serializer = TestDangerousSerializer( + data, queryset=models.User.objects.all()) + + assert serializer.is_valid() is True From 256d98b77d7cf6e490b9950ea54a84ad99c544d2 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Tue, 29 May 2018 22:22:51 +0200 Subject: [PATCH 5/8] See #223: delete actions is now dangerous anc cannot be applied to all files --- api/funkwhale_api/manage/serializers.py | 1 + front/src/components/common/ActionTable.vue | 4 ++-- front/src/components/manage/library/FilesTable.vue | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/funkwhale_api/manage/serializers.py b/api/funkwhale_api/manage/serializers.py index bbd393236..02300ec06 100644 --- a/api/funkwhale_api/manage/serializers.py +++ b/api/funkwhale_api/manage/serializers.py @@ -74,6 +74,7 @@ class ManageTrackFileSerializer(serializers.ModelSerializer): class ManageTrackFileActionSerializer(common_serializers.ActionSerializer): actions = ['delete'] + dangerous_actions = ['delete'] filterset_class = filters.ManageTrackFileFilterSet @transaction.atomic diff --git a/front/src/components/common/ActionTable.vue b/front/src/components/common/ActionTable.vue index 718e57b19..5221c3282 100644 --- a/front/src/components/common/ActionTable.vue +++ b/front/src/components/common/ActionTable.vue @@ -21,7 +21,7 @@ :class="['ui', {disabled: checked.length === 0}, {'loading': actionLoading}, 'button']"> {{ $t('Go') }} @@ -36,7 +36,7 @@
{{ $t('{% count %} on {% total %} selected', {count: objectsData.count, total: objectsData.count}) }} {{ $t('{% count %} on {% total %} selected', {count: checked.length, total: objectsData.count}) }} -