From 788c12748fc6a269f2d0f0a19a6744bae16ced3e Mon Sep 17 00:00:00 2001 From: Agate Date: Mon, 3 Aug 2020 10:16:25 +0200 Subject: [PATCH] Fix #1105: Can now launch server import from the UI --- .../music/management/commands/import_files.py | 50 +++++++++- api/funkwhale_api/music/serializers.py | 20 ++++ api/funkwhale_api/music/tasks.py | 34 ++++++- api/funkwhale_api/music/utils.py | 20 ++++ api/funkwhale_api/music/views.py | 59 ++++++++++++ api/setup.cfg | 1 + api/tests/music/test_tasks.py | 36 ++++++++ api/tests/music/test_utils.py | 20 +++- api/tests/music/test_views.py | 66 ++++++++++++++ changes/changelog.d/1105.feature | 1 + front/scripts/fix-fomantic-css.py | 1 + front/src/components/library/FileUpload.vue | 91 ++++++++++++++++++- front/src/components/library/FsBrowser.vue | 54 +++++++++++ front/src/components/library/FsLogs.vue | 17 ++++ front/src/style/_main.scss | 2 + front/src/style/components/_fs_browser.scss | 4 + front/src/style/components/_fs_logs.scss | 6 ++ 17 files changed, 476 insertions(+), 6 deletions(-) create mode 100644 changes/changelog.d/1105.feature create mode 100644 front/src/components/library/FsBrowser.vue create mode 100644 front/src/components/library/FsLogs.vue create mode 100644 front/src/style/components/_fs_browser.scss create mode 100644 front/src/style/components/_fs_logs.scss diff --git a/api/funkwhale_api/music/management/commands/import_files.py b/api/funkwhale_api/music/management/commands/import_files.py index 0d44af49c..c2f94da5a 100644 --- a/api/funkwhale_api/music/management/commands/import_files.py +++ b/api/funkwhale_api/music/management/commands/import_files.py @@ -12,6 +12,7 @@ import watchdog.events import watchdog.observers from django.conf import settings +from django.core.cache import cache from django.core.files import File from django.core.management import call_command from django.core.management.base import BaseCommand, CommandError @@ -68,8 +69,34 @@ def batch(iterable, n=1): yield current +class CacheWriter: + """ + Output to cache instead of console + """ + + def __init__(self, key, stdout, buffer_size=10): + self.key = key + cache.set(self.key, []) + self.stdout = stdout + self.buffer_size = buffer_size + self.buffer = [] + + def write(self, message): + # we redispatch the message to the console, for debugging + self.stdout.write(message) + + self.buffer.append(message) + if len(self.buffer) > self.buffer_size: + self.flush() + + def flush(self): + current = cache.get(self.key) + cache.set(self.key, current + self.buffer) + self.buffer = [] + + class Command(BaseCommand): - help = "Import audio files mathinc given glob pattern" + help = "Import audio files matching given glob pattern" def add_arguments(self, parser): parser.add_argument( @@ -207,7 +234,22 @@ class Command(BaseCommand): help="Size of each batch, only used when crawling large collections", ) - def handle(self, *args, **options): + def handle(self, *args, **kwargs): + cache.set("fs-import:status", "started") + if kwargs.get("update_cache", False): + self.stdout = CacheWriter("fs-import:logs", self.stdout) + self.stderr = self.stdout + try: + return self._handle(*args, **kwargs) + except CommandError as e: + self.stdout.write(str(e)) + raise + finally: + if kwargs.get("update_cache", False): + cache.set("fs-import:status", "finished") + self.stdout.flush() + + def _handle(self, *args, **options): # handle relative directories options["path"] = [os.path.abspath(path) for path in options["path"]] self.is_confirmed = False @@ -312,6 +354,10 @@ class Command(BaseCommand): batch_duration = None self.stdout.write("Starting import of new files…") for i, entries in enumerate(batch(crawler, options["batch_size"])): + if options.get("update_cache", False) is True: + # check to see if the scan was cancelled + if cache.get("fs-import:status") == "canceled": + raise CommandError("Import cancelled") total += len(entries) batch_start = time.time() time_stats = "" diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index 124c57f62..c5e4336a6 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -838,3 +838,23 @@ class AlbumCreateSerializer(serializers.Serializer): tag_models.set_tags(instance, *(validated_data.get("tags", []) or [])) instance.artist.get_channel() return instance + + +class FSImportSerializer(serializers.Serializer): + path = serializers.CharField(allow_blank=True) + library = serializers.UUIDField() + import_reference = serializers.CharField() + + def validate_path(self, value): + try: + utils.browse_dir(settings.MUSIC_DIRECTORY_PATH, value) + except (NotADirectoryError, FileNotFoundError, ValueError): + raise serializers.ValidationError("Invalid path") + + return value + + def validate_library(self, value): + try: + return self.context["user"].actor.libraries.get(uuid=value) + except models.Library.DoesNotExist: + raise serializers.ValidationError("Invalid library") diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py index e44888a1f..2e633925b 100644 --- a/api/funkwhale_api/music/tasks.py +++ b/api/funkwhale_api/music/tasks.py @@ -3,10 +3,12 @@ import datetime import logging import os -from django.utils import timezone +from django.conf import settings +from django.core.cache import cache from django.db import transaction from django.db.models import F, Q from django.dispatch import receiver +from django.utils import timezone from musicbrainzngs import ResponseError from requests.exceptions import RequestException @@ -17,6 +19,7 @@ from funkwhale_api.common import utils as common_utils from funkwhale_api.federation import routes from funkwhale_api.federation import library as lb from funkwhale_api.federation import utils as federation_utils +from funkwhale_api.music.management.commands import import_files from funkwhale_api.tags import models as tags_models from funkwhale_api.tags import tasks as tags_tasks from funkwhale_api.taskapp import celery @@ -938,3 +941,32 @@ def update_track_metadata(audio_metadata, track): common_utils.attach_file( track.album, "attachment_cover", new_data["album"].get("cover_data") ) + + +@celery.app.task(name="music.fs_import") +@celery.require_instance(models.Library.objects.all(), "library") +def fs_import(library, path, import_reference): + if cache.get("fs-import:status") != "pending": + raise ValueError("Invalid import status") + + command = import_files.Command() + + options = { + "recursive": True, + "library_id": str(library.uuid), + "path": [os.path.join(settings.MUSIC_DIRECTORY_PATH, path)], + "update_cache": True, + "in_place": True, + "reference": import_reference, + "watch": False, + "interactive": False, + "batch_size": 1000, + "async_": False, + "prune": True, + "replace": False, + "verbosity": 1, + "exit_on_failure": False, + "outbox": False, + "broadcast": False, + } + command.handle(**options) diff --git a/api/funkwhale_api/music/utils.py b/api/funkwhale_api/music/utils.py index b61c8223b..17b5b7292 100644 --- a/api/funkwhale_api/music/utils.py +++ b/api/funkwhale_api/music/utils.py @@ -1,3 +1,5 @@ +import os +import pathlib import mimetypes import magic @@ -130,3 +132,21 @@ def increment_downloads_count(upload, user, wsgi_request): duration = max(upload.duration or 0, settings.MIN_DELAY_BETWEEN_DOWNLOADS_COUNT) cache.set(cache_key, 1, duration) + + +def browse_dir(root, path): + if ".." in path: + raise ValueError("Relative browsing is not allowed") + + root = pathlib.Path(root) + real_path = root / path + + dirs = [] + files = [] + for el in sorted(os.listdir(real_path)): + if os.path.isdir(real_path / el): + dirs.append({"name": el, "dir": True}) + else: + files.append({"name": el, "dir": False}) + + return dirs + files diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index 5dab86abd..18c2b30aa 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -3,6 +3,7 @@ import datetime import logging import urllib.parse from django.conf import settings +from django.core.cache import cache from django.db import transaction from django.db.models import Count, Prefetch, Sum, F, Q import django.db.utils @@ -314,6 +315,64 @@ class LibraryViewSet( serializer = self.get_serializer(queryset, many=True) return Response(serializer.data) + @action( + methods=["get", "post", "delete"], + detail=False, + url_name="fs-import", + url_path="fs-import", + ) + @transaction.non_atomic_requests + def fs_import(self, request, *args, **kwargs): + if not request.user.is_authenticated: + return Response({}, status=403) + if not request.user.all_permissions["library"]: + return Response({}, status=403) + if request.method == "GET": + path = request.GET.get("path", "") + data = { + "root": settings.MUSIC_DIRECTORY_PATH, + "path": path, + "import": None, + } + status = cache.get("fs-import:status", default=None) + if status: + data["import"] = { + "status": status, + "reference": cache.get("fs-import:reference"), + "logs": cache.get("fs-import:logs", default=[]), + } + try: + data["content"] = utils.browse_dir(data["root"], data["path"]) + except (NotADirectoryError, ValueError, FileNotFoundError) as e: + return Response({"detail": str(e)}, status=400) + + return Response(data) + if request.method == "POST": + if cache.get("fs-import:status", default=None) in [ + "pending", + "started", + ]: + return Response({"detail": "An import is already running"}, status=400) + + data = request.data + serializer = serializers.FSImportSerializer( + data=data, context={"user": request.user} + ) + serializer.is_valid(raise_exception=True) + cache.set("fs-import:status", "pending") + cache.set( + "fs-import:reference", serializer.validated_data["import_reference"] + ) + tasks.fs_import.delay( + library_id=serializer.validated_data["library"].pk, + path=serializer.validated_data["path"], + import_reference=serializer.validated_data["import_reference"], + ) + return Response(status=201) + if request.method == "DELETE": + cache.set("fs-import:status", "canceled") + return Response(status=204) + class TrackViewSet( HandleInvalidSearch, diff --git a/api/setup.cfg b/api/setup.cfg index 724d1c87f..1f09309bd 100644 --- a/api/setup.cfg +++ b/api/setup.cfg @@ -36,3 +36,4 @@ env = DISABLE_PASSWORD_VALIDATORS=false DISABLE_PASSWORD_VALIDATORS=false FUNKWHALE_PLUGINS= + MUSIC_DIRECTORY_PATH=/music diff --git a/api/tests/music/test_tasks.py b/api/tests/music/test_tasks.py index 6aecd35fe..edb937d81 100644 --- a/api/tests/music/test_tasks.py +++ b/api/tests/music/test_tasks.py @@ -1382,3 +1382,39 @@ def test_update_track_metadata(factories): assert str(track.artist.mbid) == data["musicbrainz_artistid"] assert track.album.artist.name == "Edvard Grieg" assert str(track.album.artist.mbid) == "013c8e5b-d72a-4cd3-8dee-6c64d6125823" + + +def test_fs_import_not_pending(factories): + with pytest.raises(ValueError): + tasks.fs_import( + library_id=factories["music.Library"]().pk, + path="path", + import_reference="test", + ) + + +def test_fs_import(factories, cache, mocker, settings): + _handle = mocker.spy(tasks.import_files.Command, "_handle") + cache.set("fs-import:status", "pending") + library = factories["music.Library"](actor__local=True) + tasks.fs_import(library_id=library.pk, path="path", import_reference="test") + assert _handle.call_args[1] == { + "recursive": True, + "path": [settings.MUSIC_DIRECTORY_PATH + "/path"], + "library_id": str(library.uuid), + "update_cache": True, + "in_place": True, + "reference": "test", + "watch": False, + "interactive": False, + "batch_size": 1000, + "async_": False, + "prune": True, + "broadcast": False, + "outbox": False, + "exit_on_failure": False, + "replace": False, + "verbosity": 1, + } + assert cache.get("fs-import:status") == "finished" + assert "Pruning dangling tracks" in cache.get("fs-import:logs")[-1] diff --git a/api/tests/music/test_utils.py b/api/tests/music/test_utils.py index c05b530aa..67a4212d8 100644 --- a/api/tests/music/test_utils.py +++ b/api/tests/music/test_utils.py @@ -1,5 +1,5 @@ import os - +import pathlib import pytest from funkwhale_api.music import utils @@ -91,3 +91,21 @@ def test_increment_downloads_count_already_tracked( assert upload.downloads_count == 0 assert upload.track.downloads_count == 0 + + +@pytest.mark.parametrize( + "path,expected", + [ + ("", [{"name": "Magic", "dir": True}, {"name": "System", "dir": True}]), + ("Magic", [{"name": "file.mp3", "dir": False}]), + ("System", [{"name": "file.ogg", "dir": False}]), + ], +) +def test_get_dirs_and_files(path, expected, tmpdir): + root_path = pathlib.Path(tmpdir) + (root_path / "Magic").mkdir() + (root_path / "Magic" / "file.mp3").touch() + (root_path / "System").mkdir() + (root_path / "System" / "file.ogg").touch() + + assert utils.browse_dir(root_path, path) == expected diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py index 2fa20aa1d..7e96f46c0 100644 --- a/api/tests/music/test_views.py +++ b/api/tests/music/test_views.py @@ -2,6 +2,7 @@ import datetime import io import magic import os +import pathlib import urllib.parse import uuid @@ -1514,3 +1515,68 @@ def test_listen_to_track_with_scoped_token(factories, api_client): response = api_client.get(url, {"token": token}) assert response.status_code == 200 + + +def test_fs_import_get(factories, superuser_api_client, mocker, settings): + browse_dir = mocker.patch.object( + views.utils, "browse_dir", return_value={"hello": "world"} + ) + url = reverse("api:v1:libraries-fs-import") + + expected = { + "root": settings.MUSIC_DIRECTORY_PATH, + "path": "", + "content": {"hello": "world"}, + "import": None, + } + response = superuser_api_client.get(url, {"path": ""}) + + assert response.status_code == 200 + assert response.data == expected + browse_dir.assert_called_once_with(expected["root"], expected["path"]) + + +def test_fs_import_post( + factories, superuser_api_client, cache, mocker, settings, tmpdir +): + actor = superuser_api_client.user.create_actor() + library = factories["music.Library"](actor=actor) + settings.MUSIC_DIRECTORY_PATH = tmpdir + (pathlib.Path(tmpdir) / "test").mkdir() + fs_import = mocker.patch( + "funkwhale_api.music.tasks.fs_import.delay", return_value={"hello": "world"} + ) + url = reverse("api:v1:libraries-fs-import") + + response = superuser_api_client.post( + url, {"path": "test", "library": library.uuid, "import_reference": "test"} + ) + + assert response.status_code == 201 + fs_import.assert_called_once_with( + path="test", library_id=library.pk, import_reference="test" + ) + assert cache.get("fs-import:status") == "pending" + + +def test_fs_import_post_already_running( + factories, superuser_api_client, cache, mocker, settings, tmpdir +): + url = reverse("api:v1:libraries-fs-import") + cache.set("fs-import:status", "pending") + + response = superuser_api_client.post(url, {"path": "test"}) + + assert response.status_code == 400 + + +def test_fs_import_cancel_already_running( + factories, superuser_api_client, cache, mocker, settings, tmpdir +): + url = reverse("api:v1:libraries-fs-import") + cache.set("fs-import:status", "pending") + + response = superuser_api_client.delete(url) + + assert response.status_code == 204 + assert cache.get("fs-import:status") == "canceled" diff --git a/changes/changelog.d/1105.feature b/changes/changelog.d/1105.feature new file mode 100644 index 000000000..152c2a7be --- /dev/null +++ b/changes/changelog.d/1105.feature @@ -0,0 +1 @@ +Can now launch server import from the UI (#1105) \ No newline at end of file diff --git a/front/scripts/fix-fomantic-css.py b/front/scripts/fix-fomantic-css.py index 692383f59..095f66a08 100755 --- a/front/scripts/fix-fomantic-css.py +++ b/front/scripts/fix-fomantic-css.py @@ -114,6 +114,7 @@ def discard_unused_icons(rule): ".eye", ".feed", ".file", + ".folder", ".forward", ".globe", ".hashtag", diff --git a/front/src/components/library/FileUpload.vue b/front/src/components/library/FileUpload.vue index e4dd6a4a7..6d7ac605e 100644 --- a/front/src/components/library/FileUpload.vue +++ b/front/src/components/library/FileUpload.vue @@ -56,6 +56,40 @@ + +
@@ -163,6 +197,8 @@ import $ from "jquery"; import axios from "axios"; import logger from "@/logging"; import FileUploadWidget from "./FileUploadWidget"; +import FsBrowser from "./FsBrowser"; +import FsLogs from "./FsLogs"; import LibraryFilesTable from "@/views/content/libraries/FilesTable"; import moment from "moment"; @@ -170,7 +206,9 @@ export default { props: ["library", "defaultImportReference"], components: { FileUploadWidget, - LibraryFilesTable + LibraryFilesTable, + FsBrowser, + FsLogs, }, data() { let importReference = this.defaultImportReference || moment().format(); @@ -190,11 +228,22 @@ export default { errored: 0, objects: {} }, - processTimestamp: new Date() + processTimestamp: new Date(), + fsStatus: null, + fsPath: [], + isLoadingFs: false, + fsInterval: null, + fsErrors: [] }; }, created() { this.fetchStatus(); + if (this.$store.state.auth.availablePermissions['library']) { + this.fetchFs(true) + setInterval(() => { + this.fetchFs(false) + }, 5000); + } this.fetchQuota(); this.$store.commit("ui/addWebsocketEventHandler", { eventName: "import.status_updated", @@ -209,6 +258,9 @@ export default { id: "fileUpload" }); window.onbeforeunload = null; + if (this.fsInterval) { + clearInterval(this.fsInterval) + } }, methods: { onBeforeUnload(e = {}) { @@ -227,6 +279,38 @@ export default { self.isLoadingQuota = false }) }, + fetchFs (updateLoading) { + let self = this + if (updateLoading) { + self.isLoadingFs = true + } + axios.get('libraries/fs-import', {params: {path: this.fsPath.join('/')}}).then((response) => { + self.fsStatus = response.data + if (updateLoading) { + self.isLoadingFs = false + } + }) + }, + importFs () { + let self = this + self.isLoadingFs = true + let payload = { + path: this.fsPath.join('/'), + library: this.library.uuid, + import_reference: this.importReference, + } + axios.post('libraries/fs-import', payload).then((response) => { + self.fsStatus = response.data + self.isLoadingFs = false + }, error => { + self.isLoadingFs = false + self.fsErrors = error.backendErrors + }) + }, + async cancelFsScan () { + await axios.delete('libraries/fs-import') + this.fetchFs() + }, inputFile(newFile, oldFile) { if (!newFile) { return @@ -392,6 +476,9 @@ export default { if (v > o) { this.$emit('uploads-finished', v - o) } + }, + "fsPath" () { + this.fetchFs(true) } } }; diff --git a/front/src/components/library/FsBrowser.vue b/front/src/components/library/FsBrowser.vue new file mode 100644 index 000000000..6140475b7 --- /dev/null +++ b/front/src/components/library/FsBrowser.vue @@ -0,0 +1,54 @@ + + \ No newline at end of file diff --git a/front/src/components/library/FsLogs.vue b/front/src/components/library/FsLogs.vue new file mode 100644 index 000000000..8c26ca49e --- /dev/null +++ b/front/src/components/library/FsLogs.vue @@ -0,0 +1,17 @@ + + \ No newline at end of file diff --git a/front/src/style/_main.scss b/front/src/style/_main.scss index 95997ca56..5e41ffc14 100644 --- a/front/src/style/_main.scss +++ b/front/src/style/_main.scss @@ -26,6 +26,8 @@ $bottom-player-height: 4rem; @import "./components/_empty_state.scss"; @import "./components/_form.scss"; @import "./components/_file_upload.scss"; +@import "./components/_fs_browser.scss"; +@import "./components/_fs_logs.scss"; @import "./components/_header.scss"; @import "./components/_label.scss"; @import "./components/_modal.scss"; diff --git a/front/src/style/components/_fs_browser.scss b/front/src/style/components/_fs_browser.scss new file mode 100644 index 000000000..e55cbe8cb --- /dev/null +++ b/front/src/style/components/_fs_browser.scss @@ -0,0 +1,4 @@ +.component-fs-browser { + max-height: 400px; + overflow-y: auto; +} \ No newline at end of file diff --git a/front/src/style/components/_fs_logs.scss b/front/src/style/components/_fs_logs.scss new file mode 100644 index 000000000..0f5778714 --- /dev/null +++ b/front/src/style/components/_fs_logs.scss @@ -0,0 +1,6 @@ +.component-fs-logs { + max-height: 300px; + overflow-y: auto; + background-color: rgba(25, 25, 25) !important; + color: white !important; +} \ No newline at end of file