Merge branch '1105-scan-ui' into 'develop'

Fix #1105: Can now launch server import from the UI

Closes #1105

See merge request funkwhale/funkwhale!1192

api/funkwhale_api/music/management/commands/import_files.py

@@ -12,6 +12,7 @@ import watchdog.events
 import watchdog.observers
 
 from django.conf import settings
+from django.core.cache import cache
 from django.core.files import File
 from django.core.management import call_command
 from django.core.management.base import BaseCommand, CommandError
@@ -68,8 +69,34 @@ def batch(iterable, n=1):
         yield current
 
 
+class CacheWriter:
+    """
+    Output to cache instead of console
+    """
+
+    def __init__(self, key, stdout, buffer_size=10):
+        self.key = key
+        cache.set(self.key, [])
+        self.stdout = stdout
+        self.buffer_size = buffer_size
+        self.buffer = []
+
+    def write(self, message):
+        # we redispatch the message to the console, for debugging
+        self.stdout.write(message)
+
+        self.buffer.append(message)
+        if len(self.buffer) > self.buffer_size:
+            self.flush()
+
+    def flush(self):
+        current = cache.get(self.key)
+        cache.set(self.key, current + self.buffer)
+        self.buffer = []
+
+
 class Command(BaseCommand):
-    help = "Import audio files mathinc given glob pattern"
+    help = "Import audio files matching given glob pattern"
 
     def add_arguments(self, parser):
         parser.add_argument(
@@ -207,7 +234,22 @@ class Command(BaseCommand):
             help="Size of each batch, only used when crawling large collections",
         )
 
-    def handle(self, *args, **options):
+    def handle(self, *args, **kwargs):
+        cache.set("fs-import:status", "started")
+        if kwargs.get("update_cache", False):
+            self.stdout = CacheWriter("fs-import:logs", self.stdout)
+            self.stderr = self.stdout
+        try:
+            return self._handle(*args, **kwargs)
+        except CommandError as e:
+            self.stdout.write(str(e))
+            raise
+        finally:
+            if kwargs.get("update_cache", False):
+                cache.set("fs-import:status", "finished")
+                self.stdout.flush()
+
+    def _handle(self, *args, **options):
         # handle relative directories
         options["path"] = [os.path.abspath(path) for path in options["path"]]
         self.is_confirmed = False
@@ -312,6 +354,10 @@ class Command(BaseCommand):
         batch_duration = None
         self.stdout.write("Starting import of new files…")
         for i, entries in enumerate(batch(crawler, options["batch_size"])):
+            if options.get("update_cache", False) is True:
+                # check to see if the scan was cancelled
+                if cache.get("fs-import:status") == "canceled":
+                    raise CommandError("Import cancelled")
             total += len(entries)
             batch_start = time.time()
             time_stats = ""

api/funkwhale_api/music/serializers.py

@@ -838,3 +838,23 @@ class AlbumCreateSerializer(serializers.Serializer):
         tag_models.set_tags(instance, *(validated_data.get("tags", []) or []))
         instance.artist.get_channel()
         return instance
+
+
+class FSImportSerializer(serializers.Serializer):
+    path = serializers.CharField(allow_blank=True)
+    library = serializers.UUIDField()
+    import_reference = serializers.CharField()
+
+    def validate_path(self, value):
+        try:
+            utils.browse_dir(settings.MUSIC_DIRECTORY_PATH, value)
+        except (NotADirectoryError, FileNotFoundError, ValueError):
+            raise serializers.ValidationError("Invalid path")
+
+        return value
+
+    def validate_library(self, value):
+        try:
+            return self.context["user"].actor.libraries.get(uuid=value)
+        except models.Library.DoesNotExist:
+            raise serializers.ValidationError("Invalid library")

api/funkwhale_api/music/tasks.py

@@ -3,10 +3,12 @@ import datetime
 import logging
 import os
 
-from django.utils import timezone
+from django.conf import settings
+from django.core.cache import cache
 from django.db import transaction
 from django.db.models import F, Q
 from django.dispatch import receiver
+from django.utils import timezone
 
 from musicbrainzngs import ResponseError
 from requests.exceptions import RequestException
@@ -17,6 +19,7 @@ from funkwhale_api.common import utils as common_utils
 from funkwhale_api.federation import routes
 from funkwhale_api.federation import library as lb
 from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.music.management.commands import import_files
 from funkwhale_api.tags import models as tags_models
 from funkwhale_api.tags import tasks as tags_tasks
 from funkwhale_api.taskapp import celery
@@ -938,3 +941,32 @@ def update_track_metadata(audio_metadata, track):
         common_utils.attach_file(
             track.album, "attachment_cover", new_data["album"].get("cover_data")
         )
+
+
+@celery.app.task(name="music.fs_import")
+@celery.require_instance(models.Library.objects.all(), "library")
+def fs_import(library, path, import_reference):
+    if cache.get("fs-import:status") != "pending":
+        raise ValueError("Invalid import status")
+
+    command = import_files.Command()
+
+    options = {
+        "recursive": True,
+        "library_id": str(library.uuid),
+        "path": [os.path.join(settings.MUSIC_DIRECTORY_PATH, path)],
+        "update_cache": True,
+        "in_place": True,
+        "reference": import_reference,
+        "watch": False,
+        "interactive": False,
+        "batch_size": 1000,
+        "async_": False,
+        "prune": True,
+        "replace": False,
+        "verbosity": 1,
+        "exit_on_failure": False,
+        "outbox": False,
+        "broadcast": False,
+    }
+    command.handle(**options)

api/funkwhale_api/music/utils.py

@@ -1,3 +1,5 @@
+import os
+import pathlib
 import mimetypes
 
 import magic
@@ -130,3 +132,21 @@ def increment_downloads_count(upload, user, wsgi_request):
     duration = max(upload.duration or 0, settings.MIN_DELAY_BETWEEN_DOWNLOADS_COUNT)
 
     cache.set(cache_key, 1, duration)
+
+
+def browse_dir(root, path):
+    if ".." in path:
+        raise ValueError("Relative browsing is not allowed")
+
+    root = pathlib.Path(root)
+    real_path = root / path
+
+    dirs = []
+    files = []
+    for el in sorted(os.listdir(real_path)):
+        if os.path.isdir(real_path / el):
+            dirs.append({"name": el, "dir": True})
+        else:
+            files.append({"name": el, "dir": False})
+
+    return dirs + files

api/funkwhale_api/music/views.py

@@ -3,6 +3,7 @@ import datetime
 import logging
 import urllib.parse
 from django.conf import settings
+from django.core.cache import cache
 from django.db import transaction
 from django.db.models import Count, Prefetch, Sum, F, Q
 import django.db.utils
@@ -314,6 +315,64 @@ class LibraryViewSet(
         serializer = self.get_serializer(queryset, many=True)
         return Response(serializer.data)
 
+    @action(
+        methods=["get", "post", "delete"],
+        detail=False,
+        url_name="fs-import",
+        url_path="fs-import",
+    )
+    @transaction.non_atomic_requests
+    def fs_import(self, request, *args, **kwargs):
+        if not request.user.is_authenticated:
+            return Response({}, status=403)
+        if not request.user.all_permissions["library"]:
+            return Response({}, status=403)
+        if request.method == "GET":
+            path = request.GET.get("path", "")
+            data = {
+                "root": settings.MUSIC_DIRECTORY_PATH,
+                "path": path,
+                "import": None,
+            }
+            status = cache.get("fs-import:status", default=None)
+            if status:
+                data["import"] = {
+                    "status": status,
+                    "reference": cache.get("fs-import:reference"),
+                    "logs": cache.get("fs-import:logs", default=[]),
+                }
+            try:
+                data["content"] = utils.browse_dir(data["root"], data["path"])
+            except (NotADirectoryError, ValueError, FileNotFoundError) as e:
+                return Response({"detail": str(e)}, status=400)
+
+            return Response(data)
+        if request.method == "POST":
+            if cache.get("fs-import:status", default=None) in [
+                "pending",
+                "started",
+            ]:
+                return Response({"detail": "An import is already running"}, status=400)
+
+            data = request.data
+            serializer = serializers.FSImportSerializer(
+                data=data, context={"user": request.user}
+            )
+            serializer.is_valid(raise_exception=True)
+            cache.set("fs-import:status", "pending")
+            cache.set(
+                "fs-import:reference", serializer.validated_data["import_reference"]
+            )
+            tasks.fs_import.delay(
+                library_id=serializer.validated_data["library"].pk,
+                path=serializer.validated_data["path"],
+                import_reference=serializer.validated_data["import_reference"],
+            )
+            return Response(status=201)
+        if request.method == "DELETE":
+            cache.set("fs-import:status", "canceled")
+            return Response(status=204)
+
 
 class TrackViewSet(
     HandleInvalidSearch,

api/setup.cfg

@@ -36,3 +36,4 @@ env =
     DISABLE_PASSWORD_VALIDATORS=false
     DISABLE_PASSWORD_VALIDATORS=false
     FUNKWHALE_PLUGINS=
+    MUSIC_DIRECTORY_PATH=/music

api/tests/music/test_tasks.py

@@ -1382,3 +1382,39 @@ def test_update_track_metadata(factories):
     assert str(track.artist.mbid) == data["musicbrainz_artistid"]
     assert track.album.artist.name == "Edvard Grieg"
     assert str(track.album.artist.mbid) == "013c8e5b-d72a-4cd3-8dee-6c64d6125823"
+
+
+def test_fs_import_not_pending(factories):
+    with pytest.raises(ValueError):
+        tasks.fs_import(
+            library_id=factories["music.Library"]().pk,
+            path="path",
+            import_reference="test",
+        )
+
+
+def test_fs_import(factories, cache, mocker, settings):
+    _handle = mocker.spy(tasks.import_files.Command, "_handle")
+    cache.set("fs-import:status", "pending")
+    library = factories["music.Library"](actor__local=True)
+    tasks.fs_import(library_id=library.pk, path="path", import_reference="test")
+    assert _handle.call_args[1] == {
+        "recursive": True,
+        "path": [settings.MUSIC_DIRECTORY_PATH + "/path"],
+        "library_id": str(library.uuid),
+        "update_cache": True,
+        "in_place": True,
+        "reference": "test",
+        "watch": False,
+        "interactive": False,
+        "batch_size": 1000,
+        "async_": False,
+        "prune": True,
+        "broadcast": False,
+        "outbox": False,
+        "exit_on_failure": False,
+        "replace": False,
+        "verbosity": 1,
+    }
+    assert cache.get("fs-import:status") == "finished"
+    assert "Pruning dangling tracks" in cache.get("fs-import:logs")[-1]

api/tests/music/test_utils.py

@@ -1,5 +1,5 @@
 import os
-
+import pathlib
 import pytest
 
 from funkwhale_api.music import utils
@@ -91,3 +91,21 @@ def test_increment_downloads_count_already_tracked(
 
     assert upload.downloads_count == 0
     assert upload.track.downloads_count == 0
+
+
+@pytest.mark.parametrize(
+    "path,expected",
+    [
+        ("", [{"name": "Magic", "dir": True}, {"name": "System", "dir": True}]),
+        ("Magic", [{"name": "file.mp3", "dir": False}]),
+        ("System", [{"name": "file.ogg", "dir": False}]),
+    ],
+)
+def test_get_dirs_and_files(path, expected, tmpdir):
+    root_path = pathlib.Path(tmpdir)
+    (root_path / "Magic").mkdir()
+    (root_path / "Magic" / "file.mp3").touch()
+    (root_path / "System").mkdir()
+    (root_path / "System" / "file.ogg").touch()
+
+    assert utils.browse_dir(root_path, path) == expected

api/tests/music/test_views.py

@@ -2,6 +2,7 @@ import datetime
 import io
 import magic
 import os
+import pathlib
 import urllib.parse
 import uuid
 
@@ -1514,3 +1515,68 @@ def test_listen_to_track_with_scoped_token(factories, api_client):
     response = api_client.get(url, {"token": token})
 
     assert response.status_code == 200
+
+
+def test_fs_import_get(factories, superuser_api_client, mocker, settings):
+    browse_dir = mocker.patch.object(
+        views.utils, "browse_dir", return_value={"hello": "world"}
+    )
+    url = reverse("api:v1:libraries-fs-import")
+
+    expected = {
+        "root": settings.MUSIC_DIRECTORY_PATH,
+        "path": "",
+        "content": {"hello": "world"},
+        "import": None,
+    }
+    response = superuser_api_client.get(url, {"path": ""})
+
+    assert response.status_code == 200
+    assert response.data == expected
+    browse_dir.assert_called_once_with(expected["root"], expected["path"])
+
+
+def test_fs_import_post(
+    factories, superuser_api_client, cache, mocker, settings, tmpdir
+):
+    actor = superuser_api_client.user.create_actor()
+    library = factories["music.Library"](actor=actor)
+    settings.MUSIC_DIRECTORY_PATH = tmpdir
+    (pathlib.Path(tmpdir) / "test").mkdir()
+    fs_import = mocker.patch(
+        "funkwhale_api.music.tasks.fs_import.delay", return_value={"hello": "world"}
+    )
+    url = reverse("api:v1:libraries-fs-import")
+
+    response = superuser_api_client.post(
+        url, {"path": "test", "library": library.uuid, "import_reference": "test"}
+    )
+
+    assert response.status_code == 201
+    fs_import.assert_called_once_with(
+        path="test", library_id=library.pk, import_reference="test"
+    )
+    assert cache.get("fs-import:status") == "pending"
+
+
+def test_fs_import_post_already_running(
+    factories, superuser_api_client, cache, mocker, settings, tmpdir
+):
+    url = reverse("api:v1:libraries-fs-import")
+    cache.set("fs-import:status", "pending")
+
+    response = superuser_api_client.post(url, {"path": "test"})
+
+    assert response.status_code == 400
+
+
+def test_fs_import_cancel_already_running(
+    factories, superuser_api_client, cache, mocker, settings, tmpdir
+):
+    url = reverse("api:v1:libraries-fs-import")
+    cache.set("fs-import:status", "pending")
+
+    response = superuser_api_client.delete(url)
+
+    assert response.status_code == 204
+    assert cache.get("fs-import:status") == "canceled"

changes/changelog.d/1105.feature

@@ -0,0 +1 @@
+Can now launch server import from the UI (#1105)

front/scripts/fix-fomantic-css.py

@@ -114,6 +114,7 @@ def discard_unused_icons(rule):
         ".eye",
         ".feed",
         ".file",
+        ".folder",
         ".forward",
         ".globe",
         ".hashtag",

front/src/components/library/FileUpload.vue

@@ -56,6 +56,40 @@
 
         <button type="submit" class="ui success button"><translate translate-context="Content/Library/Button.Label">Proceed</translate></button>
       </form>
+
+      <template v-if="$store.state.auth.availablePermissions['library']">
+        <div class="ui divider"></div>
+        <h2 class="ui header"><translate translate-context="Content/Library/Title/Verb">Import music from your server</translate></h2>
+        <div v-if="fsErrors.length > 0" role="alert" class="ui negative message">
+          <h3 class="header"><translate translate-context="Content/*/Error message.Title">Error while launching import</translate></h3>
+          <ul class="list">
+            <li v-for="error in fsErrors">{{ error }}</li>
+          </ul>
+        </div>
+        <fs-browser
+          v-if="fsStatus"
+          v-model="fsPath"
+          @import="importFs"
+          :loading="isLoadingFs"
+          :data="fsStatus"></fs-browser>
+        
+        <template v-if="fsStatus && fsStatus.import">
+          <h3 class="ui header"><translate translate-context="Content/Library/Title/Verb">Import status</translate></h3>
+          <p v-if="fsStatus.import.reference != importReference">
+            <translate translate-context="Content/Library/Paragraph">Results of your previous import:</translate>
+          </p>
+          <p v-else>
+            <translate translate-context="Content/Library/Paragraph">Results of your import:</translate>
+          </p>
+          <button
+            class="ui button"
+            @click="cancelFsScan"
+            v-if="fsStatus.import.status === 'started' || fsStatus.import.status === 'pending'">
+            <translate translate-context="*/*/Button.Label/Verb">Cancel</translate>
+          </button>
+          <fs-logs :data="fsStatus.import"></fs-logs>
+        </template>
+      </template>
     </div>
     <div :class="['ui', 'bottom', 'attached', 'segment', {hidden: currentTab != 'uploads'}]">
       <div :class="['ui', {loading: isLoadingQuota}, 'container']">
@@ -163,6 +197,8 @@ import $ from "jquery";
 import axios from "axios";
 import logger from "@/logging";
 import FileUploadWidget from "./FileUploadWidget";
+import FsBrowser from "./FsBrowser";
+import FsLogs from "./FsLogs";
 import LibraryFilesTable from "@/views/content/libraries/FilesTable";
 import moment from "moment";
 
@@ -170,7 +206,9 @@ export default {
   props: ["library", "defaultImportReference"],
   components: {
     FileUploadWidget,
-    LibraryFilesTable
+    LibraryFilesTable,
+    FsBrowser,
+    FsLogs,
   },
   data() {
     let importReference = this.defaultImportReference || moment().format();
@@ -190,11 +228,22 @@ export default {
         errored: 0,
         objects: {}
       },
-      processTimestamp: new Date()
+      processTimestamp: new Date(),
+      fsStatus: null,
+      fsPath: [],
+      isLoadingFs: false,
+      fsInterval: null,
+      fsErrors: []
     };
   },
   created() {
     this.fetchStatus();
+    if (this.$store.state.auth.availablePermissions['library']) {
+      this.fetchFs(true)
+      setInterval(() => {
+        this.fetchFs(false)
+      }, 5000);
+    }
     this.fetchQuota();
     this.$store.commit("ui/addWebsocketEventHandler", {
       eventName: "import.status_updated",
@@ -209,6 +258,9 @@ export default {
       id: "fileUpload"
     });
     window.onbeforeunload = null;
+    if (this.fsInterval) {
+      clearInterval(this.fsInterval)
+    }
   },
   methods: {
     onBeforeUnload(e = {}) {
@@ -227,6 +279,38 @@ export default {
         self.isLoadingQuota = false
       })
     },
+    fetchFs (updateLoading) {
+      let self = this
+      if (updateLoading) {
+        self.isLoadingFs = true
+      }
+      axios.get('libraries/fs-import', {params: {path: this.fsPath.join('/')}}).then((response) => {
+        self.fsStatus = response.data
+        if (updateLoading) {
+          self.isLoadingFs = false
+        }
+      })
+    },
+    importFs () {
+      let self = this
+      self.isLoadingFs = true
+      let payload = {
+        path: this.fsPath.join('/'),
+        library: this.library.uuid,
+        import_reference: this.importReference,
+      }
+      axios.post('libraries/fs-import', payload).then((response) => {
+        self.fsStatus = response.data
+        self.isLoadingFs = false
+      }, error => {
+        self.isLoadingFs = false
+        self.fsErrors = error.backendErrors
+      })
+    },
+    async cancelFsScan () {
+      await axios.delete('libraries/fs-import')
+      this.fetchFs()
+    },
     inputFile(newFile, oldFile) {
       if (!newFile) {
         return
@@ -392,6 +476,9 @@ export default {
       if (v > o) {
         this.$emit('uploads-finished', v - o)
       }
+    },
+    "fsPath" () {
+      this.fetchFs(true)
     }
   }
 };

front/src/components/library/FsBrowser.vue

@@ -0,0 +1,54 @@
+<template>
+  <div :class="['ui', {loading}, 'segment']">
+    <div class="ui fluid action input">
+      <input class="ui disabled" disabled :value="data.root + '/' + value.join('/')" />
+      <button class="ui button" @click.prevent="$emit('import')">
+        <translate translate-context="Content/Library/Button/Verb">Import</translate>
+      </button>
+    </div>
+    <div class="ui list component-fs-browser">
+      <a
+        v-if="value.length > 0"
+        class="item"
+        href=""
+        @click.prevent="handleClick({name: '..', dir: true})"
+        >
+        <i class="folder icon"></i>
+        <div class="content">
+          <div class="header">..</div>
+        </div>
+      </a>
+      <a
+        class="item"
+        href=""
+        @click.prevent="handleClick(e)"
+        v-for="e in data.content"
+        :key="e.name">
+        <i class="folder icon" v-if="e.dir"></i>
+        <i class="file icon" v-else></i>
+        <div class="content">
+          <div class="header">{{ e.name }}</div>
+        </div>
+      </a>
+    </div>
+  </div>
+</template>
+<script>
+export default {
+  props: ["data", "loading", "value"],
+  methods: {
+    handleClick (element) {
+      if (!element.dir) {
+        return
+      }
+      if (element.name === "..") {
+        let newValue = [...this.value]
+        newValue.pop()
+        this.$emit('input', newValue)
+      } else {
+        this.$emit('input', [...this.value, element.name])
+      }
+    }
+  }
+}
+</script>

front/src/components/library/FsLogs.vue

@@ -0,0 +1,17 @@
+<template>
+  <div class="ui segment component-fs-logs">
+    <div class="ui active dimmer" v-if="data.status === 'pending'">
+      <div class="ui text loader">
+        <translate translate-context="Content/Library/Paragraph">Import hasn't started yet</translate>
+      </div>
+    </div>
+    <template v-else v-for="(row, idx) in data.logs">
+      <p :key="idx">{{ row }}</p>
+    </template>
+  </div>
+</template>
+<script>
+export default {
+  props: ["data"],
+}
+</script>

front/src/style/_main.scss

@@ -26,6 +26,8 @@ $bottom-player-height: 4rem;
 @import "./components/_empty_state.scss";
 @import "./components/_form.scss";
 @import "./components/_file_upload.scss";
+@import "./components/_fs_browser.scss";
+@import "./components/_fs_logs.scss";
 @import "./components/_header.scss";
 @import "./components/_label.scss";
 @import "./components/_modal.scss";

front/src/style/components/_fs_browser.scss

@@ -0,0 +1,4 @@
+.component-fs-browser {
+    max-height: 400px;
+    overflow-y: auto;
+}

front/src/style/components/_fs_logs.scss

@@ -0,0 +1,6 @@
+.component-fs-logs {
+    max-height: 300px;
+    overflow-y: auto;
+    background-color: rgba(25, 25, 25) !important;
+    color: white !important;
+}