diff --git a/api/funkwhale_api/music/management/commands/import_files.py b/api/funkwhale_api/music/management/commands/import_files.py
index 0d44af49c..c2f94da5a 100644
--- a/api/funkwhale_api/music/management/commands/import_files.py
+++ b/api/funkwhale_api/music/management/commands/import_files.py
@@ -12,6 +12,7 @@ import watchdog.events
import watchdog.observers
from django.conf import settings
+from django.core.cache import cache
from django.core.files import File
from django.core.management import call_command
from django.core.management.base import BaseCommand, CommandError
@@ -68,8 +69,34 @@ def batch(iterable, n=1):
yield current
+class CacheWriter:
+ """
+ Output to cache instead of console
+ """
+
+ def __init__(self, key, stdout, buffer_size=10):
+ self.key = key
+ cache.set(self.key, [])
+ self.stdout = stdout
+ self.buffer_size = buffer_size
+ self.buffer = []
+
+ def write(self, message):
+ # we redispatch the message to the console, for debugging
+ self.stdout.write(message)
+
+ self.buffer.append(message)
+ if len(self.buffer) > self.buffer_size:
+ self.flush()
+
+ def flush(self):
+ current = cache.get(self.key)
+ cache.set(self.key, current + self.buffer)
+ self.buffer = []
+
+
class Command(BaseCommand):
- help = "Import audio files mathinc given glob pattern"
+ help = "Import audio files matching given glob pattern"
def add_arguments(self, parser):
parser.add_argument(
@@ -207,7 +234,22 @@ class Command(BaseCommand):
help="Size of each batch, only used when crawling large collections",
)
- def handle(self, *args, **options):
+ def handle(self, *args, **kwargs):
+ cache.set("fs-import:status", "started")
+ if kwargs.get("update_cache", False):
+ self.stdout = CacheWriter("fs-import:logs", self.stdout)
+ self.stderr = self.stdout
+ try:
+ return self._handle(*args, **kwargs)
+ except CommandError as e:
+ self.stdout.write(str(e))
+ raise
+ finally:
+ if kwargs.get("update_cache", False):
+ cache.set("fs-import:status", "finished")
+ self.stdout.flush()
+
+ def _handle(self, *args, **options):
# handle relative directories
options["path"] = [os.path.abspath(path) for path in options["path"]]
self.is_confirmed = False
@@ -312,6 +354,10 @@ class Command(BaseCommand):
batch_duration = None
self.stdout.write("Starting import of new files…")
for i, entries in enumerate(batch(crawler, options["batch_size"])):
+ if options.get("update_cache", False) is True:
+ # check to see if the scan was cancelled
+ if cache.get("fs-import:status") == "canceled":
+ raise CommandError("Import cancelled")
total += len(entries)
batch_start = time.time()
time_stats = ""
diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py
index 124c57f62..c5e4336a6 100644
--- a/api/funkwhale_api/music/serializers.py
+++ b/api/funkwhale_api/music/serializers.py
@@ -838,3 +838,23 @@ class AlbumCreateSerializer(serializers.Serializer):
tag_models.set_tags(instance, *(validated_data.get("tags", []) or []))
instance.artist.get_channel()
return instance
+
+
+class FSImportSerializer(serializers.Serializer):
+ path = serializers.CharField(allow_blank=True)
+ library = serializers.UUIDField()
+ import_reference = serializers.CharField()
+
+ def validate_path(self, value):
+ try:
+ utils.browse_dir(settings.MUSIC_DIRECTORY_PATH, value)
+ except (NotADirectoryError, FileNotFoundError, ValueError):
+ raise serializers.ValidationError("Invalid path")
+
+ return value
+
+ def validate_library(self, value):
+ try:
+ return self.context["user"].actor.libraries.get(uuid=value)
+ except models.Library.DoesNotExist:
+ raise serializers.ValidationError("Invalid library")
diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py
index e44888a1f..2e633925b 100644
--- a/api/funkwhale_api/music/tasks.py
+++ b/api/funkwhale_api/music/tasks.py
@@ -3,10 +3,12 @@ import datetime
import logging
import os
-from django.utils import timezone
+from django.conf import settings
+from django.core.cache import cache
from django.db import transaction
from django.db.models import F, Q
from django.dispatch import receiver
+from django.utils import timezone
from musicbrainzngs import ResponseError
from requests.exceptions import RequestException
@@ -17,6 +19,7 @@ from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import routes
from funkwhale_api.federation import library as lb
from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.music.management.commands import import_files
from funkwhale_api.tags import models as tags_models
from funkwhale_api.tags import tasks as tags_tasks
from funkwhale_api.taskapp import celery
@@ -938,3 +941,32 @@ def update_track_metadata(audio_metadata, track):
common_utils.attach_file(
track.album, "attachment_cover", new_data["album"].get("cover_data")
)
+
+
+@celery.app.task(name="music.fs_import")
+@celery.require_instance(models.Library.objects.all(), "library")
+def fs_import(library, path, import_reference):
+ if cache.get("fs-import:status") != "pending":
+ raise ValueError("Invalid import status")
+
+ command = import_files.Command()
+
+ options = {
+ "recursive": True,
+ "library_id": str(library.uuid),
+ "path": [os.path.join(settings.MUSIC_DIRECTORY_PATH, path)],
+ "update_cache": True,
+ "in_place": True,
+ "reference": import_reference,
+ "watch": False,
+ "interactive": False,
+ "batch_size": 1000,
+ "async_": False,
+ "prune": True,
+ "replace": False,
+ "verbosity": 1,
+ "exit_on_failure": False,
+ "outbox": False,
+ "broadcast": False,
+ }
+ command.handle(**options)
diff --git a/api/funkwhale_api/music/utils.py b/api/funkwhale_api/music/utils.py
index b61c8223b..17b5b7292 100644
--- a/api/funkwhale_api/music/utils.py
+++ b/api/funkwhale_api/music/utils.py
@@ -1,3 +1,5 @@
+import os
+import pathlib
import mimetypes
import magic
@@ -130,3 +132,21 @@ def increment_downloads_count(upload, user, wsgi_request):
duration = max(upload.duration or 0, settings.MIN_DELAY_BETWEEN_DOWNLOADS_COUNT)
cache.set(cache_key, 1, duration)
+
+
+def browse_dir(root, path):
+ if ".." in path:
+ raise ValueError("Relative browsing is not allowed")
+
+ root = pathlib.Path(root)
+ real_path = root / path
+
+ dirs = []
+ files = []
+ for el in sorted(os.listdir(real_path)):
+ if os.path.isdir(real_path / el):
+ dirs.append({"name": el, "dir": True})
+ else:
+ files.append({"name": el, "dir": False})
+
+ return dirs + files
diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py
index 5dab86abd..18c2b30aa 100644
--- a/api/funkwhale_api/music/views.py
+++ b/api/funkwhale_api/music/views.py
@@ -3,6 +3,7 @@ import datetime
import logging
import urllib.parse
from django.conf import settings
+from django.core.cache import cache
from django.db import transaction
from django.db.models import Count, Prefetch, Sum, F, Q
import django.db.utils
@@ -314,6 +315,64 @@ class LibraryViewSet(
serializer = self.get_serializer(queryset, many=True)
return Response(serializer.data)
+ @action(
+ methods=["get", "post", "delete"],
+ detail=False,
+ url_name="fs-import",
+ url_path="fs-import",
+ )
+ @transaction.non_atomic_requests
+ def fs_import(self, request, *args, **kwargs):
+ if not request.user.is_authenticated:
+ return Response({}, status=403)
+ if not request.user.all_permissions["library"]:
+ return Response({}, status=403)
+ if request.method == "GET":
+ path = request.GET.get("path", "")
+ data = {
+ "root": settings.MUSIC_DIRECTORY_PATH,
+ "path": path,
+ "import": None,
+ }
+ status = cache.get("fs-import:status", default=None)
+ if status:
+ data["import"] = {
+ "status": status,
+ "reference": cache.get("fs-import:reference"),
+ "logs": cache.get("fs-import:logs", default=[]),
+ }
+ try:
+ data["content"] = utils.browse_dir(data["root"], data["path"])
+ except (NotADirectoryError, ValueError, FileNotFoundError) as e:
+ return Response({"detail": str(e)}, status=400)
+
+ return Response(data)
+ if request.method == "POST":
+ if cache.get("fs-import:status", default=None) in [
+ "pending",
+ "started",
+ ]:
+ return Response({"detail": "An import is already running"}, status=400)
+
+ data = request.data
+ serializer = serializers.FSImportSerializer(
+ data=data, context={"user": request.user}
+ )
+ serializer.is_valid(raise_exception=True)
+ cache.set("fs-import:status", "pending")
+ cache.set(
+ "fs-import:reference", serializer.validated_data["import_reference"]
+ )
+ tasks.fs_import.delay(
+ library_id=serializer.validated_data["library"].pk,
+ path=serializer.validated_data["path"],
+ import_reference=serializer.validated_data["import_reference"],
+ )
+ return Response(status=201)
+ if request.method == "DELETE":
+ cache.set("fs-import:status", "canceled")
+ return Response(status=204)
+
class TrackViewSet(
HandleInvalidSearch,
diff --git a/api/setup.cfg b/api/setup.cfg
index 724d1c87f..1f09309bd 100644
--- a/api/setup.cfg
+++ b/api/setup.cfg
@@ -36,3 +36,4 @@ env =
DISABLE_PASSWORD_VALIDATORS=false
DISABLE_PASSWORD_VALIDATORS=false
FUNKWHALE_PLUGINS=
+ MUSIC_DIRECTORY_PATH=/music
diff --git a/api/tests/music/test_tasks.py b/api/tests/music/test_tasks.py
index 6aecd35fe..edb937d81 100644
--- a/api/tests/music/test_tasks.py
+++ b/api/tests/music/test_tasks.py
@@ -1382,3 +1382,39 @@ def test_update_track_metadata(factories):
assert str(track.artist.mbid) == data["musicbrainz_artistid"]
assert track.album.artist.name == "Edvard Grieg"
assert str(track.album.artist.mbid) == "013c8e5b-d72a-4cd3-8dee-6c64d6125823"
+
+
+def test_fs_import_not_pending(factories):
+ with pytest.raises(ValueError):
+ tasks.fs_import(
+ library_id=factories["music.Library"]().pk,
+ path="path",
+ import_reference="test",
+ )
+
+
+def test_fs_import(factories, cache, mocker, settings):
+ _handle = mocker.spy(tasks.import_files.Command, "_handle")
+ cache.set("fs-import:status", "pending")
+ library = factories["music.Library"](actor__local=True)
+ tasks.fs_import(library_id=library.pk, path="path", import_reference="test")
+ assert _handle.call_args[1] == {
+ "recursive": True,
+ "path": [settings.MUSIC_DIRECTORY_PATH + "/path"],
+ "library_id": str(library.uuid),
+ "update_cache": True,
+ "in_place": True,
+ "reference": "test",
+ "watch": False,
+ "interactive": False,
+ "batch_size": 1000,
+ "async_": False,
+ "prune": True,
+ "broadcast": False,
+ "outbox": False,
+ "exit_on_failure": False,
+ "replace": False,
+ "verbosity": 1,
+ }
+ assert cache.get("fs-import:status") == "finished"
+ assert "Pruning dangling tracks" in cache.get("fs-import:logs")[-1]
diff --git a/api/tests/music/test_utils.py b/api/tests/music/test_utils.py
index c05b530aa..67a4212d8 100644
--- a/api/tests/music/test_utils.py
+++ b/api/tests/music/test_utils.py
@@ -1,5 +1,5 @@
import os
-
+import pathlib
import pytest
from funkwhale_api.music import utils
@@ -91,3 +91,21 @@ def test_increment_downloads_count_already_tracked(
assert upload.downloads_count == 0
assert upload.track.downloads_count == 0
+
+
+@pytest.mark.parametrize(
+ "path,expected",
+ [
+ ("", [{"name": "Magic", "dir": True}, {"name": "System", "dir": True}]),
+ ("Magic", [{"name": "file.mp3", "dir": False}]),
+ ("System", [{"name": "file.ogg", "dir": False}]),
+ ],
+)
+def test_get_dirs_and_files(path, expected, tmpdir):
+ root_path = pathlib.Path(tmpdir)
+ (root_path / "Magic").mkdir()
+ (root_path / "Magic" / "file.mp3").touch()
+ (root_path / "System").mkdir()
+ (root_path / "System" / "file.ogg").touch()
+
+ assert utils.browse_dir(root_path, path) == expected
diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py
index 2fa20aa1d..7e96f46c0 100644
--- a/api/tests/music/test_views.py
+++ b/api/tests/music/test_views.py
@@ -2,6 +2,7 @@ import datetime
import io
import magic
import os
+import pathlib
import urllib.parse
import uuid
@@ -1514,3 +1515,68 @@ def test_listen_to_track_with_scoped_token(factories, api_client):
response = api_client.get(url, {"token": token})
assert response.status_code == 200
+
+
+def test_fs_import_get(factories, superuser_api_client, mocker, settings):
+ browse_dir = mocker.patch.object(
+ views.utils, "browse_dir", return_value={"hello": "world"}
+ )
+ url = reverse("api:v1:libraries-fs-import")
+
+ expected = {
+ "root": settings.MUSIC_DIRECTORY_PATH,
+ "path": "",
+ "content": {"hello": "world"},
+ "import": None,
+ }
+ response = superuser_api_client.get(url, {"path": ""})
+
+ assert response.status_code == 200
+ assert response.data == expected
+ browse_dir.assert_called_once_with(expected["root"], expected["path"])
+
+
+def test_fs_import_post(
+ factories, superuser_api_client, cache, mocker, settings, tmpdir
+):
+ actor = superuser_api_client.user.create_actor()
+ library = factories["music.Library"](actor=actor)
+ settings.MUSIC_DIRECTORY_PATH = tmpdir
+ (pathlib.Path(tmpdir) / "test").mkdir()
+ fs_import = mocker.patch(
+ "funkwhale_api.music.tasks.fs_import.delay", return_value={"hello": "world"}
+ )
+ url = reverse("api:v1:libraries-fs-import")
+
+ response = superuser_api_client.post(
+ url, {"path": "test", "library": library.uuid, "import_reference": "test"}
+ )
+
+ assert response.status_code == 201
+ fs_import.assert_called_once_with(
+ path="test", library_id=library.pk, import_reference="test"
+ )
+ assert cache.get("fs-import:status") == "pending"
+
+
+def test_fs_import_post_already_running(
+ factories, superuser_api_client, cache, mocker, settings, tmpdir
+):
+ url = reverse("api:v1:libraries-fs-import")
+ cache.set("fs-import:status", "pending")
+
+ response = superuser_api_client.post(url, {"path": "test"})
+
+ assert response.status_code == 400
+
+
+def test_fs_import_cancel_already_running(
+ factories, superuser_api_client, cache, mocker, settings, tmpdir
+):
+ url = reverse("api:v1:libraries-fs-import")
+ cache.set("fs-import:status", "pending")
+
+ response = superuser_api_client.delete(url)
+
+ assert response.status_code == 204
+ assert cache.get("fs-import:status") == "canceled"
diff --git a/changes/changelog.d/1105.feature b/changes/changelog.d/1105.feature
new file mode 100644
index 000000000..152c2a7be
--- /dev/null
+++ b/changes/changelog.d/1105.feature
@@ -0,0 +1 @@
+Can now launch server import from the UI (#1105)
\ No newline at end of file
diff --git a/front/scripts/fix-fomantic-css.py b/front/scripts/fix-fomantic-css.py
index 692383f59..095f66a08 100755
--- a/front/scripts/fix-fomantic-css.py
+++ b/front/scripts/fix-fomantic-css.py
@@ -114,6 +114,7 @@ def discard_unused_icons(rule):
".eye",
".feed",
".file",
+ ".folder",
".forward",
".globe",
".hashtag",
diff --git a/front/src/components/library/FileUpload.vue b/front/src/components/library/FileUpload.vue
index e4dd6a4a7..6d7ac605e 100644
--- a/front/src/components/library/FileUpload.vue
+++ b/front/src/components/library/FileUpload.vue
@@ -56,6 +56,40 @@
+
+
+
+
+
+
+
+
+
{{ row }}
+ +