MirrorMirror
/
funkwhale
mirror of https://dev.funkwhale.audio/funkwhale/funkwhale.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch '1105-scan-ui' into 'develop' 

Fix #1105: Can now launch server import from the UI

Closes #1105

See merge request funkwhale/funkwhale!1192
This commit is contained in:
Agate 2020-08-03 13:56:06 +02:00
parent 36a6a0018e 788c12748f
commit b7f1c02c6f
17 changed files with 476 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
api/funkwhale_api/music/management/commands/import_files.py
View File

@ -12,6 +12,7 @@ import watchdog.events
import watchdog.observers
from django.conf import settings
from django.core.cache import cache
from django.core.files import File
from django.core.management import call_command
from django.core.management.base import BaseCommand, CommandError
@ -68,8 +69,34 @@ def batch(iterable, n=1):
yield current
class CacheWriter:
"""
Output to cache instead of console
"""
def __init__(self, key, stdout, buffer_size=10):
self.key = key
cache.set(self.key, [])
self.stdout = stdout
self.buffer_size = buffer_size
self.buffer = []
def write(self, message):
# we redispatch the message to the console, for debugging
self.stdout.write(message)
self.buffer.append(message)
if len(self.buffer) > self.buffer_size:
self.flush()
def flush(self):
current = cache.get(self.key)
cache.set(self.key, current + self.buffer)
self.buffer = []
class Command(BaseCommand):
help = "Import audio files mathinc given glob pattern"
help = "Import audio files matching given glob pattern"
def add_arguments(self, parser):
parser.add_argument(
@ -207,7 +234,22 @@ class Command(BaseCommand):
help="Size of each batch, only used when crawling large collections",
)
def handle(self, *args, **options):
def handle(self, *args, **kwargs):
cache.set("fs-import:status", "started")
if kwargs.get("update_cache", False):
self.stdout = CacheWriter("fs-import:logs", self.stdout)
self.stderr = self.stdout
try:
return self._handle(*args, **kwargs)
except CommandError as e:
self.stdout.write(str(e))
raise
finally:
if kwargs.get("update_cache", False):
cache.set("fs-import:status", "finished")
self.stdout.flush()
def _handle(self, *args, **options):
# handle relative directories
options["path"] = [os.path.abspath(path) for path in options["path"]]
self.is_confirmed = False
@ -312,6 +354,10 @@ class Command(BaseCommand):
batch_duration = None
self.stdout.write("Starting import of new files…")
for i, entries in enumerate(batch(crawler, options["batch_size"])):
if options.get("update_cache", False) is True:
# check to see if the scan was cancelled
if cache.get("fs-import:status") == "canceled":
raise CommandError("Import cancelled")
total += len(entries)
batch_start = time.time()
time_stats = ""

20
api/funkwhale_api/music/serializers.py
View File

@ -838,3 +838,23 @@ class AlbumCreateSerializer(serializers.Serializer):
tag_models.set_tags(instance, *(validated_data.get("tags", []) or []))
instance.artist.get_channel()
return instance
class FSImportSerializer(serializers.Serializer):
path = serializers.CharField(allow_blank=True)
library = serializers.UUIDField()
import_reference = serializers.CharField()
def validate_path(self, value):
try:
utils.browse_dir(settings.MUSIC_DIRECTORY_PATH, value)
except (NotADirectoryError, FileNotFoundError, ValueError):
raise serializers.ValidationError("Invalid path")
return value
def validate_library(self, value):
try:
return self.context["user"].actor.libraries.get(uuid=value)
except models.Library.DoesNotExist:
raise serializers.ValidationError("Invalid library")

34
api/funkwhale_api/music/tasks.py
View File

@ -3,10 +3,12 @@ import datetime
import logging
import os
from django.utils import timezone
from django.conf import settings
from django.core.cache import cache
from django.db import transaction
from django.db.models import F, Q
from django.dispatch import receiver
from django.utils import timezone
from musicbrainzngs import ResponseError
from requests.exceptions import RequestException
@ -17,6 +19,7 @@ from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import routes
from funkwhale_api.federation import library as lb
from funkwhale_api.federation import utils as federation_utils
from funkwhale_api.music.management.commands import import_files
from funkwhale_api.tags import models as tags_models
from funkwhale_api.tags import tasks as tags_tasks
from funkwhale_api.taskapp import celery
@ -938,3 +941,32 @@ def update_track_metadata(audio_metadata, track):
common_utils.attach_file(
track.album, "attachment_cover", new_data["album"].get("cover_data")
)
@celery.app.task(name="music.fs_import")
@celery.require_instance(models.Library.objects.all(), "library")
def fs_import(library, path, import_reference):
if cache.get("fs-import:status") != "pending":
raise ValueError("Invalid import status")
command = import_files.Command()
options = {
"recursive": True,
"library_id": str(library.uuid),
"path": [os.path.join(settings.MUSIC_DIRECTORY_PATH, path)],
"update_cache": True,
"in_place": True,
"reference": import_reference,
"watch": False,
"interactive": False,
"batch_size": 1000,
"async_": False,
"prune": True,
"replace": False,
"verbosity": 1,
"exit_on_failure": False,
"outbox": False,
"broadcast": False,
}
command.handle(**options)

20
api/funkwhale_api/music/utils.py
View File

@ -1,3 +1,5 @@
import os
import pathlib
import mimetypes
import magic
@ -130,3 +132,21 @@ def increment_downloads_count(upload, user, wsgi_request):
duration = max(upload.duration or 0, settings.MIN_DELAY_BETWEEN_DOWNLOADS_COUNT)
cache.set(cache_key, 1, duration)
def browse_dir(root, path):
if ".." in path:
raise ValueError("Relative browsing is not allowed")
root = pathlib.Path(root)
real_path = root / path
dirs = []
files = []
for el in sorted(os.listdir(real_path)):
if os.path.isdir(real_path / el):
dirs.append({"name": el, "dir": True})
else:
files.append({"name": el, "dir": False})
return dirs + files

59
api/funkwhale_api/music/views.py
View File

@ -3,6 +3,7 @@ import datetime
import logging
import urllib.parse
from django.conf import settings
from django.core.cache import cache
from django.db import transaction
from django.db.models import Count, Prefetch, Sum, F, Q
import django.db.utils
@ -314,6 +315,64 @@ class LibraryViewSet(
serializer = self.get_serializer(queryset, many=True)
return Response(serializer.data)
@action(
methods=["get", "post", "delete"],
detail=False,
url_name="fs-import",
url_path="fs-import",
)
@transaction.non_atomic_requests
def fs_import(self, request, *args, **kwargs):
if not request.user.is_authenticated:
return Response({}, status=403)
if not request.user.all_permissions["library"]:
return Response({}, status=403)
if request.method == "GET":
path = request.GET.get("path", "")
data = {
"root": settings.MUSIC_DIRECTORY_PATH,
"path": path,
"import": None,
}
status = cache.get("fs-import:status", default=None)
if status:
data["import"] = {
"status": status,
"reference": cache.get("fs-import:reference"),
"logs": cache.get("fs-import:logs", default=[]),
}
try:
data["content"] = utils.browse_dir(data["root"], data["path"])
except (NotADirectoryError, ValueError, FileNotFoundError) as e:
return Response({"detail": str(e)}, status=400)
return Response(data)
if request.method == "POST":
if cache.get("fs-import:status", default=None) in [
"pending",
"started",
]:
return Response({"detail": "An import is already running"}, status=400)
data = request.data
serializer = serializers.FSImportSerializer(
data=data, context={"user": request.user}
)
serializer.is_valid(raise_exception=True)
cache.set("fs-import:status", "pending")
cache.set(
"fs-import:reference", serializer.validated_data["import_reference"]
)
tasks.fs_import.delay(
library_id=serializer.validated_data["library"].pk,
path=serializer.validated_data["path"],
import_reference=serializer.validated_data["import_reference"],
)
return Response(status=201)
if request.method == "DELETE":
cache.set("fs-import:status", "canceled")
return Response(status=204)
class TrackViewSet(
HandleInvalidSearch,

1
api/setup.cfg
View File

@ -36,3 +36,4 @@ env =
DISABLE_PASSWORD_VALIDATORS=false
DISABLE_PASSWORD_VALIDATORS=false
FUNKWHALE_PLUGINS=
MUSIC_DIRECTORY_PATH=/music

36
api/tests/music/test_tasks.py
View File

@ -1382,3 +1382,39 @@ def test_update_track_metadata(factories):
assert str(track.artist.mbid) == data["musicbrainz_artistid"]
assert track.album.artist.name == "Edvard Grieg"
assert str(track.album.artist.mbid) == "013c8e5b-d72a-4cd3-8dee-6c64d6125823"
def test_fs_import_not_pending(factories):
with pytest.raises(ValueError):
tasks.fs_import(
library_id=factories["music.Library"]().pk,
path="path",
import_reference="test",
)
def test_fs_import(factories, cache, mocker, settings):
_handle = mocker.spy(tasks.import_files.Command, "_handle")
cache.set("fs-import:status", "pending")
library = factories["music.Library"](actor__local=True)
tasks.fs_import(library_id=library.pk, path="path", import_reference="test")
assert _handle.call_args[1] == {
"recursive": True,
"path": [settings.MUSIC_DIRECTORY_PATH + "/path"],
"library_id": str(library.uuid),
"update_cache": True,
"in_place": True,
"reference": "test",
"watch": False,
"interactive": False,
"batch_size": 1000,
"async_": False,
"prune": True,
"broadcast": False,
"outbox": False,
"exit_on_failure": False,
"replace": False,
"verbosity": 1,
}
assert cache.get("fs-import:status") == "finished"
assert "Pruning dangling tracks" in cache.get("fs-import:logs")[-1]

20
api/tests/music/test_utils.py
View File

@ -1,5 +1,5 @@
import os
import pathlib
import pytest
from funkwhale_api.music import utils
@ -91,3 +91,21 @@ def test_increment_downloads_count_already_tracked(
assert upload.downloads_count == 0
assert upload.track.downloads_count == 0
@pytest.mark.parametrize(
"path,expected",
[
("", [{"name": "Magic", "dir": True}, {"name": "System", "dir": True}]),
("Magic", [{"name": "file.mp3", "dir": False}]),
("System", [{"name": "file.ogg", "dir": False}]),
],
)
def test_get_dirs_and_files(path, expected, tmpdir):
root_path = pathlib.Path(tmpdir)
(root_path / "Magic").mkdir()
(root_path / "Magic" / "file.mp3").touch()
(root_path / "System").mkdir()
(root_path / "System" / "file.ogg").touch()
assert utils.browse_dir(root_path, path) == expected

66
api/tests/music/test_views.py
View File

@ -2,6 +2,7 @@ import datetime
import io
import magic
import os
import pathlib
import urllib.parse
import uuid
@ -1514,3 +1515,68 @@ def test_listen_to_track_with_scoped_token(factories, api_client):
response = api_client.get(url, {"token": token})
assert response.status_code == 200
def test_fs_import_get(factories, superuser_api_client, mocker, settings):
browse_dir = mocker.patch.object(
views.utils, "browse_dir", return_value={"hello": "world"}
)
url = reverse("api:v1:libraries-fs-import")
expected = {
"root": settings.MUSIC_DIRECTORY_PATH,
"path": "",
"content": {"hello": "world"},
"import": None,
}
response = superuser_api_client.get(url, {"path": ""})
assert response.status_code == 200
assert response.data == expected
browse_dir.assert_called_once_with(expected["root"], expected["path"])
def test_fs_import_post(
factories, superuser_api_client, cache, mocker, settings, tmpdir
):
actor = superuser_api_client.user.create_actor()
library = factories["music.Library"](actor=actor)
settings.MUSIC_DIRECTORY_PATH = tmpdir
(pathlib.Path(tmpdir) / "test").mkdir()
fs_import = mocker.patch(
"funkwhale_api.music.tasks.fs_import.delay", return_value={"hello": "world"}
)
url = reverse("api:v1:libraries-fs-import")
response = superuser_api_client.post(
url, {"path": "test", "library": library.uuid, "import_reference": "test"}
)
assert response.status_code == 201
fs_import.assert_called_once_with(
path="test", library_id=library.pk, import_reference="test"
)
assert cache.get("fs-import:status") == "pending"
def test_fs_import_post_already_running(
factories, superuser_api_client, cache, mocker, settings, tmpdir
):
url = reverse("api:v1:libraries-fs-import")
cache.set("fs-import:status", "pending")
response = superuser_api_client.post(url, {"path": "test"})
assert response.status_code == 400
def test_fs_import_cancel_already_running(
factories, superuser_api_client, cache, mocker, settings, tmpdir
):
url = reverse("api:v1:libraries-fs-import")
cache.set("fs-import:status", "pending")
response = superuser_api_client.delete(url)
assert response.status_code == 204
assert cache.get("fs-import:status") == "canceled"

1
changes/changelog.d/1105.feature Normal file
View File

@ -0,0 +1 @@
Can now launch server import from the UI (#1105)

1
front/scripts/fix-fomantic-css.py
View File

@ -114,6 +114,7 @@ def discard_unused_icons(rule):
".eye",
".feed",
".file",
".folder",
".forward",
".globe",
".hashtag",

91
front/src/components/library/FileUpload.vue
View File

@ -56,6 +56,40 @@
<button type="submit" class="ui success button"><translate translate-context="Content/Library/Button.Label">Proceed</translate></button>
</form>
<template v-if="$store.state.auth.availablePermissions['library']">
<div class="ui divider"></div>
<h2 class="ui header"><translate translate-context="Content/Library/Title/Verb">Import music from your server</translate></h2>
<div v-if="fsErrors.length > 0" role="alert" class="ui negative message">
<h3 class="header"><translate translate-context="Content/*/Error message.Title">Error while launching import</translate></h3>
<ul class="list">
<li v-for="error in fsErrors">{{ error }}</li>
</ul>
</div>
<fs-browser
v-if="fsStatus"
v-model="fsPath"
@import="importFs"
:loading="isLoadingFs"
:data="fsStatus"></fs-browser>
<template v-if="fsStatus && fsStatus.import">
<h3 class="ui header"><translate translate-context="Content/Library/Title/Verb">Import status</translate></h3>
<p v-if="fsStatus.import.reference != importReference">
<translate translate-context="Content/Library/Paragraph">Results of your previous import:</translate>
</p>
<p v-else>
<translate translate-context="Content/Library/Paragraph">Results of your import:</translate>
</p>
<button
class="ui button"
@click="cancelFsScan"
v-if="fsStatus.import.status === 'started' || fsStatus.import.status === 'pending'">
<translate translate-context="*/*/Button.Label/Verb">Cancel</translate>
</button>
<fs-logs :data="fsStatus.import"></fs-logs>
</template>
</template>
</div>
<div :class="['ui', 'bottom', 'attached', 'segment', {hidden: currentTab != 'uploads'}]">
<div :class="['ui', {loading: isLoadingQuota}, 'container']">
@ -163,6 +197,8 @@ import $ from "jquery";
import axios from "axios";
import logger from "@/logging";
import FileUploadWidget from "./FileUploadWidget";
import FsBrowser from "./FsBrowser";
import FsLogs from "./FsLogs";
import LibraryFilesTable from "@/views/content/libraries/FilesTable";
import moment from "moment";
@ -170,7 +206,9 @@ export default {
props: ["library", "defaultImportReference"],
components: {
FileUploadWidget,
LibraryFilesTable
LibraryFilesTable,
FsBrowser,
FsLogs,
},
data() {
let importReference = this.defaultImportReference || moment().format();
@ -190,11 +228,22 @@ export default {
errored: 0,
objects: {}
},
processTimestamp: new Date()
processTimestamp: new Date(),
fsStatus: null,
fsPath: [],
isLoadingFs: false,
fsInterval: null,
fsErrors: []
};
},
created() {
this.fetchStatus();
if (this.$store.state.auth.availablePermissions['library']) {
this.fetchFs(true)
setInterval(() => {
this.fetchFs(false)
}, 5000);
}
this.fetchQuota();
this.$store.commit("ui/addWebsocketEventHandler", {
eventName: "import.status_updated",
@ -209,6 +258,9 @@ export default {
id: "fileUpload"
});
window.onbeforeunload = null;
if (this.fsInterval) {
clearInterval(this.fsInterval)
}
},
methods: {
onBeforeUnload(e = {}) {
@ -227,6 +279,38 @@ export default {
self.isLoadingQuota = false
})
},
fetchFs (updateLoading) {
let self = this
if (updateLoading) {
self.isLoadingFs = true
}
axios.get('libraries/fs-import', {params: {path: this.fsPath.join('/')}}).then((response) => {
self.fsStatus = response.data
if (updateLoading) {
self.isLoadingFs = false
}
})
},
importFs () {
let self = this
self.isLoadingFs = true
let payload = {
path: this.fsPath.join('/'),
library: this.library.uuid,
import_reference: this.importReference,
}
axios.post('libraries/fs-import', payload).then((response) => {
self.fsStatus = response.data
self.isLoadingFs = false
}, error => {
self.isLoadingFs = false
self.fsErrors = error.backendErrors
})
},
async cancelFsScan () {
await axios.delete('libraries/fs-import')
this.fetchFs()
},
inputFile(newFile, oldFile) {
if (!newFile) {
return
@ -392,6 +476,9 @@ export default {
if (v > o) {
this.$emit('uploads-finished', v - o)
}
},
"fsPath" () {
this.fetchFs(true)
}
}
};

54
front/src/components/library/FsBrowser.vue Normal file
View File

@ -0,0 +1,54 @@
<template>
<div :class="['ui', {loading}, 'segment']">
<div class="ui fluid action input">
<input class="ui disabled" disabled :value="data.root + '/' + value.join('/')" />
<button class="ui button" @click.prevent="$emit('import')">
<translate translate-context="Content/Library/Button/Verb">Import</translate>
</button>
</div>
<div class="ui list component-fs-browser">
<a
v-if="value.length > 0"
class="item"
href=""
@click.prevent="handleClick({name: '..', dir: true})"
>
<i class="folder icon"></i>
<div class="content">
<div class="header">..</div>
</div>
</a>
<a
class="item"
href=""
@click.prevent="handleClick(e)"
v-for="e in data.content"
:key="e.name">
<i class="folder icon" v-if="e.dir"></i>
<i class="file icon" v-else></i>
<div class="content">
<div class="header">{{ e.name }}</div>
</div>
</a>
</div>
</div>
</template>
<script>
export default {
props: ["data", "loading", "value"],
methods: {
handleClick (element) {
if (!element.dir) {
return
}
if (element.name === "..") {
let newValue = [...this.value]
newValue.pop()
this.$emit('input', newValue)
} else {
this.$emit('input', [...this.value, element.name])
}
}
}
}
</script>

17
front/src/components/library/FsLogs.vue Normal file
View File

@ -0,0 +1,17 @@
<template>
<div class="ui segment component-fs-logs">
<div class="ui active dimmer" v-if="data.status === 'pending'">
<div class="ui text loader">
<translate translate-context="Content/Library/Paragraph">Import hasn't started yet</translate>
</div>
</div>
<template v-else v-for="(row, idx) in data.logs">
<p :key="idx">{{ row }}</p>
</template>
</div>
</template>
<script>
export default {
props: ["data"],
}
</script>

2
front/src/style/_main.scss
View File

@ -26,6 +26,8 @@ $bottom-player-height: 4rem;
@import "./components/_empty_state.scss";
@import "./components/_form.scss";
@import "./components/_file_upload.scss";
@import "./components/_fs_browser.scss";
@import "./components/_fs_logs.scss";
@import "./components/_header.scss";
@import "./components/_label.scss";
@import "./components/_modal.scss";

4
front/src/style/components/_fs_browser.scss Normal file
View File

@ -0,0 +1,4 @@
.component-fs-browser {
max-height: 400px;
overflow-y: auto;
}

6
front/src/style/components/_fs_logs.scss Normal file
View File

@ -0,0 +1,6 @@
.component-fs-logs {
max-height: 300px;
overflow-y: auto;
background-color: rgba(25, 25, 25) !important;
color: white !important;
}