diff --git a/api/funkwhale_api/providers/audiofile/management/commands/import_files.py b/api/funkwhale_api/providers/audiofile/management/commands/import_files.py index fca98ccd3..a2757c692 100644 --- a/api/funkwhale_api/providers/audiofile/management/commands/import_files.py +++ b/api/funkwhale_api/providers/audiofile/management/commands/import_files.py @@ -1,6 +1,7 @@ import glob import os +from django.conf import settings from django.core.files import File from django.core.management.base import BaseCommand, CommandError @@ -38,7 +39,20 @@ class Command(BaseCommand): action='store_true', dest='exit_on_failure', default=False, - help='use this flag to disable error catching', + help='Use this flag to disable error catching', + ) + parser.add_argument( + '--in-place', '-i', + action='store_true', + dest='in_place', + default=False, + help=( + 'Import files without duplicating them into the media directory.' + 'For in-place import to work, the music files must be readable' + 'by the web-server and funkwhale api and celeryworker processes.' + 'You may want to use this if you have a big music library to ' + 'import and not much disk space available.' + ) ) parser.add_argument( '--no-acoustid', @@ -53,10 +67,6 @@ class Command(BaseCommand): ) def handle(self, *args, **options): - # self.stdout.write(self.style.SUCCESS('Successfully closed poll "%s"' % poll_id)) - - # Recursive is supported only on Python 3.5+, so we pass the option - # only if it's True to avoid breaking on older versions of Python glob_kwargs = {} if options['recursive']: glob_kwargs['recursive'] = True @@ -65,6 +75,21 @@ class Command(BaseCommand): except TypeError: raise Exception('You need Python 3.5 to use the --recursive flag') + if options['in_place']: + self.stdout.write( + 'Checking imported paths against settings.MUSIC_DIRECTORY_PATH') + p = settings.MUSIC_DIRECTORY_PATH + if not p: + raise CommandError( + 'Importing in-place requires setting the ' + 'MUSIC_DIRECTORY_PATH variable') + for m in matching: + if not m.startswith(p): + raise CommandError( + 'Importing in-place only works if importing' + 'from {} (MUSIC_DIRECTORY_PATH), as this directory' + 'needs to be accessible by the webserver.' + 'Culprit: {}'.format(p, m)) if not matching: raise CommandError('No file matching pattern, aborting') @@ -92,6 +117,10 @@ class Command(BaseCommand): self.stdout.write('- {} new files'.format( len(filtered['new']))) + self.stdout.write('Selected options: {}'.format(', '.join([ + 'no acoustid' if options['no_acoustid'] else 'use acoustid', + 'in place' if options['in_place'] else 'copy music files', + ]))) if len(filtered['new']) == 0: self.stdout.write('Nothing new to import, exiting') return @@ -164,11 +193,12 @@ class Command(BaseCommand): job = batch.jobs.create( source='file://' + path, ) - name = os.path.basename(path) - with open(path, 'rb') as f: - job.audio_file.save(name, File(f)) + if not options['in_place']: + name = os.path.basename(path) + with open(path, 'rb') as f: + job.audio_file.save(name, File(f)) - job.save() + job.save() import_handler( import_job_id=job.pk, use_acoustid=not options['no_acoustid']) diff --git a/api/tests/test_import_audio_file.py b/api/tests/test_import_audio_file.py index 0fe484fa5..8217ffa0b 100644 --- a/api/tests/test_import_audio_file.py +++ b/api/tests/test_import_audio_file.py @@ -58,6 +58,20 @@ def test_management_command_requires_a_valid_username(factories, mocker): call_command('import_files', path, username='me', interactive=False) +def test_in_place_import_only_from_music_dir(factories, settings): + user = factories['users.User'](username='me') + settings.MUSIC_DIRECTORY_PATH = '/nope' + path = os.path.join(DATA_DIR, 'dummy_file.ogg') + with pytest.raises(CommandError): + call_command( + 'import_files', + path, + in_place=True, + username='me', + interactive=False + ) + + def test_import_files_creates_a_batch_and_job(factories, mocker): m = mocker.patch('funkwhale_api.music.tasks.import_job_run') user = factories['users.User'](username='me') @@ -137,6 +151,27 @@ def test_import_files_works_with_utf8_file_name(factories, mocker): use_acoustid=False) +def test_import_files_in_place(factories, mocker, settings): + settings.MUSIC_DIRECTORY_PATH = DATA_DIR + m = mocker.patch('funkwhale_api.music.tasks.import_job_run') + user = factories['users.User'](username='me') + path = os.path.join(DATA_DIR, 'utf8-éà◌.ogg') + call_command( + 'import_files', + path, + username='me', + async=False, + in_place=True, + no_acoustid=True, + interactive=False) + batch = user.imports.latest('id') + job = batch.jobs.first() + assert bool(job.audio_file) is False + m.assert_called_once_with( + import_job_id=job.pk, + use_acoustid=False) + + def test_storage_rename_utf_8_files(factories): tf = factories['music.TrackFile'](audio_file__filename='été.ogg') assert tf.audio_file.name.endswith('ete.ogg')