diff --git a/api/funkwhale_api/common/management/commands/inplace_to_s3.py b/api/funkwhale_api/common/management/commands/inplace_to_s3.py new file mode 100644 index 000000000..c9fca94ce --- /dev/null +++ b/api/funkwhale_api/common/management/commands/inplace_to_s3.py @@ -0,0 +1,93 @@ +import pathlib +from argparse import RawTextHelpFormatter + +from django.core.management.base import BaseCommand +from django.db import transaction + +from funkwhale_api.music import models + + +class Command(BaseCommand): + help = """ + Update the reference for Uploads that have been imported with --in-place and are now moved to s3. + + Please note: This does not move any file! Make sure you already moved the files to your s3 bucket. + + Specify --source to filter the reference to update to files from a specific in-place directory. If no + --source is given, all in-place imported track references will be updated. + + Specify --target to specify a subdirectory in the S3 bucket where you moved the files. If no --target is + given, the file is expected to be stored in the same path as before. + + Examples: + + Music File: /music/Artist/Album/track.ogg + --source: /music + --target unset + + All files imported from /music will be updated and expected to be in the same folder structure in the bucket + + Music File: /music/Artist/Album/track.ogg + --source: /music + --target: /in_place + + The music file is expected to be stored in the bucket in the directory /in_place/Artist/Album/track.ogg + """ + + def create_parser(self, *args, **kwargs): + parser = super().create_parser(*args, **kwargs) + parser.formatter_class = RawTextHelpFormatter + return parser + + def add_arguments(self, parser): + parser.add_argument( + "--no-dry-run", + action="store_false", + dest="dry_run", + default=True, + help="Disable dry run mode and apply updates for real on the database", + ) + parser.add_argument( + "--source", + type=pathlib.Path, + required=True, + help="Specify the path of the directory where the files originally were stored to update their reference.", + ) + parser.add_argument( + "--target", + type=pathlib.Path, + help="Specify a subdirectory in the S3 bucket where you moved the files to.", + ) + + @transaction.atomic + def handle(self, *args, **options): + if options["dry_run"]: + self.stdout.write("Dry-run on, will not touch the database") + else: + self.stdout.write("Dry-run off, *changing the database*") + self.stdout.write("") + + prefix = f"file://{options['source']}" + + to_change = models.Upload.objects.filter(source__startswith=prefix) + + self.stdout.write(f"Found {to_change.count()} uploads to update.") + + target = options["target"] if options["target"] else options["source"] + + for upl in to_change: + upl.audio_file = str(upl.source).replace(str(prefix), str(target)) + upl.source = None + self.stdout.write(f"Upload expected in {upl.audio_file}") + if not options["dry_run"]: + upl.save() + + self.stdout.write("") + if options["dry_run"]: + self.stdout.write( + "Nothing was updated, rerun this command with --no-dry-run to apply the changes" + ) + else: + self.stdout.write("Updating completed!") + + self.stdout.write("") diff --git a/api/tests/common/test_commands.py b/api/tests/common/test_commands.py index 304ee799b..6b9a997a7 100644 --- a/api/tests/common/test_commands.py +++ b/api/tests/common/test_commands.py @@ -1,4 +1,5 @@ import os +from io import StringIO import pytest from django.core.management import call_command @@ -116,3 +117,56 @@ def test_unblocked_commands(command, mocker): mocker.patch.dict(os.environ, {"FORCE": "1"}) call_command(command) + + +def test_inplace_to_s3_without_source(): + with pytest.raises(CommandError): + call_command("inplace_to_s3") + + +def test_inplace_to_s3_dryrun(factories): + upload = factories["music.Upload"](in_place=True, source="file:///music/music.mp3") + call_command("inplace_to_s3", "--source", "/music") + assert upload.source == "file:///music/music.mp3" + assert upload.audio_file is None + + +data = [ + { + "file": "/music/test.mp3", + "source": "/", + "target": None, + "expected": "/music/test.mp3", + }, + { + "file": "/music/test.mp3", + "source": "/music", + "target": "/in-place", + "expected": "/in-place/test.mp3", + }, + { + "file": "/music/test.mp3", + "source": "/music", + "target": "/in-place/music", + "expected": "/in-place/music/test.mp3", + }, + {"file": "/music/test.mp3", "source": "/abcd", "target": "/music", "expected": "0"}, +] + + +@pytest.mark.parametrize("data", data) +def test_inplace_to_s3(factories, data): + out = StringIO() + factories["music.Upload"](in_place=True, source=f"file://{data['file']}") + if data["target"]: + call_command( + "inplace_to_s3", + "--source", + data["source"], + "--target", + data["target"], + stdout=out, + ) + else: + call_command("inplace_to_s3", "--source", data["source"], stdout=out) + assert data["expected"] in out.getvalue() diff --git a/changes/changelog.d/2156.feature b/changes/changelog.d/2156.feature new file mode 100644 index 000000000..98d0e437d --- /dev/null +++ b/changes/changelog.d/2156.feature @@ -0,0 +1,2 @@ +New management command to update Uploads which have been imported using --in-place and are now +stored in s3 (#2156)