Fix #706: Added a 'fix_federation_ids' management command to deal with protocol/domain issues in federation

IDs after deployments

api/funkwhale_api/common/utils.py

@@ -147,3 +147,28 @@ def order_for_search(qs, field):
     this function will order the given qs based on the length of the given field
     """
     return qs.annotate(__size=models.functions.Length(field)).order_by("__size")
+
+
+def replace_prefix(queryset, field, old, new):
+    """
+    Given a queryset of objects and a field name, will find objects
+    for which the field have the given value, and replace the old prefix by
+    the new one.
+
+    This is especially useful to find/update bad federation ids, to replace:
+
+    http://wrongprotocolanddomain/path
+
+    by
+
+    https://goodprotocalanddomain/path
+
+    on a whole table with a single query.
+    """
+    qs = queryset.filter(**{"{}__startswith".format(field): old})
+    # we extract the part after the old prefix, and Concat it with our new prefix
+    update = models.functions.Concat(
+        models.Value(new),
+        models.functions.Substr(field, len(old) + 1, output_field=models.CharField()),
+    )
+    return qs.update(**{field: update})

api/funkwhale_api/federation/management/commands/fix_federation_ids.py

@@ -0,0 +1,122 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from funkwhale_api.common import utils
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.music import models as music_models
+
+
+MODELS = [
+    (music_models.Artist, ["fid"]),
+    (music_models.Album, ["fid"]),
+    (music_models.Track, ["fid"]),
+    (music_models.Upload, ["fid"]),
+    (music_models.Library, ["fid", "followers_url"]),
+    (
+        federation_models.Actor,
+        [
+            "fid",
+            "url",
+            "outbox_url",
+            "inbox_url",
+            "following_url",
+            "followers_url",
+            "shared_inbox_url",
+        ],
+    ),
+    (federation_models.Activity, ["fid"]),
+    (federation_models.Follow, ["fid"]),
+    (federation_models.LibraryFollow, ["fid"]),
+]
+
+
+class Command(BaseCommand):
+    help = """
+    Find and replace wrong protocal/domain in local federation ids.
+
+    Use with caution and only if you know what you are doing.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "old_base_url",
+            type=str,
+            help="The invalid url prefix you want to find and replace, e.g 'http://baddomain'",
+        )
+        parser.add_argument(
+            "new_base_url",
+            type=str,
+            help="The url prefix you want to use in place of the bad one, e.g 'https://gooddomain'",
+        )
+        parser.add_argument(
+            "--noinput",
+            "--no-input",
+            action="store_false",
+            dest="interactive",
+            help="Do NOT prompt the user for input of any kind.",
+        )
+
+        parser.add_argument(
+            "--no-dry-run",
+            action="store_false",
+            dest="dry_run",
+            help="Commit the changes to the database",
+        )
+
+    def handle(self, *args, **options):
+        results = {}
+        old_prefix, new_prefix = options["old_base_url"], options["new_base_url"]
+        for kls, fields in MODELS:
+            results[kls] = {}
+            for field in fields:
+                candidates = kls.objects.filter(
+                    **{"{}__startswith".format(field): old_prefix}
+                )
+                results[kls][field] = candidates.count()
+
+        total = sum([t for k in results.values() for t in k.values()])
+        self.stdout.write("")
+        if total:
+            self.stdout.write(
+                self.style.WARNING(
+                    "Will replace {} found occurences of '{}' by '{}':".format(
+                        total, old_prefix, new_prefix
+                    )
+                )
+            )
+            self.stdout.write("")
+            for kls, fields in results.items():
+                for field, count in fields.items():
+                    self.stdout.write(
+                        "- {}/{} {}.{}".format(
+                            count, kls.objects.count(), kls._meta.label, field
+                        )
+                    )
+
+        else:
+            self.stdout.write(
+                "No objects found with prefix {}, exiting.".format(old_prefix)
+            )
+            return
+        if options["dry_run"]:
+            self.stdout.write(
+                "Run this command with --no-dry-run to perform the replacement."
+            )
+            return
+        self.stdout.write("")
+        if options["interactive"]:
+            message = (
+                "Are you sure you want to perform the replacement on {} objects?\n\n"
+                "Type 'yes' to continue, or 'no' to cancel: "
+            ).format(total)
+            if input("".join(message)) != "yes":
+                raise CommandError("Command canceled.")
+
+        for kls, fields in results.items():
+            for field, count in fields.items():
+                self.stdout.write(
+                    "Replacing {} on {} {}…".format(field, count, kls._meta.label)
+                )
+                candidates = kls.objects.all()
+                utils.replace_prefix(candidates, field, old=old_prefix, new=new_prefix)
+        self.stdout.write("")
+        self.stdout.write(self.style.SUCCESS("Done!"))

api/tests/common/test_utils.py

@@ -8,3 +8,37 @@ def test_chunk_queryset(factories):
 
     assert list(chunks[0]) == actors[0:2]
     assert list(chunks[1]) == actors[2:4]
+
+
+def test_update_prefix(factories):
+    actors = []
+    fid = "http://hello.world/actor/{}/"
+    for i in range(3):
+        actors.append(factories["federation.Actor"](fid=fid.format(i)))
+    noop = [
+        factories["federation.Actor"](fid="https://hello.world/actor/witness/"),
+        factories["federation.Actor"](fid="http://another.world/actor/witness/"),
+        factories["federation.Actor"](fid="http://foo.bar/actor/witness/"),
+    ]
+
+    qs = actors[0].__class__.objects.filter(fid__startswith="http://hello.world")
+    assert qs.count() == 3
+
+    result = utils.replace_prefix(
+        actors[0].__class__.objects.all(),
+        "fid",
+        "http://hello.world",
+        "https://hello.world",
+    )
+
+    assert result == 3
+
+    for n in noop:
+        old = n.fid
+        n.refresh_from_db()
+        assert old == n.fid
+
+    for n in actors:
+        old = n.fid
+        n.refresh_from_db()
+        assert n.fid == old.replace("http://", "https://")

api/tests/federation/test_commands.py

@@ -0,0 +1,55 @@
+from django.core.management import call_command
+
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation.management.commands import fix_federation_ids
+from funkwhale_api.music import models as music_models
+
+
+def test_fix_fids_dry_run(factories, mocker):
+    replace_prefix = mocker.patch("funkwhale_api.common.utils.replace_prefix")
+
+    call_command("fix_federation_ids", "http://old/", "https://new/", interactive=False)
+
+    replace_prefix.assert_not_called()
+
+
+def test_fix_fids_no_dry_run(factories, mocker, queryset_equal_queries):
+    replace_prefix = mocker.patch("funkwhale_api.common.utils.replace_prefix")
+    factories["federation.Actor"](fid="http://old/test")
+    call_command(
+        "fix_federation_ids",
+        "http://old",
+        "https://new",
+        interactive=False,
+        dry_run=False,
+    )
+
+    models = [
+        (music_models.Artist, ["fid"]),
+        (music_models.Album, ["fid"]),
+        (music_models.Track, ["fid"]),
+        (music_models.Upload, ["fid"]),
+        (music_models.Library, ["fid", "followers_url"]),
+        (
+            federation_models.Actor,
+            [
+                "fid",
+                "url",
+                "outbox_url",
+                "inbox_url",
+                "following_url",
+                "followers_url",
+                "shared_inbox_url",
+            ],
+        ),
+        (federation_models.Activity, ["fid"]),
+        (federation_models.Follow, ["fid"]),
+        (federation_models.LibraryFollow, ["fid"]),
+    ]
+    assert models == fix_federation_ids.MODELS
+
+    for kls, fields in models:
+        for field in fields:
+            replace_prefix.assert_any_call(
+                kls.objects.all(), field, old="http://old", new="https://new"
+            )

changes/changelog.d/706.enhancement

@@ -0,0 +1,2 @@
+Added a 'fix_federation_ids' management command to deal with protocol/domain issues in federation
+IDs after deployments (#706)