Merge branch '317-user-actors' into 'develop'

Resolve "Have an actor for our users"

Closes #317

See merge request funkwhale/funkwhale!338

api/funkwhale_api/common/scripts/__init__.py

@@ -1,6 +1,12 @@
+from . import create_actors
 from . import create_image_variations
 from . import django_permissions_to_user_permissions
 from . import test
 
 
-__all__ = ["create_image_variations", "django_permissions_to_user_permissions", "test"]
+__all__ = [
+    "create_actors",
+    "create_image_variations",
+    "django_permissions_to_user_permissions",
+    "test",
+]

api/funkwhale_api/common/scripts/create_actors.py

@@ -0,0 +1,23 @@
+"""
+Compute different sizes of image used for Album covers and User avatars
+"""
+from django.db.utils import IntegrityError
+
+from funkwhale_api.users.models import User, create_actor
+
+
+def main(command, **kwargs):
+    qs = User.objects.filter(actor__isnull=True).order_by("username")
+    total = len(qs)
+    command.stdout.write("{} users found without actors".format(total))
+    for i, user in enumerate(qs):
+        command.stdout.write(
+            "{}/{} creating actor for {}".format(i + 1, total, user.username)
+        )
+        try:
+            user.actor = create_actor(user)
+        except IntegrityError as e:
+            # somehow, an actor with the the url exists in the database
+            command.stderr.write("Error while creating actor: {}".format(str(e)))
+            continue
+        user.save(update_fields=["actor"])

api/funkwhale_api/federation/models.py

@@ -20,6 +20,11 @@ TYPE_CHOICES = [
 ]
 
 
+class ActorQuerySet(models.QuerySet):
+    def local(self, include=True):
+        return self.exclude(user__isnull=include)
+
+
 class Actor(models.Model):
     ap_type = "Actor"
 
@@ -47,6 +52,8 @@ class Actor(models.Model):
         related_name="following",
     )
 
+    objects = ActorQuerySet.as_manager()
+
     class Meta:
         unique_together = ["domain", "preferred_username"]
 

api/funkwhale_api/federation/serializers.py

@@ -1,6 +1,8 @@
 import logging
+import mimetypes
 import urllib.parse
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.core.paginator import Paginator
 from django.db import transaction
 from rest_framework import serializers
@@ -63,6 +65,15 @@ class ActorSerializer(serializers.Serializer):
         ret["endpoints"] = {}
         if instance.shared_inbox_url:
             ret["endpoints"]["sharedInbox"] = instance.shared_inbox_url
+        try:
+            if instance.user.avatar:
+                ret["icon"] = {
+                    "type": "Image",
+                    "mediaType": mimetypes.guess_type(instance.user.avatar.path)[0],
+                    "url": utils.full_url(instance.user.avatar.crop["400x400"].url),
+                }
+        except ObjectDoesNotExist:
+            pass
         return ret
 
     def prepare_missing_fields(self):

api/funkwhale_api/federation/urls.py

@@ -8,6 +8,7 @@ music_router = routers.SimpleRouter(trailing_slash=False)
 router.register(
     r"federation/instance/actors", views.InstanceActorViewSet, "instance-actors"
 )
+router.register(r"federation/actors", views.ActorViewSet, "actors")
 router.register(r".well-known", views.WellKnownViewSet, "well-known")
 
 music_router.register(r"files", views.MusicFilesViewSet, "files")

api/funkwhale_api/federation/utils.py

@@ -5,6 +5,8 @@ def full_url(path):
     """
     Given a relative path, return a full url usable for federation purpose
     """
+    if path.startswith("http://") or path.startswith("https://"):
+        return path
     root = settings.FUNKWHALE_URL
     if path.startswith("/") and root.endswith("/"):
         return root + path[1:]

api/funkwhale_api/federation/views.py

@@ -32,6 +32,23 @@ class FederationMixin(object):
         return super().dispatch(request, *args, **kwargs)
 
 
+class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+    lookup_field = "user__username"
+    authentication_classes = [authentication.SignatureAuthentication]
+    permission_classes = []
+    renderer_classes = [renderers.ActivityPubRenderer]
+    queryset = models.Actor.objects.local().select_related("user")
+    serializer_class = serializers.ActorSerializer
+
+    @detail_route(methods=["get", "post"])
+    def inbox(self, request, *args, **kwargs):
+        return response.Response({}, status=200)
+
+    @detail_route(methods=["get", "post"])
+    def outbox(self, request, *args, **kwargs):
+        return response.Response({}, status=200)
+
+
 class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet):
     lookup_field = "actor"
     lookup_value_regex = "[a-z]*"
@@ -100,6 +117,8 @@ class WellKnownViewSet(viewsets.GenericViewSet):
             resource_type, resource = webfinger.clean_resource(request.GET["resource"])
             cleaner = getattr(webfinger, "clean_{}".format(resource_type))
             result = cleaner(resource)
+            handler = getattr(self, "handler_{}".format(resource_type))
+            data = handler(result)
         except forms.ValidationError as e:
             return response.Response({"errors": {"resource": e.message}}, status=400)
         except KeyError:
@@ -107,14 +126,19 @@ class WellKnownViewSet(viewsets.GenericViewSet):
                 {"errors": {"resource": "This field is required"}}, status=400
             )
 
-        handler = getattr(self, "handler_{}".format(resource_type))
-        data = handler(result)
-
         return response.Response(data)
 
     def handler_acct(self, clean_result):
         username, hostname = clean_result
-        actor = actors.SYSTEM_ACTORS[username].get_actor_instance()
+
+        if username in actors.SYSTEM_ACTORS:
+            actor = actors.SYSTEM_ACTORS[username].get_actor_instance()
+        else:
+            try:
+                actor = models.Actor.objects.local().get(user__username=username)
+            except models.Actor.DoesNotExist:
+                raise forms.ValidationError("Invalid username")
+
         return serializers.ActorWebfingerSerializer(actor).data
 
 

api/funkwhale_api/federation/webfinger.py

@@ -3,7 +3,7 @@ from django.conf import settings
 
 from funkwhale_api.common import session
 
-from . import actors, serializers
+from . import serializers
 
 VALID_RESOURCE_TYPES = ["acct"]
 
@@ -32,9 +32,6 @@ def clean_acct(acct_string, ensure_local=True):
     if ensure_local and hostname.lower() != settings.FEDERATION_HOSTNAME:
         raise forms.ValidationError("Invalid hostname {}".format(hostname))
 
-    if ensure_local and username not in actors.SYSTEM_ACTORS:
-        raise forms.ValidationError("Invalid username")
-
     return username, hostname
 
 

api/funkwhale_api/users/factories.py

@@ -4,6 +4,8 @@ from django.utils import timezone
 
 from funkwhale_api.factories import ManyToManyFromList, registry
 
+from . import models
+
 
 @registry.register
 class GroupFactory(factory.django.DjangoModelFactory):
@@ -47,6 +49,7 @@ class UserFactory(factory.django.DjangoModelFactory):
     password = factory.PostGenerationMethodCall("set_password", "test")
     subsonic_api_token = None
     groups = ManyToManyFromList("groups")
+    avatar = factory.django.ImageField()
 
     class Meta:
         model = "users.User"
@@ -71,6 +74,14 @@ class UserFactory(factory.django.DjangoModelFactory):
             # A list of permissions were passed in, use them
             self.user_permissions.add(*perms)
 
+    @factory.post_generation
+    def with_actor(self, create, extracted, **kwargs):
+        if not create or not extracted:
+            return
+        self.actor = models.create_actor(self)
+        self.save(update_fields=["actor"])
+        return self.actor
+
 
 @registry.register(name="users.SuperUser")
 class SuperUserFactory(UserFactory):

api/funkwhale_api/users/migrations/0011_auto_20180721_1317.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.0.7 on 2018-07-21 13:17
+
+from django.db import migrations, models
+import django.db.models.deletion
+import funkwhale_api.common.utils
+import funkwhale_api.common.validators
+import versatileimagefield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0006_auto_20180521_1702'),
+        ('users', '0010_user_avatar'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='user',
+            name='actor',
+            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='user', to='federation.Actor'),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='avatar',
+            field=versatileimagefield.fields.VersatileImageField(blank=True, max_length=150, null=True, upload_to=funkwhale_api.common.utils.ChunkedPath('users/avatars', preserve_file_name=False), validators=[funkwhale_api.common.validators.ImageDimensionsValidator(min_height=50, min_width=50), funkwhale_api.common.validators.FileValidator(allowed_extensions=['png', 'jpg', 'jpeg', 'gif'], max_size=2097152)]),
+        ),
+    ]

api/funkwhale_api/users/models.py

@@ -23,6 +23,9 @@ from versatileimagefield.image_warmer import VersatileImageFieldWarmer
 from funkwhale_api.common import fields, preferences
 from funkwhale_api.common import utils as common_utils
 from funkwhale_api.common import validators as common_validators
+from funkwhale_api.federation import keys
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import utils as federation_utils
 
 
 def get_token():
@@ -110,6 +113,13 @@ class User(AbstractUser):
             ),
         ],
     )
+    actor = models.OneToOneField(
+        "federation.Actor",
+        related_name="user",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+    )
 
     def __str__(self):
         return self.username
@@ -209,6 +219,34 @@ class Invitation(models.Model):
         return super().save(**kwargs)
 
 
+def create_actor(user):
+    username = user.username
+    private, public = keys.get_key_pair()
+    args = {
+        "preferred_username": username,
+        "domain": settings.FEDERATION_HOSTNAME,
+        "type": "Person",
+        "name": username,
+        "manually_approves_followers": False,
+        "url": federation_utils.full_url(
+            reverse("federation:actors-detail", kwargs={"user__username": username})
+        ),
+        "shared_inbox_url": federation_utils.full_url(
+            reverse("federation:actors-inbox", kwargs={"user__username": username})
+        ),
+        "inbox_url": federation_utils.full_url(
+            reverse("federation:actors-inbox", kwargs={"user__username": username})
+        ),
+        "outbox_url": federation_utils.full_url(
+            reverse("federation:actors-outbox", kwargs={"user__username": username})
+        ),
+    }
+    args["private_key"] = private.decode("utf-8")
+    args["public_key"] = public.decode("utf-8")
+
+    return federation_models.Actor.objects.create(**args)
+
+
 @receiver(models.signals.post_save, sender=User)
 def warm_user_avatar(sender, instance, **kwargs):
     if not instance.avatar:

api/funkwhale_api/users/serializers.py

@@ -29,6 +29,9 @@ class RegisterSerializer(RS):
         if self.validated_data.get("invitation"):
             user.invitation = self.validated_data.get("invitation")
             user.save(update_fields=["invitation"])
+        user.actor = models.create_actor(user)
+        user.save(update_fields=["actor"])
+
         return user
 
 

api/tests/federation/test_serializers.py

@@ -681,3 +681,56 @@ def test_tapi_library_track_serializer_import_pending(factories):
     serializer = serializers.APILibraryTrackSerializer(lt)
 
     assert serializer.get_status(lt) == "import_pending"
+
+
+def test_local_actor_serializer_to_ap(factories):
+    expected = {
+        "@context": [
+            "https://www.w3.org/ns/activitystreams",
+            "https://w3id.org/security/v1",
+            {},
+        ],
+        "id": "https://test.federation/user",
+        "type": "Person",
+        "following": "https://test.federation/user/following",
+        "followers": "https://test.federation/user/followers",
+        "inbox": "https://test.federation/user/inbox",
+        "outbox": "https://test.federation/user/outbox",
+        "preferredUsername": "user",
+        "name": "Real User",
+        "summary": "Hello world",
+        "manuallyApprovesFollowers": False,
+        "publicKey": {
+            "id": "https://test.federation/user#main-key",
+            "owner": "https://test.federation/user",
+            "publicKeyPem": "yolo",
+        },
+        "endpoints": {"sharedInbox": "https://test.federation/inbox"},
+    }
+    ac = models.Actor.objects.create(
+        url=expected["id"],
+        inbox_url=expected["inbox"],
+        outbox_url=expected["outbox"],
+        shared_inbox_url=expected["endpoints"]["sharedInbox"],
+        followers_url=expected["followers"],
+        following_url=expected["following"],
+        public_key=expected["publicKey"]["publicKeyPem"],
+        preferred_username=expected["preferredUsername"],
+        name=expected["name"],
+        domain="test.federation",
+        summary=expected["summary"],
+        type="Person",
+        manually_approves_followers=False,
+    )
+    user = factories["users.User"]()
+    user.actor = ac
+    user.save()
+    ac.refresh_from_db()
+    expected["icon"] = {
+        "type": "Image",
+        "mediaType": "image/jpeg",
+        "url": utils.full_url(user.avatar.crop["400x400"].url),
+    }
+    serializer = serializers.ActorSerializer(ac)
+
+    assert serializer.data == expected

api/tests/federation/test_views.py

@@ -417,3 +417,28 @@ def test_library_track_action_import(factories, superuser_api_client, mocker):
     for i, job in enumerate(batch.jobs.all()):
         assert job.library_track == imported_lts[i]
     mocked_run.assert_called_once_with(import_batch_id=batch.pk)
+
+
+def test_local_actor_detail(factories, api_client):
+    user = factories["users.User"](with_actor=True)
+    url = reverse("federation:actors-detail", kwargs={"user__username": user.username})
+    serializer = serializers.ActorSerializer(user.actor)
+    response = api_client.get(url)
+
+    assert response.status_code == 200
+    assert response.data == serializer.data
+
+
+def test_wellknown_webfinger_local(factories, api_client, settings, mocker):
+    user = factories["users.User"](with_actor=True)
+    url = reverse("federation:well-known-webfinger")
+    response = api_client.get(
+        url,
+        data={"resource": "acct:{}".format(user.actor.webfinger_subject)},
+        HTTP_ACCEPT="application/jrd+json",
+    )
+    serializer = serializers.ActorWebfingerSerializer(user.actor)
+
+    assert response.status_code == 200
+    assert response["Content-Type"] == "application/jrd+json"
+    assert response.data == serializer.data

api/tests/users/test_models.py

@@ -1,7 +1,10 @@
 import datetime
 import pytest
 
+from django.urls import reverse
+
 from funkwhale_api.users import models
+from funkwhale_api.federation import utils as federation_utils
 
 
 def test__str__(factories):
@@ -127,3 +130,26 @@ def test_can_filter_closed_invitations(factories):
 
     assert models.Invitation.objects.count() == 3
     assert list(models.Invitation.objects.order_by("id").open(False)) == [expired, used]
+
+
+def test_creating_actor_from_user(factories, settings):
+    user = factories["users.User"]()
+    actor = models.create_actor(user)
+
+    assert actor.preferred_username == user.username
+    assert actor.domain == settings.FEDERATION_HOSTNAME
+    assert actor.type == "Person"
+    assert actor.name == user.username
+    assert actor.manually_approves_followers is False
+    assert actor.url == federation_utils.full_url(
+        reverse("federation:actors-detail", kwargs={"user__username": user.username})
+    )
+    assert actor.shared_inbox_url == federation_utils.full_url(
+        reverse("federation:actors-inbox", kwargs={"user__username": user.username})
+    )
+    assert actor.inbox_url == federation_utils.full_url(
+        reverse("federation:actors-inbox", kwargs={"user__username": user.username})
+    )
+    assert actor.outbox_url == federation_utils.full_url(
+        reverse("federation:actors-outbox", kwargs={"user__username": user.username})
+    )

api/tests/users/test_views.py

@@ -249,3 +249,25 @@ def test_user_can_patch_their_own_avatar(logged_in_api_client, avatar):
     user.refresh_from_db()
 
     assert user.avatar.read() == content
+
+
+def test_creating_user_creates_actor_as_well(
+    api_client, factories, mocker, preferences
+):
+    actor = factories["federation.Actor"]()
+    url = reverse("rest_register")
+    data = {
+        "username": "test1",
+        "email": "test1@test.com",
+        "password1": "testtest",
+        "password2": "testtest",
+    }
+    preferences["users__registration_enabled"] = True
+    mocker.patch("funkwhale_api.users.models.create_actor", return_value=actor)
+    response = api_client.post(url, data)
+
+    assert response.status_code == 201
+
+    user = User.objects.get(username="test1")
+
+    assert user.actor == actor

changes/changelog.d/317.feature

@@ -0,0 +1,22 @@
+Expose ActivityPub actors for users (#317)
+
+Users now have an ActivityPub Actor [Manual action required]
+------------------------------------------------------------
+
+In the process of implementing federation for user activity such as listening
+history, we are now making user profiles (a.k.a. ActivityPub actors) available through federation.
+
+This does not means the federation is working, but this is a needed step to implement it.
+
+Those profiles will be created automatically for new users, but you have to run a command
+to create them for existing users.
+
+On docker setups::
+
+    docker-compose run --rm api python manage.py script create_actors --no-input
+
+On non-docker setups::
+
+    python manage.py script create_actors --no-input
+
+This should only take a few seconds to run. It is safe to interrupt the process or rerun it multiple times.