diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py
index 22b231e08..d70ce23e5 100644
--- a/api/funkwhale_api/federation/actors.py
+++ b/api/funkwhale_api/federation/actors.py
@@ -255,8 +255,6 @@ class TestActor(SystemActor):
             accept,
             to=[ac['actor']],
             on_behalf_of=test_actor)
-        # we persist the sender in database
-        sender.save()
         models.Follow.objects.get_or_create(
             actor=sender,
             target=test_actor,
diff --git a/api/funkwhale_api/federation/authentication.py b/api/funkwhale_api/federation/authentication.py
index e199ef134..f2926bb30 100644
--- a/api/funkwhale_api/federation/authentication.py
+++ b/api/funkwhale_api/federation/authentication.py
@@ -7,6 +7,7 @@ from rest_framework import exceptions
 
 from . import actors
 from . import keys
+from . import models
 from . import serializers
 from . import signing
 from . import utils
@@ -42,7 +43,10 @@ class SignatureAuthentication(authentication.BaseAuthentication):
         except cryptography.exceptions.InvalidSignature:
             raise exceptions.AuthenticationFailed('Invalid signature')
 
-        return serializer.build()
+        try:
+            return models.Actor.objects.get(url=actor_data['id'])
+        except models.Actor.DoesNotExist:
+            return serializer.save()
 
     def authenticate(self, request):
         setattr(request, 'actor', None)