MirrorMirror
/
funkwhale
mirror of https://dev.funkwhale.audio/funkwhale/funkwhale.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'system-actor' into 'develop' 

System actor

See merge request funkwhale/funkwhale!577
This commit is contained in:
Eliot Berriot 2019-01-30 11:54:43 +01:00
parent 8963218bb0 253f026dc0
commit 602a4c3b29
18 changed files with 209 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/config/settings/common.py
View File

@ -94,6 +94,9 @@ FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
)
# XXX: deprecated, see #186
FEDERATION_ACTOR_FETCH_DELAY = env.int("FEDERATION_ACTOR_FETCH_DELAY", default=60 * 12)
FEDERATION_SERVICE_ACTOR_USERNAME = env(
"FEDERATION_SERVICE_ACTOR_USERNAME", default="service"
)
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNAME]
# APP CONFIGURATION

21
api/funkwhale_api/common/utils.py
View File

@ -147,3 +147,24 @@ def order_for_search(qs, field):
this function will order the given qs based on the length of the given field
"""
return qs.annotate(__size=models.functions.Length(field)).order_by("__size")
def recursive_getattr(obj, key, permissive=False):
"""
Given a dictionary such as {'user': {'name': 'Bob'}} and
a dotted string such as user.name, returns 'Bob'.
If the value is not present, returns None
"""
v = obj
for k in key.split("."):
try:
v = v.get(k)
except (TypeError, AttributeError):
if not permissive:
raise
return
if v is None:
return
return v

12
api/funkwhale_api/federation/activity.py
View File

@ -9,6 +9,8 @@ from django.db.models import Q
from funkwhale_api.common import channels
from funkwhale_api.common import utils as funkwhale_utils
recursive_getattr = funkwhale_utils.recursive_getattr
logger = logging.getLogger(__name__)
PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
@ -89,9 +91,9 @@ def should_reject(id, actor_id=None, payload={}):
media_types = ["Audio", "Artist", "Album", "Track", "Library", "Image"]
relevant_values = [
recursive_gettattr(payload, "type", permissive=True),
recursive_gettattr(payload, "object.type", permissive=True),
recursive_gettattr(payload, "target.type", permissive=True),
recursive_getattr(payload, "type", permissive=True),
recursive_getattr(payload, "object.type", permissive=True),
recursive_getattr(payload, "target.type", permissive=True),
]
# if one of the payload types match our internal media types, then
# we apply policies that reject media
@ -343,7 +345,7 @@ class OutboxRouter(Router):
return activities
def recursive_gettattr(obj, key, permissive=False):
def recursive_getattr(obj, key, permissive=False):
"""
Given a dictionary such as {'user': {'name': 'Bob'}} and
a dotted string such as user.name, returns 'Bob'.
@ -366,7 +368,7 @@ def recursive_gettattr(obj, key, permissive=False):
def match_route(route, payload):
for key, value in route.items():
payload_value = recursive_gettattr(payload, key)
payload_value = recursive_getattr(payload, key)
if payload_value != value:
return False

25
api/funkwhale_api/federation/actors.py
View File

@ -5,8 +5,9 @@ from django.conf import settings
from django.utils import timezone
from funkwhale_api.common import preferences, session
from funkwhale_api.users import models as users_models
from . import models, serializers
from . import keys, models, serializers
logger = logging.getLogger(__name__)
@ -28,7 +29,7 @@ def get_actor_data(actor_url):
def get_actor(fid, skip_cache=False):
if not skip_cache:
try:
actor = models.Actor.objects.get(fid=fid)
actor = models.Actor.objects.select_related().get(fid=fid)
except models.Actor.DoesNotExist:
actor = None
fetch_delta = datetime.timedelta(
@ -42,3 +43,23 @@ def get_actor(fid, skip_cache=False):
serializer.is_valid(raise_exception=True)
return serializer.save(last_fetch_date=timezone.now())
def get_service_actor():
name, domain = (
settings.FEDERATION_SERVICE_ACTOR_USERNAME,
settings.FEDERATION_HOSTNAME,
)
try:
return models.Actor.objects.select_related().get(
preferred_username=name, domain__name=domain
)
except models.Actor.DoesNotExist:
pass
args = users_models.get_actor_data(name)
private, public = keys.get_key_pair()
args["private_key"] = private.decode("utf-8")
args["public_key"] = public.decode("utf-8")
args["type"] = "Service"
return models.Actor.objects.create(**args)

15
api/funkwhale_api/federation/authentication.py
View File

@ -1,11 +1,13 @@
import cryptography
import logging
import datetime
from django.contrib.auth.models import AnonymousUser
from rest_framework import authentication, exceptions as rest_exceptions
from django.utils import timezone
from rest_framework import authentication, exceptions as rest_exceptions
from funkwhale_api.moderation import models as moderation_models
from . import actors, exceptions, keys, signing, utils
from . import actors, exceptions, keys, signing, tasks, utils
logger = logging.getLogger(__name__)
@ -57,6 +59,15 @@ class SignatureAuthentication(authentication.BaseAuthentication):
actor = actors.get_actor(actor_url, skip_cache=True)
signing.verify_django(request, actor.public_key.encode("utf-8"))
# we trigger a nodeinfo update on the actor's domain, if needed
fetch_delay = 24 * 3600
now = timezone.now()
last_fetch = actor.domain.nodeinfo_fetch_date
if not last_fetch or (
last_fetch < (now - datetime.timedelta(seconds=fetch_delay))
):
tasks.update_domain_nodeinfo(domain_name=actor.domain.name)
actor.domain.refresh_from_db()
return actor
def authenticate(self, request):

1
api/funkwhale_api/federation/factories.py
View File

@ -69,6 +69,7 @@ def create_user(actor):
@registry.register
class DomainFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
name = factory.Faker("domain_name")
nodeinfo_fetch_date = factory.LazyFunction(lambda: timezone.now())
class Meta:
model = "federation.Domain"

36
api/funkwhale_api/federation/migrations/0017_auto_20190130_0926.py Normal file
View File

@ -0,0 +1,36 @@
# Generated by Django 2.1.5 on 2019-01-30 09:26
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
import funkwhale_api.common.validators
import funkwhale_api.federation.models
class Migration(migrations.Migration):
dependencies = [
('federation', '0016_auto_20181227_1605'),
]
operations = [
migrations.RemoveField(
model_name='actor',
name='old_domain',
),
migrations.AddField(
model_name='domain',
name='service_actor',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='managed_domains', to='federation.Actor'),
),
migrations.AlterField(
model_name='domain',
name='name',
field=models.CharField(max_length=255, primary_key=True, serialize=False, validators=[funkwhale_api.common.validators.DomainValidator()]),
),
migrations.AlterField(
model_name='domain',
name='nodeinfo',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, max_length=50000),
),
]

12
api/funkwhale_api/federation/models.py
View File

@ -46,7 +46,9 @@ class FederationMixin(models.Model):
class ActorQuerySet(models.QuerySet):
def local(self, include=True):
return self.exclude(user__isnull=include)
if include:
return self.filter(domain__name=settings.FEDERATION_HOSTNAME)
return self.exclude(domain__name=settings.FEDERATION_HOSTNAME)
def with_current_usage(self):
qs = self
@ -92,7 +94,13 @@ class Domain(models.Model):
creation_date = models.DateTimeField(default=timezone.now)
nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True)
nodeinfo = JSONField(default=empty_dict, max_length=50000, blank=True)
service_actor = models.ForeignKey(
"Actor",
related_name="managed_domains",
on_delete=models.SET_NULL,
null=True,
blank=True,
)
objects = DomainQuerySet.as_manager()
def __str__(self):

22
api/funkwhale_api/federation/tasks.py
View File

@ -11,6 +11,7 @@ from requests.exceptions import RequestException
from funkwhale_api.common import preferences
from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils
from funkwhale_api.music import models as music_models
from funkwhale_api.taskapp import celery
@ -18,6 +19,7 @@ from . import keys
from . import models, signing
from . import serializers
from . import routes
from . import utils
logger = logging.getLogger(__name__)
@ -184,9 +186,27 @@ def update_domain_nodeinfo(domain):
nodeinfo = {"status": "ok", "payload": fetch_nodeinfo(domain.name)}
except (requests.RequestException, serializers.serializers.ValidationError) as e:
nodeinfo = {"status": "error", "error": str(e)}
service_actor_id = common_utils.recursive_getattr(
nodeinfo, "payload.metadata.actorId", permissive=True
)
try:
domain.service_actor = (
utils.retrieve_ap_object(
service_actor_id,
queryset=models.Actor,
serializer_class=serializers.ActorSerializer,
)
if service_actor_id
else None
)
except (serializers.serializers.ValidationError, RequestException) as e:
logger.warning(
"Cannot fetch system actor for domain %s: %s", domain.name, str(e)
)
domain.nodeinfo_fetch_date = now
domain.nodeinfo = nodeinfo
domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date"])
domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date", "service_actor"])
def delete_qs(qs):

2
api/funkwhale_api/instance/nodeinfo.py
View File

@ -2,6 +2,7 @@ import memoize.djangocache
import funkwhale_api
from funkwhale_api.common import preferences
from funkwhale_api.federation import actors
from . import stats
@ -19,6 +20,7 @@ def get():
"openRegistrations": preferences.get("users__registration_enabled"),
"usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}},
"metadata": {
"actorId": actors.get_service_actor().fid,
"private": preferences.get("instance__nodeinfo_private"),
"shortDescription": preferences.get("instance__short_description"),
"longDescription": preferences.get("instance__long_description"),

31
api/funkwhale_api/users/models.py
View File

@ -245,41 +245,52 @@ class Invitation(models.Model):
return super().save(**kwargs)
def get_actor_data(user):
username = federation_utils.slugify_username(user.username)
def get_actor_data(username):
slugified_username = federation_utils.slugify_username(username)
return {
"preferred_username": username,
"preferred_username": slugified_username,
"domain": federation_models.Domain.objects.get_or_create(
name=settings.FEDERATION_HOSTNAME
)[0],
"type": "Person",
"name": user.username,
"name": username,
"manually_approves_followers": False,
"fid": federation_utils.full_url(
reverse("federation:actors-detail", kwargs={"preferred_username": username})
reverse(
"federation:actors-detail",
kwargs={"preferred_username": slugified_username},
)
),
"shared_inbox_url": federation_models.get_shared_inbox_url(),
"inbox_url": federation_utils.full_url(
reverse("federation:actors-inbox", kwargs={"preferred_username": username})
reverse(
"federation:actors-inbox",
kwargs={"preferred_username": slugified_username},
)
),
"outbox_url": federation_utils.full_url(
reverse("federation:actors-outbox", kwargs={"preferred_username": username})
reverse(
"federation:actors-outbox",
kwargs={"preferred_username": slugified_username},
)
),
"followers_url": federation_utils.full_url(
reverse(
"federation:actors-followers", kwargs={"preferred_username": username}
"federation:actors-followers",
kwargs={"preferred_username": slugified_username},
)
),
"following_url": federation_utils.full_url(
reverse(
"federation:actors-following", kwargs={"preferred_username": username}
"federation:actors-following",
kwargs={"preferred_username": slugified_username},
)
),
}
def create_actor(user):
args = get_actor_data(user)
args = get_actor_data(user.username)
private, public = keys.get_key_pair()
args["private_key"] = private.decode("utf-8")
args["public_key"] = public.decode("utf-8")

12
api/tests/federation/test_actors.py
View File

@ -46,3 +46,15 @@ def test_get_actor_refresh(factories, preferences, mocker):
assert new_actor == actor
assert new_actor.last_fetch_date > actor.last_fetch_date
assert new_actor.preferred_username == "New me"
def test_get_service_actor(db, settings):
settings.FEDERATION_HOSTNAME = "test.hello"
settings.FEDERATION_SERVICE_ACTOR_USERNAME = "bob"
actor = actors.get_service_actor()
assert actor.preferred_username == "bob"
assert actor.domain.name == "test.hello"
assert actor.private_key is not None
assert actor.type == "Service"
assert actor.public_key is not None

6
api/tests/federation/test_authentication.py
View File

@ -5,6 +5,7 @@ from funkwhale_api.federation import authentication, exceptions, keys
def test_authenticate(factories, mocker, api_request):
private, public = keys.get_key_pair()
factories["federation.Domain"](name="test.federation", nodeinfo_fetch_date=None)
actor_url = "https://test.federation/actor"
mocker.patch(
"funkwhale_api.federation.actors.get_actor_data",
@ -22,6 +23,10 @@ def test_authenticate(factories, mocker, api_request):
},
},
)
update_domain_nodeinfo = mocker.patch(
"funkwhale_api.federation.tasks.update_domain_nodeinfo"
)
signed_request = factories["federation.SignedRequest"](
auth__key=private, auth__key_id=actor_url + "#main-key", auth__headers=["date"]
)
@ -40,6 +45,7 @@ def test_authenticate(factories, mocker, api_request):
assert user.is_anonymous is True
assert actor.public_key == public.decode("utf-8")
assert actor.fid == actor_url
update_domain_nodeinfo.assert_called_once_with(domain_name="test.federation")
def test_authenticate_skips_blocked_domain(factories, api_request):

18
api/tests/federation/test_tasks.py
View File

@ -161,22 +161,32 @@ def test_fetch_nodeinfo(factories, r_mock, now):
def test_update_domain_nodeinfo(factories, mocker, now):
domain = factories["federation.Domain"]()
mocker.patch.object(tasks, "fetch_nodeinfo", return_value={"hello": "world"})
domain = factories["federation.Domain"](nodeinfo_fetch_date=None)
actor = factories["federation.Actor"](fid="https://actor.id")
mocker.patch.object(
tasks,
"fetch_nodeinfo",
return_value={"hello": "world", "metadata": {"actorId": "https://actor.id"}},
)
assert domain.nodeinfo == {}
assert domain.nodeinfo_fetch_date is None
assert domain.service_actor is None
tasks.update_domain_nodeinfo(domain_name=domain.name)
domain.refresh_from_db()
assert domain.nodeinfo_fetch_date == now
assert domain.nodeinfo == {"status": "ok", "payload": {"hello": "world"}}
assert domain.nodeinfo == {
"status": "ok",
"payload": {"hello": "world", "metadata": {"actorId": "https://actor.id"}},
}
assert domain.service_actor == actor
def test_update_domain_nodeinfo_error(factories, r_mock, now):
domain = factories["federation.Domain"]()
domain = factories["federation.Domain"](nodeinfo_fetch_date=None)
wellknown_url = "https://{}/.well-known/nodeinfo".format(domain.name)
r_mock.get(wellknown_url, status_code=500)

15
api/tests/federation/test_views.py
View File

@ -2,7 +2,7 @@ import pytest
from django.core.paginator import Paginator
from django.urls import reverse
from funkwhale_api.federation import serializers, webfinger
from funkwhale_api.federation import actors, serializers, webfinger
def test_wellknown_webfinger_validates_resource(db, api_client, settings, mocker):
@ -54,6 +54,19 @@ def test_local_actor_detail(factories, api_client):
assert response.data == serializer.data
def test_service_actor_detail(factories, api_client):
actor = actors.get_service_actor()
url = reverse(
"federation:actors-detail",
kwargs={"preferred_username": actor.preferred_username},
)
serializer = serializers.ActorSerializer(actor)
response = api_client.get(url)
assert response.status_code == 200
assert response.data == serializer.data
def test_local_actor_inbox_post_requires_auth(factories, api_client):
user = factories["users.User"](with_actor=True)
url = reverse(

3
api/tests/instance/test_nodeinfo.py
View File

@ -1,5 +1,6 @@
import funkwhale_api
from funkwhale_api.instance import nodeinfo
from funkwhale_api.federation import actors
def test_nodeinfo_dump(preferences, mocker):
@ -23,6 +24,7 @@ def test_nodeinfo_dump(preferences, mocker):
"openRegistrations": preferences["users__registration_enabled"],
"usage": {"users": {"total": 1, "activeHalfyear": 12, "activeMonth": 13}},
"metadata": {
"actorId": actors.get_service_actor().fid,
"private": preferences["instance__nodeinfo_private"],
"shortDescription": preferences["instance__short_description"],
"longDescription": preferences["instance__long_description"],
@ -60,6 +62,7 @@ def test_nodeinfo_dump_stats_disabled(preferences, mocker):
"openRegistrations": preferences["users__registration_enabled"],
"usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}},
"metadata": {
"actorId": actors.get_service_actor().fid,
"private": preferences["instance__nodeinfo_private"],
"shortDescription": preferences["instance__short_description"],
"longDescription": preferences["instance__long_description"],

2
api/tests/manage/test_serializers.py
View File

@ -40,7 +40,7 @@ def test_user_update_permission(factories):
def test_manage_domain_serializer(factories, now):
domain = factories["federation.Domain"]()
domain = factories["federation.Domain"](nodeinfo_fetch_date=None)
setattr(domain, "actors_count", 42)
setattr(domain, "outbox_activities_count", 23)
expected = {

1
changes/changelog.d/system-actor.enhancement Normal file
View File

@ -0,0 +1 @@
Expose an instance-level actor (service@domain) in nodeinfo endpoint (#689)