From c63b7f929d8e3a251cf3df14fcaed9fa259a5d46 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Fri, 30 Mar 2018 21:59:58 +0200 Subject: [PATCH 01/24] Use own requests-http-signing to be compatible with Signature header --- api/funkwhale_api/federation/factories.py | 1 + api/funkwhale_api/federation/signing.py | 5 +-- api/requirements/base.txt | 4 ++- api/tests/federation/test_signing.py | 39 +++++++++++------------ 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index f5d612b0d..3cfecfa96 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -15,6 +15,7 @@ class SignatureAuthFactory(factory.Factory): algorithm = 'rsa-sha256' key = factory.LazyFunction(lambda: keys.get_key_pair()[0]) key_id = factory.Faker('url') + use_auth_header = False class Meta: model = requests_http_signature.HTTPSignatureAuth diff --git a/api/funkwhale_api/federation/signing.py b/api/funkwhale_api/federation/signing.py index 87ac82bac..6f77cbd42 100644 --- a/api/funkwhale_api/federation/signing.py +++ b/api/funkwhale_api/federation/signing.py @@ -5,7 +5,8 @@ import requests_http_signature def verify(request, public_key): return requests_http_signature.HTTPSignatureAuth.verify( request, - key_resolver=lambda **kwargs: public_key + key_resolver=lambda **kwargs: public_key, + use_auth_header=False, ) @@ -20,7 +21,7 @@ def verify_django(django_request, public_key): # with requests_http_signature headers[h.lower()] = v try: - signature = headers['authorization'] + signature = headers['signature'] except KeyError: raise exceptions.MissingSignature diff --git a/api/requirements/base.txt b/api/requirements/base.txt index 02cf1c702..b66e297a9 100644 --- a/api/requirements/base.txt +++ b/api/requirements/base.txt @@ -61,4 +61,6 @@ django-cacheops>=4,<4.1 daphne==2.0.4 cryptography>=2,<3 -requests-http-signature==0.0.3 +# requests-http-signature==0.0.3 +# clone until the branch is merged and released upstream +git+https://github.com/EliotBerriot/requests-http-signature.git@signature-header-support diff --git a/api/tests/federation/test_signing.py b/api/tests/federation/test_signing.py index dc678f749..9da7a0f87 100644 --- a/api/tests/federation/test_signing.py +++ b/api/tests/federation/test_signing.py @@ -7,23 +7,23 @@ from funkwhale_api.federation import signing from funkwhale_api.federation import keys -def test_can_sign_and_verify_request(factories): - private, public = factories['federation.KeyPair']() - auth = factories['federation.SignatureAuth'](key=private) - request = factories['federation.SignedRequest']( +def test_can_sign_and_verify_request(nodb_factories): + private, public = nodb_factories['federation.KeyPair']() + auth = nodb_factories['federation.SignatureAuth'](key=private) + request = nodb_factories['federation.SignedRequest']( auth=auth ) prepared_request = request.prepare() assert 'date' in prepared_request.headers - assert 'authorization' in prepared_request.headers - assert prepared_request.headers['authorization'].startswith('Signature') - assert signing.verify(prepared_request, public) is None + assert 'signature' in prepared_request.headers + assert signing.verify( + prepared_request, public) is None -def test_can_sign_and_verify_request_digest(factories): - private, public = factories['federation.KeyPair']() - auth = factories['federation.SignatureAuth'](key=private) - request = factories['federation.SignedRequest']( +def test_can_sign_and_verify_request_digest(nodb_factories): + private, public = nodb_factories['federation.KeyPair']() + auth = nodb_factories['federation.SignatureAuth'](key=private) + request = nodb_factories['federation.SignedRequest']( auth=auth, method='post', data=b'hello=world' @@ -31,14 +31,13 @@ def test_can_sign_and_verify_request_digest(factories): prepared_request = request.prepare() assert 'date' in prepared_request.headers assert 'digest' in prepared_request.headers - assert 'authorization' in prepared_request.headers - assert prepared_request.headers['authorization'].startswith('Signature') + assert 'signature' in prepared_request.headers assert signing.verify(prepared_request, public) is None -def test_verify_fails_with_wrong_key(factories): - wrong_private, wrong_public = factories['federation.KeyPair']() - request = factories['federation.SignedRequest']() +def test_verify_fails_with_wrong_key(nodb_factories): + wrong_private, wrong_public = nodb_factories['federation.KeyPair']() + request = nodb_factories['federation.SignedRequest']() prepared_request = request.prepare() with pytest.raises(cryptography.exceptions.InvalidSignature): @@ -55,7 +54,7 @@ def test_can_verify_django_request(factories, api_request): '/', headers={ 'Date': prepared.headers['date'], - 'Authorization': prepared.headers['authorization'], + 'Signature': prepared.headers['signature'], } ) assert signing.verify_django(django_request, public_key) is None @@ -74,7 +73,7 @@ def test_can_verify_django_request_digest(factories, api_request): headers={ 'Date': prepared.headers['date'], 'Digest': prepared.headers['digest'], - 'Authorization': prepared.headers['authorization'], + 'Signature': prepared.headers['signature'], } ) @@ -94,7 +93,7 @@ def test_can_verify_django_request_digest_failure(factories, api_request): headers={ 'Date': prepared.headers['date'], 'Digest': prepared.headers['digest'] + 'noop', - 'Authorization': prepared.headers['authorization'], + 'Signature': prepared.headers['signature'], } ) @@ -112,7 +111,7 @@ def test_can_verify_django_request_failure(factories, api_request): '/', headers={ 'Date': 'Wrong', - 'Authorization': prepared.headers['authorization'], + 'Signature': prepared.headers['signature'], } ) with pytest.raises(cryptography.exceptions.InvalidSignature): From f9c649472a627429967370bf5b5203627661d61f Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 13:20:42 +0200 Subject: [PATCH 02/24] Moved duplicated dev variables to env file --- .env.dev | 6 ++++++ dev.yml | 10 ---------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.env.dev b/.env.dev index 9923c3148..2e8834143 100644 --- a/.env.dev +++ b/.env.dev @@ -1,3 +1,9 @@ API_AUTHENTICATION_REQUIRED=True RAVEN_ENABLED=false RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5 +DJANGO_ALLOWED_HOSTS=localhost,nginx +DJANGO_SETTINGS_MODULE=config.settings.local +DJANGO_SECRET_KEY=dev +C_FORCE_ROOT=true +FUNKWHALE_URL=http://localhost +PYTHONDONTWRITEBYTECODE=true diff --git a/dev.yml b/dev.yml index 126efa683..c0470a2ab 100644 --- a/dev.yml +++ b/dev.yml @@ -38,13 +38,8 @@ services: - redis command: celery -A funkwhale_api.taskapp worker -l debug environment: - - "DJANGO_ALLOWED_HOSTS=localhost" - - "DJANGO_SETTINGS_MODULE=config.settings.local" - - "DJANGO_SECRET_KEY=dev" - - C_FORCE_ROOT=true - "DATABASE_URL=postgresql://postgres@postgres/postgres" - "CACHE_URL=redis://redis:6379/0" - - "FUNKWHALE_URL=http://localhost" volumes: - ./api:/app - ./data/music:/music @@ -60,13 +55,8 @@ services: - ./api:/app - ./data/music:/music environment: - - "PYTHONDONTWRITEBYTECODE=true" - - "DJANGO_ALLOWED_HOSTS=localhost,nginx" - - "DJANGO_SETTINGS_MODULE=config.settings.local" - - "DJANGO_SECRET_KEY=dev" - "DATABASE_URL=postgresql://postgres@postgres/postgres" - "CACHE_URL=redis://redis:6379/0" - - "FUNKWHALE_URL=http://localhost" links: - postgres - redis From 22370d1b2cc482329acea80c20bfb05f99f462c2 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 14:45:11 +0200 Subject: [PATCH 03/24] Use pytest-env plugin to manage test settings instead of settings file --- api/config/settings/test.py | 29 ----------------------------- api/requirements/test.txt | 1 + api/setup.cfg | 8 +++++++- api/tests/conftest.py | 4 ++-- 4 files changed, 10 insertions(+), 32 deletions(-) delete mode 100644 api/config/settings/test.py diff --git a/api/config/settings/test.py b/api/config/settings/test.py deleted file mode 100644 index aff29c657..000000000 --- a/api/config/settings/test.py +++ /dev/null @@ -1,29 +0,0 @@ -from .common import * # noqa -SECRET_KEY = env("DJANGO_SECRET_KEY", default='test') - -# Mail settings -# ------------------------------------------------------------------------------ -EMAIL_HOST = 'localhost' -EMAIL_PORT = 1025 -EMAIL_BACKEND = env('DJANGO_EMAIL_BACKEND', - default='django.core.mail.backends.console.EmailBackend') - -# CACHING -# ------------------------------------------------------------------------------ -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': '' - } -} - -CELERY_BROKER_URL = 'memory://' - -########## CELERY -# In development, all tasks will be executed locally by blocking until the task returns -CELERY_TASK_ALWAYS_EAGER = True -########## END CELERY - -# Your local stuff: Below this line define 3rd party library settings -API_AUTHENTICATION_REQUIRED = False -CACHEOPS_ENABLED = False diff --git a/api/requirements/test.txt b/api/requirements/test.txt index e11f26ca7..20a14abea 100644 --- a/api/requirements/test.txt +++ b/api/requirements/test.txt @@ -10,4 +10,5 @@ pytest-mock pytest-sugar pytest-xdist pytest-cov +pytest-env requests-mock diff --git a/api/setup.cfg b/api/setup.cfg index 34daa8c68..a2b8b92c6 100644 --- a/api/setup.cfg +++ b/api/setup.cfg @@ -7,6 +7,12 @@ max-line-length = 120 exclude=.tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules [tool:pytest] -DJANGO_SETTINGS_MODULE=config.settings.test python_files = tests.py test_*.py *_tests.py testpaths = tests +env = + SECRET_KEY=test + DJANGO_EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend + CELERY_BROKER_URL=memory:// + CELERY_TASK_ALWAYS_EAGER=True + CACHEOPS_ENABLED=False + FEDERATION_HOSTNAME=test.federation diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 2b5a4f799..6d5e73312 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -1,8 +1,8 @@ import factory -import tempfile -import shutil import pytest import requests_mock +import shutil +import tempfile from django.contrib.auth.models import AnonymousUser from django.core.cache import cache as django_cache From 2b9a5ffe18d6002fc8629083cd8771f079ad4be1 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 15:44:35 +0200 Subject: [PATCH 04/24] ActivityPub Actor model --- .../federation/migrations/0001_initial.py | 37 +++++++++++++++++ .../federation/migrations/__init__.py | 0 api/funkwhale_api/federation/models.py | 40 +++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 api/funkwhale_api/federation/migrations/0001_initial.py create mode 100644 api/funkwhale_api/federation/migrations/__init__.py create mode 100644 api/funkwhale_api/federation/models.py diff --git a/api/funkwhale_api/federation/migrations/0001_initial.py b/api/funkwhale_api/federation/migrations/0001_initial.py new file mode 100644 index 000000000..a9157e57e --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# Generated by Django 2.0.3 on 2018-03-31 13:43 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Actor', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.URLField(db_index=True, max_length=500, unique=True)), + ('outbox_url', models.URLField(max_length=500)), + ('inbox_url', models.URLField(max_length=500)), + ('following_url', models.URLField(blank=True, max_length=500, null=True)), + ('followers_url', models.URLField(blank=True, max_length=500, null=True)), + ('shared_inbox_url', models.URLField(blank=True, max_length=500, null=True)), + ('type', models.CharField(choices=[('Person', 'Person'), ('Application', 'Application'), ('Group', 'Group'), ('Organization', 'Organization'), ('Service', 'Service')], default='Person', max_length=25)), + ('name', models.CharField(blank=True, max_length=200, null=True)), + ('domain', models.CharField(max_length=1000)), + ('summary', models.CharField(blank=True, max_length=500, null=True)), + ('preferred_username', models.CharField(blank=True, max_length=200, null=True)), + ('public_key', models.CharField(blank=True, max_length=5000, null=True)), + ('private_key', models.CharField(blank=True, max_length=5000, null=True)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('last_fetch_date', models.DateTimeField(default=django.utils.timezone.now)), + ('manually_approves_followers', models.NullBooleanField(default=None)), + ], + ), + ] diff --git a/api/funkwhale_api/federation/migrations/__init__.py b/api/funkwhale_api/federation/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py new file mode 100644 index 000000000..201307d46 --- /dev/null +++ b/api/funkwhale_api/federation/models.py @@ -0,0 +1,40 @@ +from django.conf import settings +from django.db import models +from django.utils import timezone + +TYPE_CHOICES = [ + ('Person', 'Person'), + ('Application', 'Application'), + ('Group', 'Group'), + ('Organization', 'Organization'), + ('Service', 'Service'), +] + + +class Actor(models.Model): + url = models.URLField(unique=True, max_length=500, db_index=True) + outbox_url = models.URLField(max_length=500) + inbox_url = models.URLField(max_length=500) + following_url = models.URLField(max_length=500, null=True, blank=True) + followers_url = models.URLField(max_length=500, null=True, blank=True) + shared_inbox_url = models.URLField(max_length=500, null=True, blank=True) + type = models.CharField( + choices=TYPE_CHOICES, default='Person', max_length=25) + name = models.CharField(max_length=200, null=True, blank=True) + domain = models.CharField(max_length=1000) + summary = models.CharField(max_length=500, null=True, blank=True) + preferred_username = models.CharField( + max_length=200, null=True, blank=True) + public_key = models.CharField(max_length=5000, null=True, blank=True) + private_key = models.CharField(max_length=5000, null=True, blank=True) + creation_date = models.DateTimeField(default=timezone.now) + last_fetch_date = models.DateTimeField( + default=timezone.now) + manually_approves_followers = models.NullBooleanField(default=None) + + @property + def webfinger_subject(self): + return '{}@{}'.format( + self.preferred_username, + settings.FEDERATION_HOSTNAME, + ) From 6c3b7ce1541df396b264d6b795e598692cae7d0a Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 15:44:46 +0200 Subject: [PATCH 05/24] More reserved usernames --- api/config/settings/common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 32cdb5b7f..01fb19e6c 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -396,6 +396,9 @@ PLAYLISTS_MAX_TRACKS = env.int('PLAYLISTS_MAX_TRACKS', default=250) ACCOUNT_USERNAME_BLACKLIST = [ 'funkwhale', + 'library', + 'test', + 'status', 'root', 'admin', 'owner', From 0c8faf83c506824df162b683a0249d591d84e42d Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 15:47:21 +0200 Subject: [PATCH 06/24] Can now have multiple system actors We also handle webfinger/activity serialization properly --- api/funkwhale_api/federation/actors.py | 48 ++++++ .../federation/authentication.py | 46 ++++++ api/funkwhale_api/federation/keys.py | 36 +++-- api/funkwhale_api/federation/serializers.py | 113 ++++++++++--- api/funkwhale_api/federation/urls.py | 6 +- api/funkwhale_api/federation/views.py | 30 ++-- api/funkwhale_api/federation/webfinger.py | 20 +-- api/tests/federation/test_actors.py | 42 +++++ api/tests/federation/test_authentication.py | 39 +++++ api/tests/federation/test_keys.py | 33 ++-- api/tests/federation/test_serializers.py | 152 +++++++++++++++--- api/tests/federation/test_views.py | 52 +++--- api/tests/federation/test_webfinger.py | 28 +--- 13 files changed, 493 insertions(+), 152 deletions(-) create mode 100644 api/funkwhale_api/federation/actors.py create mode 100644 api/funkwhale_api/federation/authentication.py create mode 100644 api/tests/federation/test_actors.py create mode 100644 api/tests/federation/test_authentication.py diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py new file mode 100644 index 000000000..5560520c2 --- /dev/null +++ b/api/funkwhale_api/federation/actors.py @@ -0,0 +1,48 @@ +import requests + +from django.urls import reverse +from django.conf import settings + +from dynamic_preferences.registries import global_preferences_registry + +from . import models + + +def get_actor_data(actor_url): + response = requests.get(actor_url) + response.raise_for_status() + return response.json() + + +SYSTEM_ACTORS = { + 'library': { + 'get_actor': lambda: models.Actor(**get_base_system_actor_arguments('library')), + } +} + + +def get_base_system_actor_arguments(name): + preferences = global_preferences_registry.manager() + return { + 'preferred_username': name, + 'domain': settings.FEDERATION_HOSTNAME, + 'type': 'Person', + 'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME), + 'manually_approves_followers': True, + 'url': reverse( + 'federation:instance-actors-detail', + kwargs={'actor': name}), + 'shared_inbox_url': reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': name}), + 'inbox_url': reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': name}), + 'outbox_url': reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': name}), + 'public_key': preferences['federation__public_key'], + 'summary': 'Bot account to federate with {}\'s library'.format( + settings.FEDERATION_HOSTNAME + ), + } diff --git a/api/funkwhale_api/federation/authentication.py b/api/funkwhale_api/federation/authentication.py new file mode 100644 index 000000000..7972e78a0 --- /dev/null +++ b/api/funkwhale_api/federation/authentication.py @@ -0,0 +1,46 @@ +import cryptography + +from django.contrib.auth.models import AnonymousUser + +from rest_framework import authentication +from rest_framework import exceptions + +from . import actors +from . import keys +from . import serializers +from . import signing + + +class SignatureAuthentication(authentication.BaseAuthentication): + def authenticate(self, request): + try: + signature = request.META['headers']['Signature'] + key_id = keys.get_key_id_from_signature_header(signature) + except KeyError: + raise exceptions.AuthenticationFailed('No signature') + except ValueError as e: + raise exceptions.AuthenticationFailed(str(e)) + + try: + actor_data = actors.get_actor_data(key_id) + except Exception as e: + raise exceptions.AuthenticationFailed(str(e)) + + try: + public_key = actor_data['publicKey']['publicKeyPem'] + except KeyError: + raise exceptions.AuthenticationFailed('No public key found') + + serializer = serializers.ActorSerializer(data=actor_data) + if not serializer.is_valid(): + raise exceptions.AuthenticationFailed('Invalid actor payload') + + try: + signing.verify_django(request, public_key.encode('utf-8')) + except cryptography.exceptions.InvalidSignature: + raise exceptions.AuthenticationFailed('Invalid signature') + + user = AnonymousUser() + ac = serializer.build() + setattr(request, 'actor', ac) + return (user, None) diff --git a/api/funkwhale_api/federation/keys.py b/api/funkwhale_api/federation/keys.py index 432560ef7..08d4034ea 100644 --- a/api/funkwhale_api/federation/keys.py +++ b/api/funkwhale_api/federation/keys.py @@ -2,10 +2,14 @@ from cryptography.hazmat.primitives import serialization as crypto_serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.backends import default_backend as crypto_default_backend +import re import requests +import urllib.parse from . import exceptions +KEY_ID_REGEX = re.compile(r'keyId=\"(?P.*)\"') + def get_key_pair(size=2048): key = rsa.generate_private_key( @@ -25,19 +29,21 @@ def get_key_pair(size=2048): return private_key, public_key -def get_public_key(actor_url): - """ - Given an actor_url, request it and extract publicKey data from - the response payload. - """ - response = requests.get(actor_url) - response.raise_for_status() - payload = response.json() +def get_key_id_from_signature_header(header_string): + parts = header_string.split(',') try: - return { - 'public_key_pem': payload['publicKey']['publicKeyPem'], - 'id': payload['publicKey']['id'], - 'owner': payload['publicKey']['owner'], - } - except KeyError: - raise exceptions.MalformedPayload(str(payload)) + raw_key_id = [p for p in parts if p.startswith('keyId="')][0] + except IndexError: + raise ValueError('Missing key id') + + match = KEY_ID_REGEX.match(raw_key_id) + if not match: + raise ValueError('Invalid key id') + + key_id = match.groups()[0] + url = urllib.parse.urlparse(key_id) + if not url.scheme or not url.netloc: + raise ValueError('Invalid url') + if url.scheme not in ['http', 'https']: + raise ValueError('Invalid shceme') + return key_id diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index d1533b62d..5f5516b2d 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -1,38 +1,101 @@ +import urllib.parse + from django.urls import reverse from django.conf import settings +from rest_framework import serializers from dynamic_preferences.registries import global_preferences_registry +from . import models from . import utils -def repr_instance_actor(): - """ - We do not use a serializer here, since it's pretty static - """ - actor_url = utils.full_url(reverse('federation:instance-actor')) - preferences = global_preferences_registry.manager() - public_key = preferences['federation__public_key'] +class ActorSerializer(serializers.ModelSerializer): + # left maps to activitypub fields, right to our internal models + id = serializers.URLField(source='url') + outbox = serializers.URLField(source='outbox_url') + inbox = serializers.URLField(source='inbox_url') + following = serializers.URLField(source='following_url', required=False) + followers = serializers.URLField(source='followers_url', required=False) + preferredUsername = serializers.CharField( + source='preferred_username', required=False) + publicKey = serializers.JSONField(source='public_key', required=False) + manuallyApprovesFollowers = serializers.NullBooleanField( + source='manually_approves_followers', required=False) - return { - '@context': [ + class Meta: + model = models.Actor + fields = [ + 'id', + 'type', + 'name', + 'summary', + 'preferredUsername', + 'publicKey', + 'inbox', + 'outbox', + 'following', + 'followers', + 'manuallyApprovesFollowers', + ] + + def to_representation(self, instance): + ret = super().to_representation(instance) + ret['@context'] = [ 'https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1', {}, - ], - 'id': utils.full_url(reverse('federation:instance-actor')), - 'type': 'Person', - 'inbox': utils.full_url(reverse('federation:instance-inbox')), - 'outbox': utils.full_url(reverse('federation:instance-outbox')), - 'preferredUsername': 'service', - 'name': 'Service Bot - {}'.format(settings.FEDERATION_HOSTNAME), - 'summary': 'Bot account for federating with {}'.format( - settings.FEDERATION_HOSTNAME - ), - 'publicKey': { - 'id': '{}#main-key'.format(actor_url), - 'owner': actor_url, - 'publicKeyPem': public_key - }, + ] + if instance.public_key: + ret['publicKey'] = { + 'owner': instance.url, + 'publicKeyPem': instance.public_key, + 'id': '{}#main-key'.format(instance.url) + } + ret['endpoints'] = {} + if instance.shared_inbox_url: + ret['endpoints']['sharedInbox'] = instance.shared_inbox_url + return ret - } + def prepare_missing_fields(self): + kwargs = {} + domain = urllib.parse.urlparse(self.validated_data['url']).netloc + kwargs['domain'] = domain + for endpoint, url in self.initial_data.get('endpoints', {}).items(): + if endpoint == 'sharedInbox': + kwargs['shared_inbox_url'] = url + break + try: + kwargs['public_key'] = self.initial_data['publicKey']['publicKeyPem'] + except KeyError: + pass + return kwargs + + def build(self): + d = self.validated_data.copy() + d.update(self.prepare_missing_fields()) + return self.Meta.model(**d) + + def save(self, **kwargs): + kwargs.update(self.prepare_missing_fields()) + return super().save(**kwargs) + +class ActorWebfingerSerializer(serializers.ModelSerializer): + class Meta: + model = models.Actor + fields = ['url'] + + def to_representation(self, instance): + data = {} + data['subject'] = 'acct:{}'.format(instance.webfinger_subject) + data['links'] = [ + { + 'rel': 'self', + 'href': instance.url, + 'type': 'application/activity+json' + } + ] + data['aliases'] = [ + instance.url + ] + return data diff --git a/api/funkwhale_api/federation/urls.py b/api/funkwhale_api/federation/urls.py index 5b7895451..f2c6f4c78 100644 --- a/api/funkwhale_api/federation/urls.py +++ b/api/funkwhale_api/federation/urls.py @@ -4,9 +4,9 @@ from . import views router = routers.SimpleRouter(trailing_slash=False) router.register( - r'federation/instance', - views.InstanceViewSet, - 'instance') + r'federation/instance/actors', + views.InstanceActorViewSet, + 'instance-actors') router.register( r'.well-known', views.WellKnownViewSet, diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 5f1ee36f7..dcb806224 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -5,8 +5,9 @@ from django.http import HttpResponse from rest_framework import viewsets from rest_framework import views from rest_framework import response -from rest_framework.decorators import list_route +from rest_framework.decorators import list_route, detail_route +from . import actors from . import renderers from . import serializers from . import webfinger @@ -19,20 +20,30 @@ class FederationMixin(object): return super().dispatch(request, *args, **kwargs) -class InstanceViewSet(FederationMixin, viewsets.GenericViewSet): +class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet): + lookup_field = 'actor' + lookup_value_regex = '[a-z]*' authentication_classes = [] permission_classes = [] renderer_classes = [renderers.ActivityPubRenderer] - @list_route(methods=['get']) - def actor(self, request, *args, **kwargs): - return response.Response(serializers.repr_instance_actor()) + def get_object(self): + try: + return actors.SYSTEM_ACTORS[self.kwargs['actor']] + except KeyError: + raise Http404 - @list_route(methods=['get']) + def retrieve(self, request, *args, **kwargs): + actor_conf = self.get_object() + actor = actor_conf['get_actor']() + serializer = serializers.ActorSerializer(actor) + return response.Response(serializer.data, status=200) + + @detail_route(methods=['get']) def inbox(self, request, *args, **kwargs): raise NotImplementedError() - @list_route(methods=['get']) + @detail_route(methods=['get']) def outbox(self, request, *args, **kwargs): raise NotImplementedError() @@ -69,6 +80,5 @@ class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet): def handler_acct(self, clean_result): username, hostname = clean_result - if username == 'service': - return webfinger.serialize_system_acct() - return {} + actor = actors.SYSTEM_ACTORS[username]['get_actor']() + return serializers.ActorWebfingerSerializer(actor).data diff --git a/api/funkwhale_api/federation/webfinger.py b/api/funkwhale_api/federation/webfinger.py index a9281c2b5..d698114f1 100644 --- a/api/funkwhale_api/federation/webfinger.py +++ b/api/funkwhale_api/federation/webfinger.py @@ -2,7 +2,9 @@ from django import forms from django.conf import settings from django.urls import reverse +from . import actors from . import utils + VALID_RESOURCE_TYPES = ['acct'] @@ -30,23 +32,7 @@ def clean_acct(acct_string): if hostname != settings.FEDERATION_HOSTNAME: raise forms.ValidationError('Invalid hostname') - if username != 'service': + if username not in actors.SYSTEM_ACTORS: raise forms.ValidationError('Invalid username') return username, hostname - - -def serialize_system_acct(): - return { - 'subject': 'acct:service@{}'.format(settings.FEDERATION_HOSTNAME), - 'aliases': [ - utils.full_url(reverse('federation:instance-actor')) - ], - 'links': [ - { - 'rel': 'self', - 'type': 'application/activity+json', - 'href': utils.full_url(reverse('federation:instance-actor')), - } - ] - } diff --git a/api/tests/federation/test_actors.py b/api/tests/federation/test_actors.py new file mode 100644 index 000000000..cd4b0b82f --- /dev/null +++ b/api/tests/federation/test_actors.py @@ -0,0 +1,42 @@ +from django.urls import reverse + +from funkwhale_api.federation import actors + + +def test_actor_fetching(r_mock): + payload = { + 'id': 'https://actor.mock/users/actor#main-key', + 'owner': 'test', + 'publicKeyPem': 'test_pem', + } + actor_url = 'https://actor.mock/' + r_mock.get(actor_url, json=payload) + r = actors.get_actor_data(actor_url) + + assert r == payload + + +def test_get_library(settings, preferences): + preferences['federation__public_key'] = 'public_key' + expected = { + 'preferred_username': 'library', + 'domain': settings.FEDERATION_HOSTNAME, + 'type': 'Person', + 'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME), + 'manually_approves_followers': True, + 'url': reverse( + 'federation:instance-actors-detail', + kwargs={'actor': 'library'}), + 'shared_inbox_url': reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': 'library'}), + 'inbox_url': reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': 'library'}), + 'public_key': 'public_key', + 'summary': 'Bot account to federate with {}\'s library'.format( + settings.FEDERATION_HOSTNAME), + } + actor = actors.SYSTEM_ACTORS['library']['get_actor']() + for key, value in expected.items(): + assert getattr(actor, key) == value diff --git a/api/tests/federation/test_authentication.py b/api/tests/federation/test_authentication.py new file mode 100644 index 000000000..0c4a25a7f --- /dev/null +++ b/api/tests/federation/test_authentication.py @@ -0,0 +1,39 @@ +from funkwhale_api.federation import authentication +from funkwhale_api.federation import keys +from funkwhale_api.federation import signing + + +def test_authenticate(nodb_factories, mocker, api_request): + private, public = keys.get_key_pair() + actor_url = 'https://test.federation/actor' + mocker.patch( + 'funkwhale_api.federation.actors.get_actor_data', + return_value={ + 'id': actor_url, + 'outbox': 'https://test.com', + 'inbox': 'https://test.com', + 'publicKey': { + 'publicKeyPem': public.decode('utf-8'), + 'owner': actor_url, + 'id': actor_url + '#main-key', + } + }) + signed_request = nodb_factories['federation.SignedRequest']( + auth__key=private, + auth__key_id=actor_url + '#main-key' + ) + prepared = signed_request.prepare() + django_request = api_request.get( + '/', + headers={ + 'Date': prepared.headers['date'], + 'Signature': prepared.headers['signature'], + } + ) + authenticator = authentication.SignatureAuthentication() + user, _ = authenticator.authenticate(django_request) + actor = django_request.actor + + assert user.is_anonymous is True + assert actor.public_key == public.decode('utf-8') + assert actor.url == actor_url diff --git a/api/tests/federation/test_keys.py b/api/tests/federation/test_keys.py index 1c30c30b1..9dd71be09 100644 --- a/api/tests/federation/test_keys.py +++ b/api/tests/federation/test_keys.py @@ -1,16 +1,25 @@ +import pytest + from funkwhale_api.federation import keys -def test_public_key_fetching(r_mock): - payload = { - 'id': 'https://actor.mock/users/actor#main-key', - 'owner': 'test', - 'publicKeyPem': 'test_pem', - } - actor = 'https://actor.mock/' - r_mock.get(actor, json={'publicKey': payload}) - r = keys.get_public_key(actor) +@pytest.mark.parametrize('raw, expected', [ + ('algorithm="test",keyId="https://test.com"', 'https://test.com'), + ('keyId="https://test.com",algorithm="test"', 'https://test.com'), +]) +def test_get_key_from_header(raw, expected): + r = keys.get_key_id_from_signature_header(raw) + assert r == expected - assert r['id'] == payload['id'] - assert r['owner'] == payload['owner'] - assert r['public_key_pem'] == payload['publicKeyPem'] + +@pytest.mark.parametrize('raw', [ + 'algorithm="test",keyid="badCase"', + 'algorithm="test",wrong="wrong"', + 'keyId = "wrong"', + 'keyId=\'wrong\'', + 'keyId="notanurl"', + 'keyId="wrong://test.com"', +]) +def test_get_key_from_header_invalid(raw): + with pytest.raises(ValueError): + keys.get_key_id_from_signature_header(raw) diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index 18b8525f2..efa92b16a 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -1,36 +1,146 @@ from django.urls import reverse from funkwhale_api.federation import keys +from funkwhale_api.federation import models from funkwhale_api.federation import serializers -def test_repr_instance_actor(db, preferences, settings): - _, public_key = keys.get_key_pair() - preferences['federation__public_key'] = public_key.decode('utf-8') - settings.FEDERATION_HOSTNAME = 'test.federation' - settings.FUNKWHALE_URL = 'https://test.federation' - actor_url = settings.FUNKWHALE_URL + reverse('federation:instance-actor') - inbox_url = settings.FUNKWHALE_URL + reverse('federation:instance-inbox') - outbox_url = settings.FUNKWHALE_URL + reverse('federation:instance-outbox') +def test_actor_serializer_from_ap(db): + payload = { + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', + 'name': 'Real User', + 'summary': 'Hello world', + 'url': 'https://test.federation/@user', + 'manuallyApprovesFollowers': False, + 'publicKey': { + 'id': 'https://test.federation/user#main-key', + 'owner': 'https://test.federation/user', + 'publicKeyPem': 'yolo' + }, + 'endpoints': { + 'sharedInbox': 'https://test.federation/inbox' + }, + } + serializer = serializers.ActorSerializer(data=payload) + assert serializer.is_valid() + + actor = serializer.build() + + assert actor.url == payload['id'] + assert actor.inbox_url == payload['inbox'] + assert actor.outbox_url == payload['outbox'] + assert actor.shared_inbox_url == payload['endpoints']['sharedInbox'] + assert actor.followers_url == payload['followers'] + assert actor.following_url == payload['following'] + assert actor.public_key == payload['publicKey']['publicKeyPem'] + assert actor.preferred_username == payload['preferredUsername'] + assert actor.name == payload['name'] + assert actor.domain == 'test.federation' + assert actor.summary == payload['summary'] + assert actor.type == 'Person' + assert actor.manually_approves_followers == payload['manuallyApprovesFollowers'] + + +def test_actor_serializer_only_mandatory_field_from_ap(db): + payload = { + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', + } + + serializer = serializers.ActorSerializer(data=payload) + assert serializer.is_valid() + + actor = serializer.build() + + assert actor.url == payload['id'] + assert actor.inbox_url == payload['inbox'] + assert actor.outbox_url == payload['outbox'] + assert actor.followers_url == payload['followers'] + assert actor.following_url == payload['following'] + assert actor.preferred_username == payload['preferredUsername'] + assert actor.domain == 'test.federation' + assert actor.type == 'Person' + assert actor.manually_approves_followers is None + + +def test_actor_serializer_to_ap(): expected = { '@context': [ 'https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1', {}, ], - 'id': actor_url, - 'type': 'Person', - 'preferredUsername': 'service', - 'name': 'Service Bot - test.federation', - 'summary': 'Bot account for federating with test.federation', - 'inbox': inbox_url, - 'outbox': outbox_url, - 'publicKey': { - 'id': '{}#main-key'.format(actor_url), - 'owner': actor_url, - 'publicKeyPem': public_key.decode('utf-8') - }, + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', + 'name': 'Real User', + 'summary': 'Hello world', + 'manuallyApprovesFollowers': False, + 'publicKey': { + 'id': 'https://test.federation/user#main-key', + 'owner': 'https://test.federation/user', + 'publicKeyPem': 'yolo' + }, + 'endpoints': { + 'sharedInbox': 'https://test.federation/inbox' + }, } + ac = models.Actor( + url=expected['id'], + inbox_url=expected['inbox'], + outbox_url=expected['outbox'], + shared_inbox_url=expected['endpoints']['sharedInbox'], + followers_url=expected['followers'], + following_url=expected['following'], + public_key=expected['publicKey']['publicKeyPem'], + preferred_username=expected['preferredUsername'], + name=expected['name'], + domain='test.federation', + summary=expected['summary'], + type='Person', + manually_approves_followers=False, - assert expected == serializers.repr_instance_actor() + ) + serializer = serializers.ActorSerializer(ac) + + assert serializer.data == expected + + +def test_webfinger_serializer(): + expected = { + 'subject': 'acct:service@test.federation', + 'links': [ + { + 'rel': 'self', + 'href': 'https://test.federation/federation/instance/actor', + 'type': 'application/activity+json', + } + ], + 'aliases': [ + 'https://test.federation/federation/instance/actor', + ] + } + actor = models.Actor( + url=expected['links'][0]['href'], + preferred_username='service', + domain='test.federation', + ) + serializer = serializers.ActorWebfingerSerializer(actor) + + assert serializer.data == expected diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index 6a8de8c14..96cf8ff7f 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -2,38 +2,43 @@ from django.urls import reverse import pytest +from funkwhale_api.federation import actors from funkwhale_api.federation import serializers from funkwhale_api.federation import webfinger -def test_instance_actor(db, settings, api_client): - settings.FUNKWHALE_URL = 'http://test.com' - url = reverse('federation:instance-actor') + +@pytest.mark.parametrize('system_actor', actors.SYSTEM_ACTORS.keys()) +def test_instance_actors(system_actor, db, settings, api_client): + actor = actors.SYSTEM_ACTORS[system_actor]['get_actor']() + url = reverse( + 'federation:instance-actors-detail', + kwargs={'actor': system_actor}) response = api_client.get(url) + serializer = serializers.ActorSerializer(actor) assert response.status_code == 200 - assert response.data == serializers.repr_instance_actor() + assert response.data == serializer.data -@pytest.mark.parametrize('route', [ - 'instance-outbox', - 'instance-inbox', - 'instance-actor', - 'well-known-webfinger', -]) -def test_instance_inbox_405_if_federation_disabled( - db, settings, api_client, route): - settings.FEDERATION_ENABLED = False - url = reverse('federation:{}'.format(route)) - response = api_client.get(url) - - assert response.status_code == 405 +# @pytest.mark.parametrize('route', [ +# 'instance-outbox', +# 'instance-inbox', +# 'instance-actor', +# 'well-known-webfinger', +# ]) +# def test_instance_inbox_405_if_federation_disabled( +# db, settings, api_client, route): +# settings.FEDERATION_ENABLED = False +# url = reverse('federation:{}'.format(route)) +# response = api_client.get(url) +# +# assert response.status_code == 405 def test_wellknown_webfinger_validates_resource( db, api_client, settings, mocker): clean = mocker.spy(webfinger, 'clean_resource') - settings.FEDERATION_ENABLED = True url = reverse('federation:well-known-webfinger') response = api_client.get(url, data={'resource': 'something'}) @@ -45,14 +50,15 @@ def test_wellknown_webfinger_validates_resource( ) +@pytest.mark.parametrize('system_actor', actors.SYSTEM_ACTORS.keys()) def test_wellknown_webfinger_system( - db, api_client, settings, mocker): - settings.FEDERATION_ENABLED = True - settings.FEDERATION_HOSTNAME = 'test.federation' + system_actor, db, api_client, settings, mocker): + actor = actors.SYSTEM_ACTORS[system_actor]['get_actor']() url = reverse('federation:well-known-webfinger') response = api_client.get( - url, data={'resource': 'acct:service@test.federation'}) + url, data={'resource': 'acct:{}'.format(actor.webfinger_subject)}) + serializer = serializers.ActorWebfingerSerializer(actor) assert response.status_code == 200 assert response['Content-Type'] == 'application/jrd+json' - assert response.data == webfinger.serialize_system_acct() + assert response.data == serializer.data diff --git a/api/tests/federation/test_webfinger.py b/api/tests/federation/test_webfinger.py index d2b00f8f1..fd1cb1d05 100644 --- a/api/tests/federation/test_webfinger.py +++ b/api/tests/federation/test_webfinger.py @@ -25,9 +25,8 @@ def test_webfinger_clean_resource_errors(resource, message): def test_webfinger_clean_acct(settings): - settings.FEDERATION_HOSTNAME = 'test.federation' - username, hostname = webfinger.clean_acct('service@test.federation') - assert username == 'service' + username, hostname = webfinger.clean_acct('library@test.federation') + assert username == 'library' assert hostname == 'test.federation' @@ -37,30 +36,7 @@ def test_webfinger_clean_acct(settings): ('noop@test.federation', 'Invalid account'), ]) def test_webfinger_clean_acct_errors(resource, message, settings): - settings.FEDERATION_HOSTNAME = 'test.federation' - with pytest.raises(forms.ValidationError) as excinfo: webfinger.clean_resource(resource) assert message == str(excinfo) - - -def test_service_serializer(settings): - settings.FEDERATION_HOSTNAME = 'test.federation' - settings.FUNKWHALE_URL = 'https://test.federation' - - expected = { - 'subject': 'acct:service@test.federation', - 'links': [ - { - 'rel': 'self', - 'href': 'https://test.federation/federation/instance/actor', - 'type': 'application/activity+json', - } - ], - 'aliases': [ - 'https://test.federation/federation/instance/actor', - ] - } - - assert expected == webfinger.serialize_system_acct() From 703d70d54496fcc37222daeda0ab9443f0e48bee Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 15:57:29 +0200 Subject: [PATCH 07/24] Use absolute urls --- api/funkwhale_api/federation/actors.py | 29 +++++++++++++++----------- api/tests/federation/test_actors.py | 22 +++++++++++-------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 5560520c2..823d163f9 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -6,6 +6,7 @@ from django.conf import settings from dynamic_preferences.registries import global_preferences_registry from . import models +from . import utils def get_actor_data(actor_url): @@ -29,18 +30,22 @@ def get_base_system_actor_arguments(name): 'type': 'Person', 'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME), 'manually_approves_followers': True, - 'url': reverse( - 'federation:instance-actors-detail', - kwargs={'actor': name}), - 'shared_inbox_url': reverse( - 'federation:instance-actors-inbox', - kwargs={'actor': name}), - 'inbox_url': reverse( - 'federation:instance-actors-inbox', - kwargs={'actor': name}), - 'outbox_url': reverse( - 'federation:instance-actors-outbox', - kwargs={'actor': name}), + 'url': utils.full_url( + reverse( + 'federation:instance-actors-detail', + kwargs={'actor': name})), + 'shared_inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': name})), + 'inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': name})), + 'outbox_url': utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': name})), 'public_key': preferences['federation__public_key'], 'summary': 'Bot account to federate with {}\'s library'.format( settings.FEDERATION_HOSTNAME diff --git a/api/tests/federation/test_actors.py b/api/tests/federation/test_actors.py index cd4b0b82f..00e214bd1 100644 --- a/api/tests/federation/test_actors.py +++ b/api/tests/federation/test_actors.py @@ -1,6 +1,7 @@ from django.urls import reverse from funkwhale_api.federation import actors +from funkwhale_api.federation import utils def test_actor_fetching(r_mock): @@ -24,15 +25,18 @@ def test_get_library(settings, preferences): 'type': 'Person', 'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME), 'manually_approves_followers': True, - 'url': reverse( - 'federation:instance-actors-detail', - kwargs={'actor': 'library'}), - 'shared_inbox_url': reverse( - 'federation:instance-actors-inbox', - kwargs={'actor': 'library'}), - 'inbox_url': reverse( - 'federation:instance-actors-inbox', - kwargs={'actor': 'library'}), + 'url': utils.full_url( + reverse( + 'federation:instance-actors-detail', + kwargs={'actor': 'library'})), + 'shared_inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': 'library'})), + 'inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': 'library'})), 'public_key': 'public_key', 'summary': 'Bot account to federate with {}\'s library'.format( settings.FEDERATION_HOSTNAME), From 99e7e98bae055065b7eb8890631f6b9c009a1ee0 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 16:25:23 +0200 Subject: [PATCH 08/24] Fixed broken dev entrypoint --- api/compose/django/dev-entrypoint.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/compose/django/dev-entrypoint.sh b/api/compose/django/dev-entrypoint.sh index 416207b43..6deeebb00 100755 --- a/api/compose/django/dev-entrypoint.sh +++ b/api/compose/django/dev-entrypoint.sh @@ -1,7 +1,3 @@ #!/bin/bash set -e -if [ $1 = "pytest" ]; then - # let pytest.ini handle it - unset DJANGO_SETTINGS_MODULE -fi exec "$@" From 043153a520d81998544d8edacb9b43fbe85375ef Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 17:47:15 +0200 Subject: [PATCH 09/24] Set host properly on nginx dev container --- docker/nginx/entrypoint.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/nginx/entrypoint.sh b/docker/nginx/entrypoint.sh index 93b4a0533..1819acf1c 100755 --- a/docker/nginx/entrypoint.sh +++ b/docker/nginx/entrypoint.sh @@ -1,8 +1,9 @@ #!/bin/bash -eux - +FIRST_HOST=$(echo ${DJANGO_ALLOWED_HOSTS} | cut -d, -f1) echo "Copying template file..." cp /etc/nginx/funkwhale_proxy.conf{.template,} -sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host localhost:${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host ${FIRST_HOST}:${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/proxy_set_header Host \$host/proxy_set_header Host ${FIRST_HOST}/" /etc/nginx/funkwhale_proxy.conf sed -i "s/proxy_set_header X-Forwarded-Port \$server_port/proxy_set_header X-Forwarded-Port ${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf cat /etc/nginx/funkwhale_proxy.conf From 46d40c7ffae0e998ead307669559711ff95193a2 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 18:39:10 +0200 Subject: [PATCH 10/24] Util function to convert django meta to proper headers --- api/funkwhale_api/federation/utils.py | 21 +++++++++++++++++ api/tests/federation/test_utils.py | 34 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/api/funkwhale_api/federation/utils.py b/api/funkwhale_api/federation/utils.py index e83f54b5d..df093add8 100644 --- a/api/funkwhale_api/federation/utils.py +++ b/api/funkwhale_api/federation/utils.py @@ -12,3 +12,24 @@ def full_url(path): return root + '/' + path else: return root + path + + +def clean_wsgi_headers(raw_headers): + """ + Convert WSGI headers from CONTENT_TYPE to Content-Type notation + """ + cleaned = {} + non_prefixed = [ + 'content_type', + 'content_length', + ] + for raw_header, value in raw_headers.items(): + h = raw_header.lower() + if not h.startswith('http_') and h not in non_prefixed: + continue + + words = h.replace('http_', '', 1).split('_') + cleaned_header = '-'.join([w.capitalize() for w in words]) + cleaned[cleaned_header] = value + + return cleaned diff --git a/api/tests/federation/test_utils.py b/api/tests/federation/test_utils.py index 8bada65bb..dc371ad9e 100644 --- a/api/tests/federation/test_utils.py +++ b/api/tests/federation/test_utils.py @@ -12,3 +12,37 @@ from funkwhale_api.federation import utils def test_full_url(settings, url, path, expected): settings.FUNKWHALE_URL = url assert utils.full_url(path) == expected + + +def test_extract_headers_from_meta(): + wsgi_headers = { + 'HTTP_HOST': 'nginx', + 'HTTP_X_REAL_IP': '172.20.0.4', + 'HTTP_X_FORWARDED_FOR': '188.165.228.227, 172.20.0.4', + 'HTTP_X_FORWARDED_PROTO': 'http', + 'HTTP_X_FORWARDED_HOST': 'localhost:80', + 'HTTP_X_FORWARDED_PORT': '80', + 'HTTP_CONNECTION': 'close', + 'CONTENT_LENGTH': '1155', + 'CONTENT_TYPE': 'txt/application', + 'HTTP_SIGNATURE': 'Hello', + 'HTTP_DATE': 'Sat, 31 Mar 2018 13:53:55 GMT', + 'HTTP_USER_AGENT': 'http.rb/3.0.0 (Mastodon/2.2.0; +https://mastodon.eliotberriot.com/)'} + + cleaned_headers = utils.clean_wsgi_headers(wsgi_headers) + + expected = { + 'Host': 'nginx', + 'X-Real-Ip': '172.20.0.4', + 'X-Forwarded-For': '188.165.228.227, 172.20.0.4', + 'X-Forwarded-Proto': 'http', + 'X-Forwarded-Host': 'localhost:80', + 'X-Forwarded-Port': '80', + 'Connection': 'close', + 'Content-Length': '1155', + 'Content-Type': 'txt/application', + 'Signature': 'Hello', + 'Date': 'Sat, 31 Mar 2018 13:53:55 GMT', + 'User-Agent': 'http.rb/3.0.0 (Mastodon/2.2.0; +https://mastodon.eliotberriot.com/)' + } + assert cleaned_headers == expected From b5a4b2ca6a931fcea29035b754932abb51a16393 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 18:39:32 +0200 Subject: [PATCH 11/24] Added signature authentication on activitypub view --- api/funkwhale_api/federation/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index dcb806224..95e421b59 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -8,6 +8,7 @@ from rest_framework import response from rest_framework.decorators import list_route, detail_route from . import actors +from . import authentication from . import renderers from . import serializers from . import webfinger @@ -23,7 +24,8 @@ class FederationMixin(object): class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet): lookup_field = 'actor' lookup_value_regex = '[a-z]*' - authentication_classes = [] + authentication_classes = [ + authentication.SignatureAuthentication] permission_classes = [] renderer_classes = [renderers.ActivityPubRenderer] From de777764da0c0e9fe66d0bb76317679be964588b Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 18:39:54 +0200 Subject: [PATCH 12/24] Fake_request fixture for django requests --- api/tests/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 6d5e73312..d5bb56565 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -6,6 +6,8 @@ import tempfile from django.contrib.auth.models import AnonymousUser from django.core.cache import cache as django_cache +from django.test import client + from dynamic_preferences.registries import global_preferences_registry from rest_framework.test import APIClient @@ -118,6 +120,11 @@ def api_request(): return APIRequestFactory() +@pytest.fixture +def fake_request(): + return client.RequestFactory() + + @pytest.fixture def activity_registry(): r = record.registry From e1ebd4988bc41731abef86eb05a836077e4826a7 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 18:40:41 +0200 Subject: [PATCH 13/24] Fixed inconsistencies between test and prod requests --- .../federation/authentication.py | 17 +++-- api/funkwhale_api/federation/signing.py | 30 +++++++-- api/tests/federation/test_authentication.py | 11 ++-- api/tests/federation/test_signing.py | 66 +++++++++++-------- 4 files changed, 83 insertions(+), 41 deletions(-) diff --git a/api/funkwhale_api/federation/authentication.py b/api/funkwhale_api/federation/authentication.py index 7972e78a0..980b7006b 100644 --- a/api/funkwhale_api/federation/authentication.py +++ b/api/funkwhale_api/federation/authentication.py @@ -9,15 +9,17 @@ from . import actors from . import keys from . import serializers from . import signing +from . import utils class SignatureAuthentication(authentication.BaseAuthentication): - def authenticate(self, request): + def authenticate_actor(self, request): + headers = utils.clean_wsgi_headers(request.META) try: - signature = request.META['headers']['Signature'] + signature = headers['Signature'] key_id = keys.get_key_id_from_signature_header(signature) except KeyError: - raise exceptions.AuthenticationFailed('No signature') + return except ValueError as e: raise exceptions.AuthenticationFailed(str(e)) @@ -33,14 +35,17 @@ class SignatureAuthentication(authentication.BaseAuthentication): serializer = serializers.ActorSerializer(data=actor_data) if not serializer.is_valid(): - raise exceptions.AuthenticationFailed('Invalid actor payload') + raise exceptions.AuthenticationFailed('Invalid actor payload: {}'.format(serializer.errors)) try: signing.verify_django(request, public_key.encode('utf-8')) except cryptography.exceptions.InvalidSignature: raise exceptions.AuthenticationFailed('Invalid signature') + return serializer.build() + + def authenticate(self, request): + actor = self.authenticate_actor(request) user = AnonymousUser() - ac = serializer.build() - setattr(request, 'actor', ac) + setattr(request, 'actor', actor) return (user, None) diff --git a/api/funkwhale_api/federation/signing.py b/api/funkwhale_api/federation/signing.py index 6f77cbd42..7e4d2aa5a 100644 --- a/api/funkwhale_api/federation/signing.py +++ b/api/funkwhale_api/federation/signing.py @@ -1,6 +1,12 @@ +import logging import requests import requests_http_signature +from . import exceptions +from . import utils + +logger = logging.getLogger(__name__) + def verify(request, public_key): return requests_http_signature.HTTPSignatureAuth.verify( @@ -15,21 +21,35 @@ def verify_django(django_request, public_key): Given a django WSGI request, create an underlying requests.PreparedRequest instance we can verify """ - headers = django_request.META.get('headers', {}).copy() + headers = utils.clean_wsgi_headers(django_request.META) for h, v in list(headers.items()): # we include lower-cased version of the headers for compatibility # with requests_http_signature headers[h.lower()] = v try: - signature = headers['signature'] + signature = headers['Signature'] except KeyError: raise exceptions.MissingSignature - + url = 'http://noop{}'.format(django_request.path) + query = django_request.META['QUERY_STRING'] + if query: + url += '?{}'.format(query) + signature_headers = signature.split('headers="')[1].split('",')[0] + expected = signature_headers.split(' ') + logger.debug('Signature expected headers: %s', expected) + for header in expected: + try: + headers[header] + except KeyError: + logger.debug('Missing header: %s', header) request = requests.Request( method=django_request.method, - url='http://noop', + url=url, data=django_request.body, headers=headers) - + for h in request.headers.keys(): + v = request.headers[h] + if v: + request.headers[h] = str(v) prepared_request = request.prepare() return verify(request, public_key) diff --git a/api/tests/federation/test_authentication.py b/api/tests/federation/test_authentication.py index 0c4a25a7f..1837b3950 100644 --- a/api/tests/federation/test_authentication.py +++ b/api/tests/federation/test_authentication.py @@ -20,14 +20,17 @@ def test_authenticate(nodb_factories, mocker, api_request): }) signed_request = nodb_factories['federation.SignedRequest']( auth__key=private, - auth__key_id=actor_url + '#main-key' + auth__key_id=actor_url + '#main-key', + auth__headers=[ + 'date', + ] ) prepared = signed_request.prepare() django_request = api_request.get( '/', - headers={ - 'Date': prepared.headers['date'], - 'Signature': prepared.headers['signature'], + **{ + 'HTTP_DATE': prepared.headers['date'], + 'HTTP_SIGNATURE': prepared.headers['signature'], } ) authenticator = authentication.SignatureAuthentication() diff --git a/api/tests/federation/test_signing.py b/api/tests/federation/test_signing.py index 9da7a0f87..0c1ec2e0b 100644 --- a/api/tests/federation/test_signing.py +++ b/api/tests/federation/test_signing.py @@ -44,56 +44,67 @@ def test_verify_fails_with_wrong_key(nodb_factories): signing.verify(prepared_request, wrong_public) -def test_can_verify_django_request(factories, api_request): +def test_can_verify_django_request(factories, fake_request): private_key, public_key = keys.get_key_pair() signed_request = factories['federation.SignedRequest']( - auth__key=private_key + auth__key=private_key, + auth__headers=[ + 'date', + ] ) prepared = signed_request.prepare() - django_request = api_request.get( + django_request = fake_request.get( '/', - headers={ - 'Date': prepared.headers['date'], - 'Signature': prepared.headers['signature'], + **{ + 'HTTP_DATE': prepared.headers['date'], + 'HTTP_SIGNATURE': prepared.headers['signature'], } ) assert signing.verify_django(django_request, public_key) is None -def test_can_verify_django_request_digest(factories, api_request): +def test_can_verify_django_request_digest(factories, fake_request): private_key, public_key = keys.get_key_pair() signed_request = factories['federation.SignedRequest']( auth__key=private_key, method='post', - data=b'hello=world' + data=b'hello=world', + auth__headers=[ + 'date', + 'digest', + ] ) prepared = signed_request.prepare() - django_request = api_request.post( + django_request = fake_request.post( '/', - headers={ - 'Date': prepared.headers['date'], - 'Digest': prepared.headers['digest'], - 'Signature': prepared.headers['signature'], + **{ + 'HTTP_DATE': prepared.headers['date'], + 'HTTP_DIGEST': prepared.headers['digest'], + 'HTTP_SIGNATURE': prepared.headers['signature'], } ) assert signing.verify_django(django_request, public_key) is None -def test_can_verify_django_request_digest_failure(factories, api_request): +def test_can_verify_django_request_digest_failure(factories, fake_request): private_key, public_key = keys.get_key_pair() signed_request = factories['federation.SignedRequest']( auth__key=private_key, method='post', - data=b'hello=world' + data=b'hello=world', + auth__headers=[ + 'date', + 'digest', + ] ) prepared = signed_request.prepare() - django_request = api_request.post( + django_request = fake_request.post( '/', - headers={ - 'Date': prepared.headers['date'], - 'Digest': prepared.headers['digest'] + 'noop', - 'Signature': prepared.headers['signature'], + **{ + 'HTTP_DATE': prepared.headers['date'], + 'HTTP_DIGEST': prepared.headers['digest'] + 'noop', + 'HTTP_SIGNATURE': prepared.headers['signature'], } ) @@ -101,17 +112,20 @@ def test_can_verify_django_request_digest_failure(factories, api_request): signing.verify_django(django_request, public_key) -def test_can_verify_django_request_failure(factories, api_request): +def test_can_verify_django_request_failure(factories, fake_request): private_key, public_key = keys.get_key_pair() signed_request = factories['federation.SignedRequest']( - auth__key=private_key + auth__key=private_key, + auth__headers=[ + 'date', + ] ) prepared = signed_request.prepare() - django_request = api_request.get( + django_request = fake_request.get( '/', - headers={ - 'Date': 'Wrong', - 'Signature': prepared.headers['signature'], + **{ + 'HTTP_DATE': 'Wrong', + 'HTTP_SIGNATURE': prepared.headers['signature'], } ) with pytest.raises(cryptography.exceptions.InvalidSignature): From f526f0c1fefadd2251b3972b702db0d0a6127269 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 18:41:03 +0200 Subject: [PATCH 14/24] More flexible auth signature factory --- api/funkwhale_api/federation/factories.py | 32 ++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index 3cfecfa96..e290eee63 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -16,7 +16,12 @@ class SignatureAuthFactory(factory.Factory): key = factory.LazyFunction(lambda: keys.get_key_pair()[0]) key_id = factory.Faker('url') use_auth_header = False - + headers = [ + '(request-target)', + 'user-agent', + 'host', + 'date', + 'content-type',] class Meta: model = requests_http_signature.HTTPSignatureAuth @@ -29,3 +34,28 @@ class SignedRequestFactory(factory.Factory): class Meta: model = requests.Request + + @factory.post_generation + def headers(self, create, extracted, **kwargs): + default_headers = { + 'User-Agent': 'Test', + 'Host': 'test.host', + 'Date': 'Right now', + 'Content-Type': 'application/activity+json' + } + if extracted: + default_headers.update(extracted) + self.headers.update(default_headers) + + +# @registry.register +# class ActorFactory(factory.DjangoModelFactory): +# url = factory.Faker('url') +# inbox_url = factory.Faker('url') +# outbox_url = factory.Faker('url') +# public_key = factory.LazyFunction(lambda: keys.get_key_pair()[1]) +# preferred_username = factory.Faker('username') +# summary = factory.Faker('paragraph') +# +# class Meta: +# model = models.Actor From ee0341ba1ae46a68ab7efcd86c76bfa874c6f89d Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 18:41:15 +0200 Subject: [PATCH 15/24] Ensure we truncate summary --- api/funkwhale_api/federation/serializers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 5f5516b2d..6b12d51ca 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -22,6 +22,7 @@ class ActorSerializer(serializers.ModelSerializer): publicKey = serializers.JSONField(source='public_key', required=False) manuallyApprovesFollowers = serializers.NullBooleanField( source='manually_approves_followers', required=False) + summary = serializers.CharField(max_length=None, required=False) class Meta: model = models.Actor @@ -80,6 +81,11 @@ class ActorSerializer(serializers.ModelSerializer): kwargs.update(self.prepare_missing_fields()) return super().save(**kwargs) + def validate_summary(self, value): + if value: + return value[:500] + + class ActorWebfingerSerializer(serializers.ModelSerializer): class Meta: model = models.Actor From 741ab533b1acef1af02ee6ba8e9a26e1f9c8c346 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 18:41:35 +0200 Subject: [PATCH 16/24] Added proper header when querying activity pub actor --- api/funkwhale_api/federation/actors.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 823d163f9..56a3fc1fa 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -10,10 +10,18 @@ from . import utils def get_actor_data(actor_url): - response = requests.get(actor_url) + response = requests.get( + actor_url, + headers={ + 'Accept': 'application/activity+json', + } + ) response.raise_for_status() - return response.json() - + try: + return response.json() + except: + raise ValueError( + 'Invalid actor payload: {}'.format(response.text)) SYSTEM_ACTORS = { 'library': { From 3650c3699b3ac967bcc895cce14602a864308923 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 31 Mar 2018 18:41:53 +0200 Subject: [PATCH 17/24] Minor tweaks (logs, exceptions) --- api/config/settings/local.py | 6 +++++- api/funkwhale_api/federation/exceptions.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/api/config/settings/local.py b/api/config/settings/local.py index 24ad871f7..dcbea66d2 100644 --- a/api/config/settings/local.py +++ b/api/config/settings/local.py @@ -72,6 +72,10 @@ LOGGING = { 'handlers':['console'], 'propagate': True, 'level':'DEBUG', - } + }, + '': { + 'level': 'DEBUG', + 'handlers': ['console'], + }, }, } diff --git a/api/funkwhale_api/federation/exceptions.py b/api/funkwhale_api/federation/exceptions.py index 96fd24a7e..31d864b36 100644 --- a/api/funkwhale_api/federation/exceptions.py +++ b/api/funkwhale_api/federation/exceptions.py @@ -2,3 +2,7 @@ class MalformedPayload(ValueError): pass + + +class MissingSignature(KeyError): + pass From 6fbf8fa44cf2d9c4ccf2bfebce72786669031f36 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 1 Apr 2018 10:17:06 +0200 Subject: [PATCH 18/24] Actor factory and fixture --- api/funkwhale_api/federation/factories.py | 32 +++++++++++++++-------- api/tests/federation/conftest.py | 10 +++++++ api/tests/federation/test_views.py | 26 +++++++++--------- 3 files changed, 44 insertions(+), 24 deletions(-) create mode 100644 api/tests/federation/conftest.py diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index e290eee63..d34242fc8 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -5,6 +5,7 @@ import requests_http_signature from funkwhale_api.factories import registry from . import keys +from . import models registry.register(keys.get_key_pair, name='federation.KeyPair') @@ -48,14 +49,23 @@ class SignedRequestFactory(factory.Factory): self.headers.update(default_headers) -# @registry.register -# class ActorFactory(factory.DjangoModelFactory): -# url = factory.Faker('url') -# inbox_url = factory.Faker('url') -# outbox_url = factory.Faker('url') -# public_key = factory.LazyFunction(lambda: keys.get_key_pair()[1]) -# preferred_username = factory.Faker('username') -# summary = factory.Faker('paragraph') -# -# class Meta: -# model = models.Actor +@registry.register +class ActorFactory(factory.DjangoModelFactory): + url = factory.Faker('url') + inbox_url = factory.Faker('url') + outbox_url = factory.Faker('url') + public_key = None + private_key = None + preferred_username = factory.Faker('user_name') + summary = factory.Faker('paragraph') + + class Meta: + model = models.Actor + + @classmethod + def _generate(cls, create, attrs): + has_public = attrs.get('public_key') is None + has_private = attrs.get('private_key') is None + if not has_public and not has_private: + attrs['private_key'], attrs['public'] = keys.get_key_pair() + return super()._generate(create, attrs) diff --git a/api/tests/federation/conftest.py b/api/tests/federation/conftest.py new file mode 100644 index 000000000..c5831914b --- /dev/null +++ b/api/tests/federation/conftest.py @@ -0,0 +1,10 @@ +import pytest + + +@pytest.fixture +def authenticated_actor(nodb_factories, mocker): + actor = nodb_factories['federation.Actor']() + mocker.patch( + 'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor', + return_value=actor) + yield actor diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index 96cf8ff7f..5ec53279a 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -21,19 +21,19 @@ def test_instance_actors(system_actor, db, settings, api_client): assert response.data == serializer.data -# @pytest.mark.parametrize('route', [ -# 'instance-outbox', -# 'instance-inbox', -# 'instance-actor', -# 'well-known-webfinger', -# ]) -# def test_instance_inbox_405_if_federation_disabled( -# db, settings, api_client, route): -# settings.FEDERATION_ENABLED = False -# url = reverse('federation:{}'.format(route)) -# response = api_client.get(url) -# -# assert response.status_code == 405 +@pytest.mark.parametrize('route,kwargs', [ + ('instance-actors-outbox', {'actor': 'library'}), + ('instance-actors-inbox', {'actor': 'library'}), + ('instance-actors-detail', {'actor': 'library'}), + ('well-known-webfinger', {}), +]) +def test_instance_inbox_405_if_federation_disabled( + authenticated_actor, db, settings, api_client, route, kwargs): + settings.FEDERATION_ENABLED = False + url = reverse('federation:{}'.format(route), kwargs=kwargs) + response = api_client.get(url) + + assert response.status_code == 405 def test_wellknown_webfinger_validates_resource( From 3cf1a1708709e8006fa2315a9c78093c521c7976 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 1 Apr 2018 22:11:46 +0200 Subject: [PATCH 19/24] We are now able to receive a toot and react to it --- api/config/settings/common.py | 7 +- api/funkwhale_api/federation/activity.py | 51 ++++++ api/funkwhale_api/federation/actors.py | 170 ++++++++++++++---- .../federation/authentication.py | 1 + api/funkwhale_api/federation/parsers.py | 5 + api/funkwhale_api/federation/serializers.py | 68 +++++++ api/funkwhale_api/federation/views.py | 31 +++- api/funkwhale_api/federation/webfinger.py | 3 +- api/tests/federation/test_activity.py | 0 api/tests/federation/test_actors.py | 102 ++++++++++- api/tests/federation/test_views.py | 6 +- api/tests/federation/test_webfinger.py | 2 +- 12 files changed, 398 insertions(+), 48 deletions(-) create mode 100644 api/funkwhale_api/federation/activity.py create mode 100644 api/funkwhale_api/federation/parsers.py create mode 100644 api/tests/federation/test_activity.py diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 01fb19e6c..fbe3b7045 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -344,7 +344,12 @@ REST_FRAMEWORK = { ), 'DEFAULT_PAGINATION_CLASS': 'funkwhale_api.common.pagination.FunkwhalePagination', 'PAGE_SIZE': 25, - + 'DEFAULT_PARSER_CLASSES': ( + 'rest_framework.parsers.JSONParser', + 'rest_framework.parsers.FormParser', + 'rest_framework.parsers.MultiPartParser', + 'funkwhale_api.federation.parsers.ActivityParser', + ), 'DEFAULT_AUTHENTICATION_CLASSES': ( 'funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS', 'rest_framework_jwt.authentication.JSONWebTokenAuthentication', diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py new file mode 100644 index 000000000..00140fe53 --- /dev/null +++ b/api/funkwhale_api/federation/activity.py @@ -0,0 +1,51 @@ + + +ACTIVITY_TYPES = [ + 'Accept', + 'Add', + 'Announce', + 'Arrive', + 'Block', + 'Create', + 'Delete', + 'Dislike', + 'Flag', + 'Follow', + 'Ignore', + 'Invite', + 'Join', + 'Leave', + 'Like', + 'Listen', + 'Move', + 'Offer', + 'Question', + 'Reject', + 'Read', + 'Remove', + 'TentativeReject', + 'TentativeAccept', + 'Travel', + 'Undo', + 'Update', + 'View', +] + + +OBJECT_TYPES = [ + 'Article', + 'Audio', + 'Document', + 'Event', + 'Image', + 'Note', + 'Page', + 'Place', + 'Profile', + 'Relationship', + 'Tombstone', + 'Video', +] + +def deliver(content, on_behalf_of, to=[]): + pass diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 56a3fc1fa..1832b08bc 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -1,14 +1,23 @@ import requests +import xml from django.urls import reverse from django.conf import settings +from rest_framework.exceptions import PermissionDenied + from dynamic_preferences.registries import global_preferences_registry +from . import activity from . import models +from . import serializers from . import utils +def remove_tags(text): + return ''.join(xml.etree.ElementTree.fromstring(text).itertext()) + + def get_actor_data(actor_url): response = requests.get( actor_url, @@ -23,39 +32,132 @@ def get_actor_data(actor_url): raise ValueError( 'Invalid actor payload: {}'.format(response.text)) + +class SystemActor(object): + additional_attributes = {} + + def get_actor_instance(self): + a = models.Actor( + **self.get_instance_argument( + self.id, + name=self.name, + summary=self.summary, + **self.additional_attributes + ) + ) + a.pk = self.id + return a + + def get_instance_argument(self, id, name, summary, **kwargs): + preferences = global_preferences_registry.manager() + p = { + 'preferred_username': id, + 'domain': settings.FEDERATION_HOSTNAME, + 'type': 'Person', + 'name': name.format(host=settings.FEDERATION_HOSTNAME), + 'manually_approves_followers': True, + 'url': utils.full_url( + reverse( + 'federation:instance-actors-detail', + kwargs={'actor': id})), + 'shared_inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': id})), + 'inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': id})), + 'outbox_url': utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': id})), + 'public_key': preferences['federation__public_key'], + 'summary': summary.format(host=settings.FEDERATION_HOSTNAME) + } + p.update(kwargs) + return p + + def get_inbox(self, data, actor=None): + raise NotImplementedError + + def post_inbox(self, data, actor=None): + raise NotImplementedError + + def get_outbox(self, data, actor=None): + raise NotImplementedError + + def post_outbox(self, data, actor=None): + raise NotImplementedError + + +class LibraryActor(SystemActor): + id = 'library' + name = '{host}\'s library' + summary = 'Bot account to federate with {host}\'s library' + additional_attributes = { + 'manually_approves_followers': True + } + + +class TestActor(SystemActor): + id = 'test' + name = '{host}\'s test account' + summary = ( + 'Bot account to test federation with {host}. ' + 'Send me /ping and I\'ll answer you.' + ) + additional_attributes = { + 'manually_approves_followers': False + } + + def get_outbox(self, data, actor=None): + return { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + "id": utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': self.id})), + "type": "OrderedCollection", + "totalItems": 0, + "orderedItems": [] + } + + def post_inbox(self, data, actor=None): + if actor is None: + raise PermissionDenied('Actor not authenticated') + + serializer = serializers.ActivitySerializer( + data=data, context={'actor': actor}) + serializer.is_valid(raise_exception=True) + + ac = serializer.validated_data + if ac['type'] == 'Create' and ac['object']['type'] == 'Note': + # we received a toot \o/ + command = self.parse_command(ac['object']['content']) + if command == 'ping': + activity.deliver( + content='Pong!', + to=[ac['actor']], + on_behalf_of=self.get_actor_instance()) + + def parse_command(self, message): + """ + Remove any links or fancy markup to extract /command from + a note message. + """ + raw = remove_tags(message) + try: + return raw.split('/')[1] + except IndexError: + return + + SYSTEM_ACTORS = { - 'library': { - 'get_actor': lambda: models.Actor(**get_base_system_actor_arguments('library')), - } + 'library': LibraryActor(), + 'test': TestActor(), } - - -def get_base_system_actor_arguments(name): - preferences = global_preferences_registry.manager() - return { - 'preferred_username': name, - 'domain': settings.FEDERATION_HOSTNAME, - 'type': 'Person', - 'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME), - 'manually_approves_followers': True, - 'url': utils.full_url( - reverse( - 'federation:instance-actors-detail', - kwargs={'actor': name})), - 'shared_inbox_url': utils.full_url( - reverse( - 'federation:instance-actors-inbox', - kwargs={'actor': name})), - 'inbox_url': utils.full_url( - reverse( - 'federation:instance-actors-inbox', - kwargs={'actor': name})), - 'outbox_url': utils.full_url( - reverse( - 'federation:instance-actors-outbox', - kwargs={'actor': name})), - 'public_key': preferences['federation__public_key'], - 'summary': 'Bot account to federate with {}\'s library'.format( - settings.FEDERATION_HOSTNAME - ), - } diff --git a/api/funkwhale_api/federation/authentication.py b/api/funkwhale_api/federation/authentication.py index 980b7006b..e199ef134 100644 --- a/api/funkwhale_api/federation/authentication.py +++ b/api/funkwhale_api/federation/authentication.py @@ -45,6 +45,7 @@ class SignatureAuthentication(authentication.BaseAuthentication): return serializer.build() def authenticate(self, request): + setattr(request, 'actor', None) actor = self.authenticate_actor(request) user = AnonymousUser() setattr(request, 'actor', actor) diff --git a/api/funkwhale_api/federation/parsers.py b/api/funkwhale_api/federation/parsers.py new file mode 100644 index 000000000..874d808f9 --- /dev/null +++ b/api/funkwhale_api/federation/parsers.py @@ -0,0 +1,5 @@ +from rest_framework import parsers + + +class ActivityParser(parsers.JSONParser): + media_type = 'application/activity+json' diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 6b12d51ca..2137e8d91 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -6,6 +6,7 @@ from django.conf import settings from rest_framework import serializers from dynamic_preferences.registries import global_preferences_registry +from . import activity from . import models from . import utils @@ -105,3 +106,70 @@ class ActorWebfingerSerializer(serializers.ModelSerializer): instance.url ] return data + + +class ActivitySerializer(serializers.Serializer): + actor = serializers.URLField() + id = serializers.URLField() + type = serializers.ChoiceField( + choices=[(c, c) for c in activity.ACTIVITY_TYPES]) + object = serializers.JSONField() + + def validate_object(self, value): + try: + type = value['type'] + except KeyError: + raise serializers.ValidationError('Missing object type') + + try: + object_serializer = OBJECT_SERIALIZERS[type] + except KeyError: + raise serializers.ValidationError( + 'Unsupported type {}'.format(type)) + + serializer = object_serializer(data=value) + serializer.is_valid(raise_exception=True) + return serializer.data + + def validate_actor(self, value): + request_actor = self.context.get('actor') + if request_actor and request_actor.url != value: + raise serializers.ValidationError( + 'The actor making the request do not match' + ' the activity actor' + ) + return value + + +class ObjectSerializer(serializers.Serializer): + id = serializers.URLField() + url = serializers.URLField(required=False, allow_null=True) + type = serializers.ChoiceField( + choices=[(c, c) for c in activity.OBJECT_TYPES]) + content = serializers.CharField( + required=False, allow_null=True) + summary = serializers.CharField( + required=False, allow_null=True) + name = serializers.CharField( + required=False, allow_null=True) + published = serializers.DateTimeField( + required=False, allow_null=True) + updated = serializers.DateTimeField( + required=False, allow_null=True) + to = serializers.ListField( + child=serializers.URLField(), + required=False, allow_null=True) + cc = serializers.ListField( + child=serializers.URLField(), + required=False, allow_null=True) + bto = serializers.ListField( + child=serializers.URLField(), + required=False, allow_null=True) + bcc = serializers.ListField( + child=serializers.URLField(), + required=False, allow_null=True) + +OBJECT_SERIALIZERS = { + t: ObjectSerializer + for t in activity.OBJECT_TYPES +} diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 95e421b59..2e3feb8d0 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -36,18 +36,35 @@ class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet): raise Http404 def retrieve(self, request, *args, **kwargs): - actor_conf = self.get_object() - actor = actor_conf['get_actor']() + system_actor = self.get_object() + actor = system_actor.get_actor_instance() serializer = serializers.ActorSerializer(actor) return response.Response(serializer.data, status=200) - @detail_route(methods=['get']) + @detail_route(methods=['get', 'post']) def inbox(self, request, *args, **kwargs): - raise NotImplementedError() + system_actor = self.get_object() + handler = getattr(system_actor, '{}_inbox'.format( + request.method.lower() + )) - @detail_route(methods=['get']) + try: + data = handler(request.data, actor=request.actor) + except NotImplementedError: + return response.Response(status=405) + return response.Response(data, status=200) + + @detail_route(methods=['get', 'post']) def outbox(self, request, *args, **kwargs): - raise NotImplementedError() + system_actor = self.get_object() + handler = getattr(system_actor, '{}_outbox'.format( + request.method.lower() + )) + try: + data = handler(request.data, actor=request.actor) + except NotImplementedError: + return response.Response(status=405) + return response.Response(data, status=200) class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet): @@ -82,5 +99,5 @@ class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet): def handler_acct(self, clean_result): username, hostname = clean_result - actor = actors.SYSTEM_ACTORS[username]['get_actor']() + actor = actors.SYSTEM_ACTORS[username].get_actor_instance() return serializers.ActorWebfingerSerializer(actor).data diff --git a/api/funkwhale_api/federation/webfinger.py b/api/funkwhale_api/federation/webfinger.py index d698114f1..95a51e1c0 100644 --- a/api/funkwhale_api/federation/webfinger.py +++ b/api/funkwhale_api/federation/webfinger.py @@ -30,7 +30,8 @@ def clean_acct(acct_string): raise forms.ValidationError('Invalid format') if hostname != settings.FEDERATION_HOSTNAME: - raise forms.ValidationError('Invalid hostname') + raise forms.ValidationError( + 'Invalid hostname {}'.format(hostname)) if username not in actors.SYSTEM_ACTORS: raise forms.ValidationError('Invalid username') diff --git a/api/tests/federation/test_activity.py b/api/tests/federation/test_activity.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/tests/federation/test_actors.py b/api/tests/federation/test_actors.py index 00e214bd1..a239f4961 100644 --- a/api/tests/federation/test_actors.py +++ b/api/tests/federation/test_actors.py @@ -1,6 +1,10 @@ +import pytest + from django.urls import reverse +from rest_framework import exceptions from funkwhale_api.federation import actors +from funkwhale_api.federation import serializers from funkwhale_api.federation import utils @@ -37,10 +41,106 @@ def test_get_library(settings, preferences): reverse( 'federation:instance-actors-inbox', kwargs={'actor': 'library'})), + 'outbox_url': utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': 'library'})), 'public_key': 'public_key', 'summary': 'Bot account to federate with {}\'s library'.format( settings.FEDERATION_HOSTNAME), } - actor = actors.SYSTEM_ACTORS['library']['get_actor']() + actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() for key, value in expected.items(): assert getattr(actor, key) == value + + +def test_get_test(settings, preferences): + preferences['federation__public_key'] = 'public_key' + expected = { + 'preferred_username': 'test', + 'domain': settings.FEDERATION_HOSTNAME, + 'type': 'Person', + 'name': '{}\'s test account'.format(settings.FEDERATION_HOSTNAME), + 'manually_approves_followers': False, + 'url': utils.full_url( + reverse( + 'federation:instance-actors-detail', + kwargs={'actor': 'test'})), + 'shared_inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': 'test'})), + 'inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': 'test'})), + 'outbox_url': utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': 'test'})), + 'public_key': 'public_key', + 'summary': 'Bot account to test federation with {}. Send me /ping and I\'ll answer you.'.format( + settings.FEDERATION_HOSTNAME), + } + actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() + for key, value in expected.items(): + assert getattr(actor, key) == value + + +def test_test_get_outbox(): + expected = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + "id": utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': 'test'})), + "type": "OrderedCollection", + "totalItems": 0, + "orderedItems": [] + } + + data = actors.SYSTEM_ACTORS['test'].get_outbox({}, actor=None) + + assert data == expected + + +def test_test_post_inbox_requires_authenticated_actor(): + with pytest.raises(exceptions.PermissionDenied): + actors.SYSTEM_ACTORS['test'].post_inbox({}, actor=None) + + +def test_test_post_outbox_validates_actor(nodb_factories): + actor = nodb_factories['federation.Actor']() + data = { + 'actor': 'noop' + } + with pytest.raises(exceptions.ValidationError) as exc_info: + actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor) + msg = 'The actor making the request do not match' + assert msg in exc_info.value + + +def test_test_post_outbox_handles_create_note(mocker, factories): + deliver = mocker.patch( + 'funkwhale_api.federation.activity.deliver') + actor = factories['federation.Actor']() + data = { + 'actor': actor.url, + 'type': 'Create', + 'id': 'http://test.federation/activity', + 'object': { + 'type': 'Note', + 'id': 'http://test.federation/object', + 'content': '

@mention /ping

' + } + } + actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor) + deliver.assert_called_once_with( + content='Pong!', + to=[actor.url], + on_behalf_of=actors.SYSTEM_ACTORS['test'].get_actor_instance() + ) diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index 5ec53279a..0d2ac882f 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -10,7 +10,7 @@ from funkwhale_api.federation import webfinger @pytest.mark.parametrize('system_actor', actors.SYSTEM_ACTORS.keys()) def test_instance_actors(system_actor, db, settings, api_client): - actor = actors.SYSTEM_ACTORS[system_actor]['get_actor']() + actor = actors.SYSTEM_ACTORS[system_actor].get_actor_instance() url = reverse( 'federation:instance-actors-detail', kwargs={'actor': system_actor}) @@ -27,7 +27,7 @@ def test_instance_actors(system_actor, db, settings, api_client): ('instance-actors-detail', {'actor': 'library'}), ('well-known-webfinger', {}), ]) -def test_instance_inbox_405_if_federation_disabled( +def test_instance_endpoints_405_if_federation_disabled( authenticated_actor, db, settings, api_client, route, kwargs): settings.FEDERATION_ENABLED = False url = reverse('federation:{}'.format(route), kwargs=kwargs) @@ -53,7 +53,7 @@ def test_wellknown_webfinger_validates_resource( @pytest.mark.parametrize('system_actor', actors.SYSTEM_ACTORS.keys()) def test_wellknown_webfinger_system( system_actor, db, api_client, settings, mocker): - actor = actors.SYSTEM_ACTORS[system_actor]['get_actor']() + actor = actors.SYSTEM_ACTORS[system_actor].get_actor_instance() url = reverse('federation:well-known-webfinger') response = api_client.get( url, data={'resource': 'acct:{}'.format(actor.webfinger_subject)}) diff --git a/api/tests/federation/test_webfinger.py b/api/tests/federation/test_webfinger.py index fd1cb1d05..96258455a 100644 --- a/api/tests/federation/test_webfinger.py +++ b/api/tests/federation/test_webfinger.py @@ -32,7 +32,7 @@ def test_webfinger_clean_acct(settings): @pytest.mark.parametrize('resource,message', [ ('service', 'Invalid format'), - ('service@test.com', 'Invalid hostname'), + ('service@test.com', 'Invalid hostname test.com'), ('noop@test.federation', 'Invalid account'), ]) def test_webfinger_clean_acct_errors(resource, message, settings): From a2520513512055140cd0d8102da617361010a989 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Mon, 2 Apr 2018 18:07:46 +0200 Subject: [PATCH 20/24] PoC with receiving /ping from Mastodon and replying pong --- api/funkwhale_api/federation/activity.py | 38 +++++++++- api/funkwhale_api/federation/actors.py | 86 +++++++++++++++++++++-- api/funkwhale_api/federation/factories.py | 24 ++++++- api/funkwhale_api/federation/models.py | 4 ++ api/tests/federation/test_activity.py | 32 +++++++++ api/tests/federation/test_actors.py | 24 ++++++- 6 files changed, 196 insertions(+), 12 deletions(-) diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index 00140fe53..4eeb193b1 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -1,4 +1,11 @@ +import logging +import json +import requests +import requests_http_signature +from . import signing + +logger = logging.getLogger(__name__) ACTIVITY_TYPES = [ 'Accept', @@ -47,5 +54,32 @@ OBJECT_TYPES = [ 'Video', ] -def deliver(content, on_behalf_of, to=[]): - pass +def deliver(activity, on_behalf_of, to=[]): + from . import actors + logger.info('Preparing activity delivery to %s', to) + auth = requests_http_signature.HTTPSignatureAuth( + use_auth_header=False, + headers=[ + '(request-target)', + 'user-agent', + 'host', + 'date', + 'content-type',], + algorithm='rsa-sha256', + key=on_behalf_of.private_key.encode('utf-8'), + key_id=on_behalf_of.private_key_id, + ) + for url in to: + recipient_actor = actors.get_actor(url) + logger.debug('delivering to %s', recipient_actor.inbox_url) + logger.debug('activity content: %s', json.dumps(activity)) + response = requests.post( + auth=auth, + json=activity, + url=recipient_actor.inbox_url, + headers={ + 'Content-Type': 'application/activity+json' + } + ) + response.raise_for_status() + logger.debug('Remote answered with %s', response.status_code) diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 1832b08bc..7a9b47d18 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -1,21 +1,27 @@ +import logging import requests import xml -from django.urls import reverse from django.conf import settings +from django.urls import reverse +from django.utils import timezone from rest_framework.exceptions import PermissionDenied from dynamic_preferences.registries import global_preferences_registry from . import activity +from . import factories from . import models from . import serializers from . import utils +logger = logging.getLogger(__name__) + def remove_tags(text): - return ''.join(xml.etree.ElementTree.fromstring(text).itertext()) + logger.debug('Removing tags from %s', text) + return ''.join(xml.etree.ElementTree.fromstring('
{}
'.format(text)).itertext()) def get_actor_data(actor_url): @@ -32,6 +38,13 @@ def get_actor_data(actor_url): raise ValueError( 'Invalid actor payload: {}'.format(response.text)) +def get_actor(actor_url): + data = get_actor_data(actor_url) + serializer = serializers.ActorSerializer(data=data) + serializer.is_valid(raise_exception=True) + + return serializer.build() + class SystemActor(object): additional_attributes = {} @@ -73,6 +86,7 @@ class SystemActor(object): 'federation:instance-actors-outbox', kwargs={'actor': id})), 'public_key': preferences['federation__public_key'], + 'private_key': preferences['federation__private_key'], 'summary': summary.format(host=settings.FEDERATION_HOSTNAME) } p.update(kwargs) @@ -136,14 +150,13 @@ class TestActor(SystemActor): serializer.is_valid(raise_exception=True) ac = serializer.validated_data + logger.info('Received activity on %s inbox', self.id) if ac['type'] == 'Create' and ac['object']['type'] == 'Note': # we received a toot \o/ command = self.parse_command(ac['object']['content']) + logger.debug('Parsed command: %s', command) if command == 'ping': - activity.deliver( - content='Pong!', - to=[ac['actor']], - on_behalf_of=self.get_actor_instance()) + self.handle_ping(ac, actor) def parse_command(self, message): """ @@ -156,6 +169,67 @@ class TestActor(SystemActor): except IndexError: return + def handle_ping(self, ac, sender): + now = timezone.now() + test_actor = self.get_actor_instance() + reply_url = 'https://{}/activities/note/{}'.format( + settings.FEDERATION_HOSTNAME, now.timestamp() + ) + mention = '@{}@{}'.format( + sender.preferred_username, + sender.domain + ) + reply_content = '{} Pong!'.format( + mention + ) + reply_activity = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "sensitive": "as:sensitive", + "movedTo": "as:movedTo", + "Hashtag": "as:Hashtag", + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "toot": "http://joinmastodon.org/ns#", + "Emoji": "toot:Emoji" + } + ], + 'type': 'Create', + 'actor': test_actor.url, + 'id': '{}/activity'.format(reply_url), + 'published': now.isoformat(), + 'to': ac['actor'], + 'cc': [], + 'object': factories.NoteFactory( + content='Pong!', + summary=None, + published=now.isoformat(), + id=reply_url, + inReplyTo=ac['object']['id'], + sensitive=False, + url=reply_url, + to=[ac['actor']], + attributedTo=test_actor.url, + cc=[], + attachment=[], + tag=[ + { + "type": "Mention", + "href": ac['actor'], + "name": mention + } + ] + ) + } + activity.deliver( + reply_activity, + to=[ac['actor']], + on_behalf_of=test_actor) SYSTEM_ACTORS = { 'library': LibraryActor(), diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index d34242fc8..ebd6b2fd5 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -2,6 +2,8 @@ import factory import requests import requests_http_signature +from django.utils import timezone + from funkwhale_api.factories import registry from . import keys @@ -64,8 +66,24 @@ class ActorFactory(factory.DjangoModelFactory): @classmethod def _generate(cls, create, attrs): - has_public = attrs.get('public_key') is None - has_private = attrs.get('private_key') is None + has_public = attrs.get('public_key') is not None + has_private = attrs.get('private_key') is not None if not has_public and not has_private: - attrs['private_key'], attrs['public'] = keys.get_key_pair() + private, public = keys.get_key_pair() + attrs['private_key'] = private.decode('utf-8') + attrs['public_key'] = public.decode('utf-8') return super()._generate(create, attrs) + + +@registry.register(name='federation.Note') +class NoteFactory(factory.Factory): + type = 'Note' + id = factory.Faker('url') + published = factory.LazyFunction( + lambda: timezone.now().isoformat() + ) + inReplyTo = None + content = factory.Faker('sentence') + + class Meta: + model = dict diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index 201307d46..fa38678e9 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -38,3 +38,7 @@ class Actor(models.Model): self.preferred_username, settings.FEDERATION_HOSTNAME, ) + + @property + def private_key_id(self): + return '{}#main-key'.format(self.url) diff --git a/api/tests/federation/test_activity.py b/api/tests/federation/test_activity.py index e69de29bb..a6e1d28aa 100644 --- a/api/tests/federation/test_activity.py +++ b/api/tests/federation/test_activity.py @@ -0,0 +1,32 @@ +from funkwhale_api.federation import activity + +def test_deliver(nodb_factories, r_mock, mocker): + to = nodb_factories['federation.Actor']() + mocker.patch( + 'funkwhale_api.federation.actors.get_actor', + return_value=to) + sender = nodb_factories['federation.Actor']() + ac = { + 'id': 'http://test.federation/activity', + 'type': 'Create', + 'actor': sender.url, + 'object': { + 'id': 'http://test.federation/note', + 'type': 'Note', + 'content': 'Hello', + } + } + + r_mock.post(to.inbox_url) + + activity.deliver( + ac, + to=[to.url], + on_behalf_of=sender, + ) + request = r_mock.request_history[0] + + assert r_mock.called is True + assert r_mock.call_count == 1 + assert request.url == to.inbox_url + assert request.headers['content-type'] == 'application/activity+json' diff --git a/api/tests/federation/test_actors.py b/api/tests/federation/test_actors.py index a239f4961..88a94e562 100644 --- a/api/tests/federation/test_actors.py +++ b/api/tests/federation/test_actors.py @@ -1,6 +1,8 @@ import pytest from django.urls import reverse +from django.utils import timezone + from rest_framework import exceptions from funkwhale_api.federation import actors @@ -128,6 +130,8 @@ def test_test_post_outbox_handles_create_note(mocker, factories): deliver = mocker.patch( 'funkwhale_api.federation.activity.deliver') actor = factories['federation.Actor']() + now = timezone.now() + mocker.patch('django.utils.timezone.now', return_value=now) data = { 'actor': actor.url, 'type': 'Create', @@ -138,9 +142,27 @@ def test_test_post_outbox_handles_create_note(mocker, factories): 'content': '

@mention /ping

' } } + expected_note = factories['federation.Note']( + id='https://test.federation/activities/note/{}'.format( + now.timestamp() + ), + content='Pong!', + published=now.isoformat(), + inReplyTo=data['object']['id'], + ) + test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() + expected_activity = { + 'actor': test_actor.url, + 'id': 'https://test.federation/activities/note/{}/activity'.format( + now.timestamp() + ), + 'type': 'Create', + 'published': now.isoformat(), + 'object': expected_note + } actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor) deliver.assert_called_once_with( - content='Pong!', + expected_activity, to=[actor.url], on_behalf_of=actors.SYSTEM_ACTORS['test'].get_actor_instance() ) From 77c6bd583985eb61e5b7ba3e37e85a28a73a8d2d Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Mon, 2 Apr 2018 19:15:27 +0200 Subject: [PATCH 21/24] Fixed failing test --- api/funkwhale_api/federation/actors.py | 21 ++----------- api/funkwhale_api/federation/factories.py | 8 +++-- api/funkwhale_api/federation/models.py | 15 +++++++++ api/funkwhale_api/federation/webfinger.py | 2 +- api/tests/federation/test_actors.py | 38 ++++++++++++++++++----- 5 files changed, 54 insertions(+), 30 deletions(-) diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 7a9b47d18..a6220ed16 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -175,29 +175,14 @@ class TestActor(SystemActor): reply_url = 'https://{}/activities/note/{}'.format( settings.FEDERATION_HOSTNAME, now.timestamp() ) - mention = '@{}@{}'.format( - sender.preferred_username, - sender.domain - ) reply_content = '{} Pong!'.format( - mention + sender.mention_username ) reply_activity = { "@context": [ "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1", - { - "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", - "sensitive": "as:sensitive", - "movedTo": "as:movedTo", - "Hashtag": "as:Hashtag", - "ostatus": "http://ostatus.org#", - "atomUri": "ostatus:atomUri", - "inReplyToAtomUri": "ostatus:inReplyToAtomUri", - "conversation": "ostatus:conversation", - "toot": "http://joinmastodon.org/ns#", - "Emoji": "toot:Emoji" - } + {} ], 'type': 'Create', 'actor': test_actor.url, @@ -221,7 +206,7 @@ class TestActor(SystemActor): { "type": "Mention", "href": ac['actor'], - "name": mention + "name": sender.mention_username } ] ) diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index ebd6b2fd5..88c86f791 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -53,13 +53,15 @@ class SignedRequestFactory(factory.Factory): @registry.register class ActorFactory(factory.DjangoModelFactory): - url = factory.Faker('url') - inbox_url = factory.Faker('url') - outbox_url = factory.Faker('url') + public_key = None private_key = None preferred_username = factory.Faker('user_name') summary = factory.Faker('paragraph') + domain = factory.Faker('domain_name') + url = factory.LazyAttribute(lambda o: 'https://{}/users/{}'.format(o.domain, o.preferred_username)) + inbox_url = factory.LazyAttribute(lambda o: 'https://{}/users/{}/inbox'.format(o.domain, o.preferred_username)) + outbox_url = factory.LazyAttribute(lambda o: 'https://{}/users/{}/outbox'.format(o.domain, o.preferred_username)) class Meta: model = models.Actor diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index fa38678e9..d76ad173b 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -42,3 +42,18 @@ class Actor(models.Model): @property def private_key_id(self): return '{}#main-key'.format(self.url) + + @property + def mention_username(self): + return '@{}@{}'.format(self.preferred_username, self.domain) + + def save(self, **kwargs): + lowercase_fields = [ + 'domain', + ] + for field in lowercase_fields: + v = getattr(self, field, None) + if v: + setattr(self, field, v.lower()) + + super().save(**kwargs) diff --git a/api/funkwhale_api/federation/webfinger.py b/api/funkwhale_api/federation/webfinger.py index 95a51e1c0..4e9753385 100644 --- a/api/funkwhale_api/federation/webfinger.py +++ b/api/funkwhale_api/federation/webfinger.py @@ -29,7 +29,7 @@ def clean_acct(acct_string): except ValueError: raise forms.ValidationError('Invalid format') - if hostname != settings.FEDERATION_HOSTNAME: + if hostname.lower() != settings.FEDERATION_HOSTNAME: raise forms.ValidationError( 'Invalid hostname {}'.format(hostname)) diff --git a/api/tests/federation/test_actors.py b/api/tests/federation/test_actors.py index 88a94e562..b3b0f8df0 100644 --- a/api/tests/federation/test_actors.py +++ b/api/tests/federation/test_actors.py @@ -126,7 +126,8 @@ def test_test_post_outbox_validates_actor(nodb_factories): assert msg in exc_info.value -def test_test_post_outbox_handles_create_note(mocker, factories): +def test_test_post_outbox_handles_create_note( + settings, mocker, factories): deliver = mocker.patch( 'funkwhale_api.federation.activity.deliver') actor = factories['federation.Actor']() @@ -142,6 +143,7 @@ def test_test_post_outbox_handles_create_note(mocker, factories): 'content': '

@mention /ping

' } } + test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() expected_note = factories['federation.Note']( id='https://test.federation/activities/note/{}'.format( now.timestamp() @@ -149,16 +151,36 @@ def test_test_post_outbox_handles_create_note(mocker, factories): content='Pong!', published=now.isoformat(), inReplyTo=data['object']['id'], - ) - test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() - expected_activity = { - 'actor': test_actor.url, - 'id': 'https://test.federation/activities/note/{}/activity'.format( - now.timestamp() + cc=[], + summary=None, + sensitive=False, + attributedTo=test_actor.url, + attachment=[], + to=[actor.url], + url='https://{}/activities/note/{}'.format( + settings.FEDERATION_HOSTNAME, now.timestamp() ), + tag=[{ + 'href': actor.url, + 'name': actor.mention_username, + 'type': 'Mention', + }] + ) + expected_activity = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {} + ], + 'actor': test_actor.url, + 'id': 'https://{}/activities/note/{}/activity'.format( + settings.FEDERATION_HOSTNAME, now.timestamp() + ), + 'to': actor.url, 'type': 'Create', 'published': now.isoformat(), - 'object': expected_note + 'object': expected_note, + 'cc': [], } actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor) deliver.assert_called_once_with( From 4f50eb50fbb1249b0c18ac267f83332b78a5727c Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Mon, 2 Apr 2018 19:16:34 +0200 Subject: [PATCH 22/24] Fixed broken CI --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 01717831a..39aa3a794 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,7 @@ test_api: DATABASE_URL: "postgresql://postgres@postgres/postgres" FUNKWHALE_URL: "https://funkwhale.ci" CACHEOPS_ENABLED: "false" + DJANGO_SETTINGS_MODULE: config.settings.local before_script: - cd api From dcb3335ca454b6d4160ac8fe48676e0079306d83 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Mon, 2 Apr 2018 19:21:06 +0200 Subject: [PATCH 23/24] Changelog --- changes/changelog.d/test-bot.enhancement | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/changelog.d/test-bot.enhancement diff --git a/changes/changelog.d/test-bot.enhancement b/changes/changelog.d/test-bot.enhancement new file mode 100644 index 000000000..03100d7c8 --- /dev/null +++ b/changes/changelog.d/test-bot.enhancement @@ -0,0 +1 @@ +Implemented a @test@yourfunkwhaledomain bot to ensure federation works properly. Send it "/ping" and it will answer back :) From 76c1abe9d63b365bcff006e2e7db01fdc572193a Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Mon, 2 Apr 2018 19:24:01 +0200 Subject: [PATCH 24/24] Use redis in CI --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39aa3a794..94b40bed3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ stages: test_api: services: - postgres:9.4 + - redis:3 stage: test image: funkwhale/funkwhale:latest cache: