diff --git a/.env.dev b/.env.dev index 2e8834143..3f904078c 100644 --- a/.env.dev +++ b/.env.dev @@ -1,9 +1,11 @@ API_AUTHENTICATION_REQUIRED=True RAVEN_ENABLED=false RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5 -DJANGO_ALLOWED_HOSTS=localhost,nginx +DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx,0.0.0.0,127.0.0.1 DJANGO_SETTINGS_MODULE=config.settings.local DJANGO_SECRET_KEY=dev C_FORCE_ROOT=true -FUNKWHALE_URL=http://localhost +FUNKWHALE_HOSTNAME=localhost +FUNKWHALE_PROTOCOL=http PYTHONDONTWRITEBYTECODE=true +WEBPACK_DEVSERVER_PORT=8080 diff --git a/.gitignore b/.gitignore index 8b5117034..548cfd7b3 100644 --- a/.gitignore +++ b/.gitignore @@ -35,7 +35,6 @@ htmlcov # Translations *.mo -*.pot # Pycharm .idea @@ -75,6 +74,7 @@ api/static api/.pytest_cache # Front +front/static/translations front/node_modules/ front/dist/ front/npm-debug.log* @@ -87,3 +87,5 @@ docs/_build data/ .env + +po/*.po diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 94b40bed3..217047794 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,6 +68,8 @@ build_front: script: - yarn install + - yarn run i18n-extract + - yarn run i18n-compile - yarn run build cache: key: "$CI_PROJECT_ID__front_dependencies" diff --git a/CHANGELOG b/CHANGELOG index 0f01b5825..7cd7714af 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,79 @@ Changelog .. towncrier +0.9 (2018-04-17) +---------------- + +Features: + +- Add internationalization support (#5) +- Can now follow and import music from remote libraries (#136, #137) + + +Enhancements: + +- Added a i18n-extract yarn script to extract strings to PO files (#162) +- User admin now includes signup and last login dates (#148) +- We now use a proper user agent including instance version and url during + outgoing requests + + +Federation is here! +^^^^^^^^^^^^^^^^^^^ + +This is for real this time, and includes: + +- Following other Funkwhale libraries +- Importing tracks from remote libraries (tracks are hotlinked, and only cached for a short amount of time) +- Searching accross federated catalogs + +Note that by default, federation is opt-in, on a per-instance basis: +instances will request access to your catalog, and you can accept or refuse +those requests. You can also revoke the access at any time. + +Documentation was updated with relevant instructions to use and benefit +from this new feature: https://docs.funkwhale.audio/federation.html + +Preparing internationalization +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Funkwhale's front-end as always been english-only, and this is a barrier +to new users. The work make Funkwhale's interface translatable was started +in this release by Baptiste. Although nothing is translated yet, +this release includes behind the stage changes that will make it possible in +the near future. + +Many thank to Baptiste for the hard work and for figuring out a proper solution +to this difficult problem. + +Upgrade path +^^^^^^^^^^^^ + +In addition to the usual instructions from +https://docs.funkwhale.audio/upgrading.html, non-docker users will have +to setup an additional systemd unit file for recurrent tasks. + +This was forgotten in the deployment documentation, but recurrent tasks, +managed by the celery beat process, will be needed more and more in subsequent +releases. Right now, we'll be using to clear the cache for federated music files +and keep disk usage to a minimum. + +In the future, they will also be needed to refetch music metadata or federated +information periodically. + +Celery beat can be enabled easily:: + + curl -L -o "/etc/systemd/system/funkwhale-beat.service" "https://code.eliotberriot.com/funkwhale/funkwhale/raw/develop/deploy/funkwhale-beat.service" + # Also edit /etc/systemd/system/funkwhale.target + # and ensure the Wants= line contains the following: + # Wants=funkwhale-server.service funkwhale-worker.service funkwhale-beat.service + nano /etc/systemd/system/funkwhale.target + # reload configuration + systemctl daemon-reload + +Docker users already have celerybeat enabled. + + 0.8 (2018-04-02) ---------------- @@ -71,27 +144,16 @@ and add the following snippets:: This will ensure federation endpoints will be reachable in the future. You can of course skip this part if you know you will not federate your instance. -A new ``FEDERATION_ENABLED`` env var have also been added to control wether +A new ``FEDERATION_ENABLED`` env var have also been added to control whether federation is enabled or not on the application side. This settings defaults -to True, which should have no consequencies at the moment, since actual +to True, which should have no consequences at the moment, since actual federation is not implemented and the only available endpoints are for testing purposes. Add ``FEDERATION_ENABLED=false`` to your .env file to disable federation on the application side. -The last step involves generating RSA private and public keys for signing -your instance requests on the federation. This can be done via:: - - # on docker setups - docker-compose run --rm api python manage.py generate_keys --no-input - - # on non-docker setups - source /srv/funkwhale/virtualenv/bin/activate - source /srv/funkwhale/load_env - python manage.py generate_keys --no-input - -To test and troobleshoot federation, we've added a bot account. This bot is available at @test@yourinstancedomain, +To test and troubleshoot federation, we've added a bot account. This bot is available at @test@yourinstancedomain, and sending it "/ping", for example, via Mastodon, should trigger a response. diff --git a/README.rst b/README.rst index 2d5d2011d..8a0ea4932 100644 --- a/README.rst +++ b/README.rst @@ -206,3 +206,91 @@ Typical workflow for a merge request 6. Push your branch 7. Create your merge request 8. Take a step back and enjoy, we're really grateful you did all of this and took the time to contribute! + + +Internationalization +-------------------- + +When working on the front-end, any end-user string should be translated +using either ```` or the ``$t('yourstring')`` +function. + +Extraction is done by calling ``yarn run i18n-extract``, which +will pull all the strings from source files and put them in a PO file. + + +Working with federation locally +------------------------------- + +To achieve that, you'll need: + +1. to update your dns resolver to resolve all your .dev hostnames locally +2. a reverse proxy (such as traefik) to catch those .dev requests and + and with https certificate +3. two instances (or more) running locally, following the regular dev setup + +Resolve .dev names locally +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you use dnsmasq, this is as simple as doing:: + + echo "address=/test/172.17.0.1" | sudo tee /etc/dnsmasq.d/test.conf + sudo systemctl restart dnsmasq + +If you use NetworkManager with dnsmasq integration, use this instead:: + + echo "address=/test/172.17.0.1" | sudo tee /etc/NetworkManager/dnsmasq.d/test.conf + sudo systemctl restart NetworkManager + +Add wildcard certificate to the trusted certificates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Simply copy bundled certificates:: + + sudo cp docker/ssl/test.crt /usr/local/share/ca-certificates/ + sudo update-ca-certificates + +This certificate is a wildcard for ``*.funkwhale.test`` + +Run a reverse proxy for your instances +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + +Create docker network +^^^^^^^^^^^^^^^^^^^^ + +Create the federation network:: + + docker network create federation + +Launch everything +^^^^^^^^^^^^^^^^^ + +Launch the traefik proxy:: + + docker-compose -f docker/traefik.yml up -d + +Then, in separate terminals, you can setup as many different instances as you +need:: + + export COMPOSE_PROJECT_NAME=node2 + docker-compose -f dev.yml run --rm api python manage.py migrate + docker-compose -f dev.yml run --rm api python manage.py createsuperuser + docker-compose -f dev.yml up nginx api front nginx api celeryworker + +Note that by default, if you don't export the COMPOSE_PROJECT_NAME, +we will default to node1 as the name of your instance. + +Assuming your project name is ``node1``, your server will be reachable +at ``https://node1.funkwhale.test/``. Not that you'll have to trust +the SSL Certificate as it's self signed. + +When working on federation with traefik, ensure you have this in your ``env``:: + + # This will ensure we don't bind any port on the host, and thus enable + # multiple instances of funkwhale to be spawned concurrently. + WEBPACK_DEVSERVER_PORT_BINDING= + # This disable certificate verification + EXTERNAL_REQUESTS_VERIFY_SSL=false + # this ensure you don't have incorrect urls pointing to http resources + FUNKWHALE_PROTOCOL=https diff --git a/api/config/api_urls.py b/api/config/api_urls.py index cab6805b6..cf5b03744 100644 --- a/api/config/api_urls.py +++ b/api/config/api_urls.py @@ -32,6 +32,10 @@ v1_patterns += [ include( ('funkwhale_api.instance.urls', 'instance'), namespace='instance')), + url(r'^federation/', + include( + ('funkwhale_api.federation.api_urls', 'federation'), + namespace='federation')), url(r'^providers/', include( ('funkwhale_api.providers.urls', 'providers'), diff --git a/api/config/settings/common.py b/api/config/settings/common.py index fbe3b7045..a972ec7ef 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -13,6 +13,8 @@ from __future__ import absolute_import, unicode_literals from urllib.parse import urlsplit import os import environ +from celery.schedules import crontab + from funkwhale_api import __version__ ROOT_DIR = environ.Path(__file__) - 3 # (/a/b/myfile.py - 3 = /) @@ -25,12 +27,35 @@ try: except FileNotFoundError: pass -FUNKWHALE_URL = env('FUNKWHALE_URL') -FUNKWHALE_HOSTNAME = urlsplit(FUNKWHALE_URL).netloc +FUNKWHALE_HOSTNAME = None +FUNKWHALE_HOSTNAME_SUFFIX = env('FUNKWHALE_HOSTNAME_SUFFIX', default=None) +FUNKWHALE_HOSTNAME_PREFIX = env('FUNKWHALE_HOSTNAME_PREFIX', default=None) +if FUNKWHALE_HOSTNAME_PREFIX and FUNKWHALE_HOSTNAME_SUFFIX: + # We're in traefik case, in development + FUNKWHALE_HOSTNAME = '{}.{}'.format( + FUNKWHALE_HOSTNAME_PREFIX, FUNKWHALE_HOSTNAME_SUFFIX) + FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https') +else: + try: + FUNKWHALE_HOSTNAME = env('FUNKWHALE_HOSTNAME') + FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https') + except Exception: + FUNKWHALE_URL = env('FUNKWHALE_URL') + _parsed = urlsplit(FUNKWHALE_URL) + FUNKWHALE_HOSTNAME = _parsed.netloc + FUNKWHALE_PROTOCOL = _parsed.scheme + +FUNKWHALE_URL = '{}://{}'.format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME) + FEDERATION_ENABLED = env.bool('FEDERATION_ENABLED', default=True) FEDERATION_HOSTNAME = env('FEDERATION_HOSTNAME', default=FUNKWHALE_HOSTNAME) - +FEDERATION_COLLECTION_PAGE_SIZE = env.int( + 'FEDERATION_COLLECTION_PAGE_SIZE', default=50 +) +FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool( + 'FEDERATION_MUSIC_NEEDS_APPROVAL', default=True +) ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS') # APP CONFIGURATION @@ -144,16 +169,6 @@ FIXTURE_DIRS = ( # ------------------------------------------------------------------------------ EMAIL_BACKEND = env('DJANGO_EMAIL_BACKEND', default='django.core.mail.backends.smtp.EmailBackend') -# MANAGER CONFIGURATION -# ------------------------------------------------------------------------------ -# See: https://docs.djangoproject.com/en/dev/ref/settings/#admins -ADMINS = ( - ("""Eliot Berriot""", 'contact@eliotberriot.om'), -) - -# See: https://docs.djangoproject.com/en/dev/ref/settings/#managers -MANAGERS = ADMINS - # DATABASE CONFIGURATION # ------------------------------------------------------------------------------ # See: https://docs.djangoproject.com/en/dev/ref/settings/#databases @@ -321,6 +336,16 @@ CELERY_BROKER_URL = env( # Your common stuff: Below this line define 3rd party library settings CELERY_TASK_DEFAULT_RATE_LIMIT = 1 CELERY_TASK_TIME_LIMIT = 300 +CELERYBEAT_SCHEDULE = { + 'federation.clean_music_cache': { + 'task': 'funkwhale_api.federation.tasks.clean_music_cache', + 'schedule': crontab(hour='*/2'), + 'options': { + 'expires': 60 * 2, + }, + } +} + import datetime JWT_AUTH = { 'JWT_ALLOW_REFRESH': True, @@ -411,3 +436,8 @@ ACCOUNT_USERNAME_BLACKLIST = [ 'staff', 'service', ] + env.list('ACCOUNT_USERNAME_BLACKLIST', default=[]) + +EXTERNAL_REQUESTS_VERIFY_SSL = env.bool( + 'EXTERNAL_REQUESTS_VERIFY_SSL', + default=True +) diff --git a/api/funkwhale_api/__init__.py b/api/funkwhale_api/__init__.py index 0a12b794e..70cf5b1f8 100644 --- a/api/funkwhale_api/__init__.py +++ b/api/funkwhale_api/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__version__ = '0.7' +__version__ = '0.9' __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')]) diff --git a/api/funkwhale_api/common/fields.py b/api/funkwhale_api/common/fields.py index 1a18b5f27..98e971662 100644 --- a/api/funkwhale_api/common/fields.py +++ b/api/funkwhale_api/common/fields.py @@ -1,5 +1,9 @@ +import django_filters + from django.db import models +from funkwhale_api.music import utils + PRIVACY_LEVEL_CHOICES = [ ('me', 'Only me'), @@ -25,3 +29,15 @@ def privacy_level_query(user, lookup_field='privacy_level'): 'followers', 'instance', 'everyone' ] }) + + +class SearchFilter(django_filters.CharFilter): + def __init__(self, *args, **kwargs): + self.search_fields = kwargs.pop('search_fields') + super().__init__(*args, **kwargs) + + def filter(self, qs, value): + if not value: + return qs + query = utils.get_query(value, self.search_fields) + return qs.filter(query) diff --git a/api/funkwhale_api/common/session.py b/api/funkwhale_api/common/session.py new file mode 100644 index 000000000..7f5584bd1 --- /dev/null +++ b/api/funkwhale_api/common/session.py @@ -0,0 +1,18 @@ +import requests + +from django.conf import settings + +import funkwhale_api + + +def get_user_agent(): + return 'python-requests (funkwhale/{}; +{})'.format( + funkwhale_api.__version__, + settings.FUNKWHALE_URL + ) + + +def get_session(): + s = requests.Session() + s.headers['User-Agent'] = get_user_agent() + return s diff --git a/api/funkwhale_api/common/utils.py b/api/funkwhale_api/common/utils.py index c9d450e6a..2d7641bf5 100644 --- a/api/funkwhale_api/common/utils.py +++ b/api/funkwhale_api/common/utils.py @@ -1,3 +1,4 @@ +from urllib.parse import urlencode, parse_qs, urlsplit, urlunsplit import os import shutil @@ -25,3 +26,20 @@ def on_commit(f, *args, **kwargs): return transaction.on_commit( lambda: f(*args, **kwargs) ) + + +def set_query_parameter(url, **kwargs): + """Given a URL, set or replace a query parameter and return the + modified URL. + + >>> set_query_parameter('http://example.com?foo=bar&biz=baz', 'foo', 'stuff') + 'http://example.com?foo=stuff&biz=baz' + """ + scheme, netloc, path, query_string, fragment = urlsplit(url) + query_params = parse_qs(query_string) + + for param_name, param_value in kwargs.items(): + query_params[param_name] = [param_value] + new_query_string = urlencode(query_params, doseq=True) + + return urlunsplit((scheme, netloc, path, new_query_string, fragment)) diff --git a/api/funkwhale_api/downloader/downloader.py b/api/funkwhale_api/downloader/downloader.py index b35ed6f7d..7fc237b08 100644 --- a/api/funkwhale_api/downloader/downloader.py +++ b/api/funkwhale_api/downloader/downloader.py @@ -1,5 +1,4 @@ import os -import requests import json from urllib.parse import quote_plus import youtube_dl diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index 4eeb193b1..becf6c96f 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -1,11 +1,5 @@ -import logging -import json -import requests -import requests_http_signature - -from . import signing - -logger = logging.getLogger(__name__) +from . import serializers +from . import tasks ACTIVITY_TYPES = [ 'Accept', @@ -42,44 +36,32 @@ ACTIVITY_TYPES = [ OBJECT_TYPES = [ 'Article', 'Audio', + 'Collection', 'Document', 'Event', 'Image', 'Note', + 'OrderedCollection', 'Page', 'Place', 'Profile', 'Relationship', 'Tombstone', 'Video', -] +] + ACTIVITY_TYPES + def deliver(activity, on_behalf_of, to=[]): - from . import actors - logger.info('Preparing activity delivery to %s', to) - auth = requests_http_signature.HTTPSignatureAuth( - use_auth_header=False, - headers=[ - '(request-target)', - 'user-agent', - 'host', - 'date', - 'content-type',], - algorithm='rsa-sha256', - key=on_behalf_of.private_key.encode('utf-8'), - key_id=on_behalf_of.private_key_id, + return tasks.send.delay( + activity=activity, + actor_id=on_behalf_of.pk, + to=to ) - for url in to: - recipient_actor = actors.get_actor(url) - logger.debug('delivering to %s', recipient_actor.inbox_url) - logger.debug('activity content: %s', json.dumps(activity)) - response = requests.post( - auth=auth, - json=activity, - url=recipient_actor.inbox_url, - headers={ - 'Content-Type': 'application/activity+json' - } - ) - response.raise_for_status() - logger.debug('Remote answered with %s', response.status_code) + + +def accept_follow(follow): + serializer = serializers.AcceptFollowSerializer(follow) + return deliver( + serializer.data, + to=[follow.actor.url], + on_behalf_of=follow.target) diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 69033f5ca..380bb23c0 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -1,8 +1,9 @@ import logging -import requests +import uuid import xml from django.conf import settings +from django.db import transaction from django.urls import reverse from django.utils import timezone @@ -10,9 +11,16 @@ from rest_framework.exceptions import PermissionDenied from dynamic_preferences.registries import global_preferences_registry +from funkwhale_api.common import session +from funkwhale_api.common import utils as funkwhale_utils +from funkwhale_api.music import models as music_models +from funkwhale_api.music import tasks as music_tasks + from . import activity +from . import keys from . import models from . import serializers +from . import signing from . import utils logger = logging.getLogger(__name__) @@ -24,8 +32,10 @@ def remove_tags(text): def get_actor_data(actor_url): - response = requests.get( + response = session.get_session().get( actor_url, + timeout=5, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, headers={ 'Accept': 'application/activity+json', } @@ -37,6 +47,7 @@ def get_actor_data(actor_url): raise ValueError( 'Invalid actor payload: {}'.format(response.text)) + def get_actor(actor_url): data = get_actor_data(actor_url) serializer = serializers.ActorSerializer(data=data) @@ -47,31 +58,48 @@ def get_actor(actor_url): class SystemActor(object): additional_attributes = {} + manually_approves_followers = False + + def get_request_auth(self): + actor = self.get_actor_instance() + return signing.get_auth( + actor.private_key, actor.private_key_id) + + def serialize(self): + actor = self.get_actor_instance() + serializer = serializers.ActorSerializer(actor) + return serializer.data def get_actor_instance(self): - a = models.Actor( - **self.get_instance_argument( - self.id, - name=self.name, - summary=self.summary, - **self.additional_attributes - ) + try: + return models.Actor.objects.get(url=self.get_actor_url()) + except models.Actor.DoesNotExist: + pass + private, public = keys.get_key_pair() + args = self.get_instance_argument( + self.id, + name=self.name, + summary=self.summary, + **self.additional_attributes ) - a.pk = self.id - return a + args['private_key'] = private.decode('utf-8') + args['public_key'] = public.decode('utf-8') + return models.Actor.objects.create(**args) + + def get_actor_url(self): + return utils.full_url( + reverse( + 'federation:instance-actors-detail', + kwargs={'actor': self.id})) def get_instance_argument(self, id, name, summary, **kwargs): - preferences = global_preferences_registry.manager() p = { 'preferred_username': id, 'domain': settings.FEDERATION_HOSTNAME, 'type': 'Person', 'name': name.format(host=settings.FEDERATION_HOSTNAME), 'manually_approves_followers': True, - 'url': utils.full_url( - reverse( - 'federation:instance-actors-detail', - kwargs={'actor': id})), + 'url': self.get_actor_url(), 'shared_inbox_url': utils.full_url( reverse( 'federation:instance-actors-inbox', @@ -84,8 +112,6 @@ class SystemActor(object): reverse( 'federation:instance-actors-outbox', kwargs={'actor': id})), - 'public_key': preferences['federation__public_key'], - 'private_key': preferences['federation__private_key'], 'summary': summary.format(host=settings.FEDERATION_HOSTNAME) } p.update(kwargs) @@ -95,7 +121,7 @@ class SystemActor(object): raise NotImplementedError def post_inbox(self, data, actor=None): - raise NotImplementedError + return self.handle(data, actor=actor) def get_outbox(self, data, actor=None): raise NotImplementedError @@ -103,6 +129,70 @@ class SystemActor(object): def post_outbox(self, data, actor=None): raise NotImplementedError + def handle(self, data, actor=None): + """ + Main entrypoint for handling activities posted to the + actor's inbox + """ + logger.info('Received activity on %s inbox', self.id) + + if actor is None: + raise PermissionDenied('Actor not authenticated') + + serializer = serializers.ActivitySerializer( + data=data, context={'actor': actor}) + serializer.is_valid(raise_exception=True) + + ac = serializer.data + try: + handler = getattr( + self, 'handle_{}'.format(ac['type'].lower())) + except (KeyError, AttributeError): + logger.debug( + 'No handler for activity %s', ac['type']) + return + + return handler(data, actor) + + def handle_follow(self, ac, sender): + system_actor = self.get_actor_instance() + serializer = serializers.FollowSerializer( + data=ac, context={'follow_actor': sender}) + if not serializer.is_valid(): + return logger.info('Invalid follow payload') + approved = True if not self.manually_approves_followers else None + follow = serializer.save(approved=approved) + if follow.approved: + return activity.accept_follow(follow) + + def handle_accept(self, ac, sender): + system_actor = self.get_actor_instance() + serializer = serializers.AcceptFollowSerializer( + data=ac, + context={'follow_target': sender, 'follow_actor': system_actor}) + if not serializer.is_valid(raise_exception=True): + return logger.info('Received invalid payload') + + return serializer.save() + + def handle_undo_follow(self, ac, sender): + system_actor = self.get_actor_instance() + serializer = serializers.UndoFollowSerializer( + data=ac, context={'actor': sender, 'target': system_actor}) + if not serializer.is_valid(): + return logger.info('Received invalid payload') + serializer.save() + + def handle_undo(self, ac, sender): + if ac['object']['type'] != 'Follow': + return + + if ac['object']['actor'] != sender.url: + # not the same actor, permission issue + return + + self.handle_undo_follow(ac, sender) + class LibraryActor(SystemActor): id = 'library' @@ -112,6 +202,84 @@ class LibraryActor(SystemActor): 'manually_approves_followers': True } + def serialize(self): + data = super().serialize() + urls = data.setdefault('url', []) + urls.append({ + 'type': 'Link', + 'mediaType': 'application/activity+json', + 'name': 'library', + 'href': utils.full_url(reverse('federation:music:files-list')) + }) + return data + + @property + def manually_approves_followers(self): + return settings.FEDERATION_MUSIC_NEEDS_APPROVAL + + @transaction.atomic + def handle_create(self, ac, sender): + try: + remote_library = models.Library.objects.get( + actor=sender, + federation_enabled=True, + ) + except models.Library.DoesNotExist: + logger.info( + 'Skipping import, we\'re not following %s', sender.url) + return + + if ac['object']['type'] != 'Collection': + return + + if ac['object']['totalItems'] <= 0: + return + + try: + items = ac['object']['items'] + except KeyError: + logger.warning('No items in collection!') + return + + item_serializers = [ + serializers.AudioSerializer( + data=i, context={'library': remote_library}) + for i in items + ] + now = timezone.now() + valid_serializers = [] + for s in item_serializers: + if s.is_valid(): + valid_serializers.append(s) + else: + logger.debug( + 'Skipping invalid item %s, %s', s.initial_data, s.errors) + + lts = [] + for s in valid_serializers: + lts.append(s.save()) + + if remote_library.autoimport: + batch = music_models.ImportBatch.objects.create( + source='federation', + ) + for lt in lts: + if lt.creation_date < now: + # track was already in the library, we do not trigger + # an import + continue + job = music_models.ImportJob.objects.create( + batch=batch, + library_track=lt, + mbid=lt.mbid, + source=lt.url, + ) + funkwhale_utils.on_commit( + music_tasks.import_job_run.delay, + import_job_id=job.pk, + use_acoustid=False, + ) + class TestActor(SystemActor): id = 'test' @@ -123,40 +291,24 @@ class TestActor(SystemActor): additional_attributes = { 'manually_approves_followers': False } + manually_approves_followers = False def get_outbox(self, data, actor=None): return { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {} - ], - "id": utils.full_url( + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + "id": utils.full_url( reverse( 'federation:instance-actors-outbox', kwargs={'actor': self.id})), - "type": "OrderedCollection", - "totalItems": 0, - "orderedItems": [] + "type": "OrderedCollection", + "totalItems": 0, + "orderedItems": [] } - def post_inbox(self, data, actor=None): - if actor is None: - raise PermissionDenied('Actor not authenticated') - - serializer = serializers.ActivitySerializer( - data=data, context={'actor': actor}) - serializer.is_valid(raise_exception=True) - - ac = serializer.validated_data - logger.info('Received activity on %s inbox', self.id) - if ac['type'] == 'Create' and ac['object']['type'] == 'Note': - # we received a toot \o/ - command = self.parse_command(ac['object']['content']) - logger.debug('Parsed command: %s', command) - if command == 'ping': - self.handle_ping(ac, actor) - def parse_command(self, message): """ Remove any links or fancy markup to extract /command from @@ -168,7 +320,16 @@ class TestActor(SystemActor): except IndexError: return - def handle_ping(self, ac, sender): + def handle_create(self, ac, sender): + if ac['object']['type'] != 'Note': + return + + # we received a toot \o/ + command = self.parse_command(ac['object']['content']) + logger.debug('Parsed command: %s', command) + if command != 'ping': + return + now = timezone.now() test_actor = self.get_actor_instance() reply_url = 'https://{}/activities/note/{}'.format( @@ -179,10 +340,10 @@ class TestActor(SystemActor): ) reply_activity = { "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {} - ], + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], 'type': 'Create', 'actor': test_actor.url, 'id': '{}/activity'.format(reply_url), @@ -214,6 +375,39 @@ class TestActor(SystemActor): to=[ac['actor']], on_behalf_of=test_actor) + def handle_follow(self, ac, sender): + super().handle_follow(ac, sender) + # also, we follow back + test_actor = self.get_actor_instance() + follow_back = models.Follow.objects.get_or_create( + actor=test_actor, + target=sender, + approved=None, + )[0] + activity.deliver( + serializers.FollowSerializer(follow_back).data, + to=[follow_back.target.url], + on_behalf_of=follow_back.actor) + + def handle_undo_follow(self, ac, sender): + super().handle_undo_follow(ac, sender) + actor = self.get_actor_instance() + # we also unfollow the sender, if possible + try: + follow = models.Follow.objects.get( + target=sender, + actor=actor, + ) + except models.Follow.DoesNotExist: + return + undo = serializers.UndoFollowSerializer(follow).data + follow.delete() + activity.deliver( + undo, + to=[sender.url], + on_behalf_of=actor) + + SYSTEM_ACTORS = { 'library': LibraryActor(), 'test': TestActor(), diff --git a/api/funkwhale_api/federation/admin.py b/api/funkwhale_api/federation/admin.py new file mode 100644 index 000000000..6a097174b --- /dev/null +++ b/api/funkwhale_api/federation/admin.py @@ -0,0 +1,66 @@ +from django.contrib import admin + +from . import models + + +@admin.register(models.Actor) +class ActorAdmin(admin.ModelAdmin): + list_display = [ + 'url', + 'domain', + 'preferred_username', + 'type', + 'creation_date', + 'last_fetch_date'] + search_fields = ['url', 'domain', 'preferred_username'] + list_filter = [ + 'type' + ] + + +@admin.register(models.Follow) +class FollowAdmin(admin.ModelAdmin): + list_display = [ + 'actor', + 'target', + 'approved', + 'creation_date' + ] + list_filter = [ + 'approved' + ] + search_fields = ['actor__url', 'target__url'] + list_select_related = True + + +@admin.register(models.Library) +class LibraryAdmin(admin.ModelAdmin): + list_display = [ + 'actor', + 'url', + 'creation_date', + 'fetched_date', + 'tracks_count'] + search_fields = ['actor__url', 'url'] + list_filter = [ + 'federation_enabled', + 'download_files', + 'autoimport', + ] + list_select_related = True + + +@admin.register(models.LibraryTrack) +class LibraryTrackAdmin(admin.ModelAdmin): + list_display = [ + 'title', + 'artist_name', + 'album_title', + 'url', + 'library', + 'creation_date', + 'published_date', + ] + search_fields = [ + 'library__url', 'url', 'artist_name', 'title', 'album_title'] + list_select_related = True diff --git a/api/funkwhale_api/federation/api_urls.py b/api/funkwhale_api/federation/api_urls.py new file mode 100644 index 000000000..41dd1c0f9 --- /dev/null +++ b/api/funkwhale_api/federation/api_urls.py @@ -0,0 +1,15 @@ +from rest_framework import routers + +from . import views + +router = routers.SimpleRouter() +router.register( + r'libraries', + views.LibraryViewSet, + 'libraries') +router.register( + r'library-tracks', + views.LibraryTrackViewSet, + 'library-tracks') + +urlpatterns = router.urls diff --git a/api/funkwhale_api/federation/authentication.py b/api/funkwhale_api/federation/authentication.py index e199ef134..7f8ad6653 100644 --- a/api/funkwhale_api/federation/authentication.py +++ b/api/funkwhale_api/federation/authentication.py @@ -7,6 +7,7 @@ from rest_framework import exceptions from . import actors from . import keys +from . import models from . import serializers from . import signing from . import utils @@ -42,11 +43,16 @@ class SignatureAuthentication(authentication.BaseAuthentication): except cryptography.exceptions.InvalidSignature: raise exceptions.AuthenticationFailed('Invalid signature') - return serializer.build() + try: + return models.Actor.objects.get(url=actor_data['id']) + except models.Actor.DoesNotExist: + return serializer.save() def authenticate(self, request): setattr(request, 'actor', None) actor = self.authenticate_actor(request) + if not actor: + return user = AnonymousUser() setattr(request, 'actor', actor) return (user, None) diff --git a/api/funkwhale_api/federation/dynamic_preferences_registry.py b/api/funkwhale_api/federation/dynamic_preferences_registry.py index 83d0285be..43877c75c 100644 --- a/api/funkwhale_api/federation/dynamic_preferences_registry.py +++ b/api/funkwhale_api/federation/dynamic_preferences_registry.py @@ -7,28 +7,14 @@ federation = types.Section('federation') @global_preferences_registry.register -class FederationPrivateKey(types.StringPreference): - show_in_api = False +class MusicCacheDuration(types.IntPreference): + show_in_api = True section = federation - name = 'private_key' - default = '' + name = 'music_cache_duration' + default = 60 * 24 * 2 + verbose_name = 'Music cache duration' help_text = ( - 'Instance private key, used for signing federation HTTP requests' - ) - verbose_name = ( - 'Instance private key (keep it secret, do not change it)' - ) - - -@global_preferences_registry.register -class FederationPublicKey(types.StringPreference): - show_in_api = False - section = federation - name = 'public_key' - default = '' - help_text = ( - 'Instance public key, used for signing federation HTTP requests' - ) - verbose_name = ( - 'Instance public key (do not change it)' + 'How much minutes do you want to keep a copy of federated tracks' + 'locally? Federated files that were not listened in this interval ' + 'will be erased and refetched from the remote on the next listening.' ) diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index 88c86f791..0754c4b2f 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -1,8 +1,10 @@ import factory import requests import requests_http_signature +import uuid from django.utils import timezone +from django.conf import settings from funkwhale_api.factories import registry @@ -51,9 +53,23 @@ class SignedRequestFactory(factory.Factory): self.headers.update(default_headers) +@registry.register(name='federation.Link') +class LinkFactory(factory.Factory): + type = 'Link' + href = factory.Faker('url') + mediaType = 'text/html' + + class Meta: + model = dict + + class Params: + audio = factory.Trait( + mediaType=factory.Iterator(['audio/mp3', 'audio/ogg']) + ) + + @registry.register class ActorFactory(factory.DjangoModelFactory): - public_key = None private_key = None preferred_username = factory.Faker('user_name') @@ -66,6 +82,12 @@ class ActorFactory(factory.DjangoModelFactory): class Meta: model = models.Actor + class Params: + local = factory.Trait( + domain=factory.LazyAttribute( + lambda o: settings.FEDERATION_HOSTNAME) + ) + @classmethod def _generate(cls, create, attrs): has_public = attrs.get('public_key') is not None @@ -77,6 +99,98 @@ class ActorFactory(factory.DjangoModelFactory): return super()._generate(create, attrs) +@registry.register +class FollowFactory(factory.DjangoModelFactory): + target = factory.SubFactory(ActorFactory) + actor = factory.SubFactory(ActorFactory) + + class Meta: + model = models.Follow + + class Params: + local = factory.Trait( + actor=factory.SubFactory(ActorFactory, local=True) + ) + + +@registry.register +class LibraryFactory(factory.DjangoModelFactory): + actor = factory.SubFactory(ActorFactory) + url = factory.Faker('url') + federation_enabled = True + download_files = False + autoimport = False + + class Meta: + model = models.Library + + +class ArtistMetadataFactory(factory.Factory): + name = factory.Faker('name') + + class Meta: + model = dict + + class Params: + musicbrainz = factory.Trait( + musicbrainz_id=factory.Faker('uuid4') + ) + + +class ReleaseMetadataFactory(factory.Factory): + title = factory.Faker('sentence') + + class Meta: + model = dict + + class Params: + musicbrainz = factory.Trait( + musicbrainz_id=factory.Faker('uuid4') + ) + + +class RecordingMetadataFactory(factory.Factory): + title = factory.Faker('sentence') + + class Meta: + model = dict + + class Params: + musicbrainz = factory.Trait( + musicbrainz_id=factory.Faker('uuid4') + ) + + +@registry.register(name='federation.LibraryTrackMetadata') +class LibraryTrackMetadataFactory(factory.Factory): + artist = factory.SubFactory(ArtistMetadataFactory) + recording = factory.SubFactory(RecordingMetadataFactory) + release = factory.SubFactory(ReleaseMetadataFactory) + + class Meta: + model = dict + + +@registry.register +class LibraryTrackFactory(factory.DjangoModelFactory): + library = factory.SubFactory(LibraryFactory) + url = factory.Faker('url') + title = factory.Faker('sentence') + artist_name = factory.Faker('sentence') + album_title = factory.Faker('sentence') + audio_url = factory.Faker('url') + audio_mimetype = 'audio/ogg' + metadata = factory.SubFactory(LibraryTrackMetadataFactory) + + class Meta: + model = models.LibraryTrack + + class Params: + with_audio_file = factory.Trait( + audio_file=factory.django.FileField() + ) + + @registry.register(name='federation.Note') class NoteFactory(factory.Factory): type = 'Note' @@ -89,3 +203,51 @@ class NoteFactory(factory.Factory): class Meta: model = dict + + +@registry.register(name='federation.Activity') +class ActivityFactory(factory.Factory): + type = 'Create' + id = factory.Faker('url') + published = factory.LazyFunction( + lambda: timezone.now().isoformat() + ) + actor = factory.Faker('url') + object = factory.SubFactory( + NoteFactory, + actor=factory.SelfAttribute('..actor'), + published=factory.SelfAttribute('..published')) + + class Meta: + model = dict + + +@registry.register(name='federation.AudioMetadata') +class AudioMetadataFactory(factory.Factory): + recording = factory.LazyAttribute( + lambda o: 'https://musicbrainz.org/recording/{}'.format(uuid.uuid4()) + ) + artist = factory.LazyAttribute( + lambda o: 'https://musicbrainz.org/artist/{}'.format(uuid.uuid4()) + ) + release = factory.LazyAttribute( + lambda o: 'https://musicbrainz.org/release/{}'.format(uuid.uuid4()) + ) + + class Meta: + model = dict + + +@registry.register(name='federation.Audio') +class AudioFactory(factory.Factory): + type = 'Audio' + id = factory.Faker('url') + published = factory.LazyFunction( + lambda: timezone.now().isoformat() + ) + actor = factory.Faker('url') + url = factory.SubFactory(LinkFactory, audio=True) + metadata = factory.SubFactory(LibraryTrackMetadataFactory) + + class Meta: + model = dict diff --git a/api/funkwhale_api/federation/filters.py b/api/funkwhale_api/federation/filters.py new file mode 100644 index 000000000..7a388ff12 --- /dev/null +++ b/api/funkwhale_api/federation/filters.py @@ -0,0 +1,74 @@ +import django_filters + +from funkwhale_api.common import fields + +from . import models + + +class LibraryFilter(django_filters.FilterSet): + approved = django_filters.BooleanFilter('following__approved') + q = fields.SearchFilter(search_fields=[ + 'actor__domain', + ]) + + class Meta: + model = models.Library + fields = { + 'approved': ['exact'], + 'federation_enabled': ['exact'], + 'download_files': ['exact'], + 'autoimport': ['exact'], + 'tracks_count': ['exact'], + } + + +class LibraryTrackFilter(django_filters.FilterSet): + library = django_filters.CharFilter('library__uuid') + imported = django_filters.CharFilter(method='filter_imported') + q = fields.SearchFilter(search_fields=[ + 'artist_name', + 'title', + 'album_title', + 'library__actor__domain', + ]) + + def filter_imported(self, queryset, field_name, value): + if value.lower() in ['true', '1', 'yes']: + queryset = queryset.filter(local_track_file__isnull=False) + elif value.lower() in ['false', '0', 'no']: + queryset = queryset.filter(local_track_file__isnull=True) + return queryset + + class Meta: + model = models.LibraryTrack + fields = { + 'library': ['exact'], + 'artist_name': ['exact', 'icontains'], + 'title': ['exact', 'icontains'], + 'album_title': ['exact', 'icontains'], + 'audio_mimetype': ['exact', 'icontains'], + } + + +class FollowFilter(django_filters.FilterSet): + pending = django_filters.CharFilter(method='filter_pending') + ordering = django_filters.OrderingFilter( + # tuple-mapping retains order + fields=( + ('creation_date', 'creation_date'), + ('modification_date', 'modification_date'), + ), + ) + q = fields.SearchFilter(search_fields=[ + 'actor__domain', + 'actor__preferred_username', + ]) + + class Meta: + model = models.Follow + fields = ['approved', 'pending', 'q'] + + def filter_pending(self, queryset, field_name, value): + if value.lower() in ['true', '1', 'yes']: + queryset = queryset.filter(approved__isnull=True) + return queryset diff --git a/api/funkwhale_api/federation/keys.py b/api/funkwhale_api/federation/keys.py index 08d4034ea..7e9d316c2 100644 --- a/api/funkwhale_api/federation/keys.py +++ b/api/funkwhale_api/federation/keys.py @@ -3,7 +3,6 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.backends import default_backend as crypto_default_backend import re -import requests import urllib.parse from . import exceptions diff --git a/api/funkwhale_api/federation/library.py b/api/funkwhale_api/federation/library.py new file mode 100644 index 000000000..c53ce5430 --- /dev/null +++ b/api/funkwhale_api/federation/library.py @@ -0,0 +1,166 @@ +import json +import requests + +from django.conf import settings + +from funkwhale_api.common import session + +from . import actors +from . import models +from . import serializers +from . import signing +from . import webfinger + + +def scan_from_account_name(account_name): + """ + Given an account name such as library@test.library, will: + + 1. Perform the webfinger lookup + 2. Perform the actor lookup + 3. Perform the library's collection lookup + + and return corresponding data in a dictionary. + """ + data = {} + try: + username, domain = webfinger.clean_acct( + account_name, ensure_local=False) + except serializers.ValidationError: + return { + 'webfinger': { + 'errors': ['Invalid account string'] + } + } + system_library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + library = models.Library.objects.filter( + actor__domain=domain, + actor__preferred_username=username + ).select_related('actor').first() + data['local'] = { + 'following': False, + 'awaiting_approval': False, + } + try: + follow = models.Follow.objects.get( + target__preferred_username=username, + target__domain=username, + actor=system_library, + ) + data['local']['awaiting_approval'] = not bool(follow.approved) + data['local']['following'] = True + except models.Follow.DoesNotExist: + pass + + try: + data['webfinger'] = webfinger.get_resource( + 'acct:{}'.format(account_name)) + except requests.ConnectionError: + return { + 'webfinger': { + 'errors': ['This webfinger resource is not reachable'] + } + } + except requests.HTTPError as e: + return { + 'webfinger': { + 'errors': [ + 'Error {} during webfinger request'.format( + e.response.status_code)] + } + } + except json.JSONDecodeError as e: + return { + 'webfinger': { + 'errors': ['Could not process webfinger response'] + } + } + + try: + data['actor'] = actors.get_actor_data(data['webfinger']['actor_url']) + except requests.ConnectionError: + data['actor'] = { + 'errors': ['This actor is not reachable'] + } + return data + except requests.HTTPError as e: + data['actor'] = { + 'errors': [ + 'Error {} during actor request'.format( + e.response.status_code)] + } + return data + + serializer = serializers.LibraryActorSerializer(data=data['actor']) + if not serializer.is_valid(): + data['actor'] = { + 'errors': ['Invalid ActivityPub actor'] + } + return data + data['library'] = get_library_data( + serializer.validated_data['library_url']) + + return data + + +def get_library_data(library_url): + actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + auth = signing.get_auth(actor.private_key, actor.private_key_id) + try: + response = session.get_session().get( + library_url, + auth=auth, + timeout=5, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, + headers={ + 'Content-Type': 'application/activity+json' + } + ) + except requests.ConnectionError: + return { + 'errors': ['This library is not reachable'] + } + scode = response.status_code + if scode == 401: + return { + 'errors': ['This library requires authentication'] + } + elif scode == 403: + return { + 'errors': ['Permission denied while scanning library'] + } + elif scode >= 400: + return { + 'errors': ['Error {} while fetching the library'.format(scode)] + } + serializer = serializers.PaginatedCollectionSerializer( + data=response.json(), + ) + if not serializer.is_valid(): + return { + 'errors': [ + 'Invalid ActivityPub response from remote library'] + } + + return serializer.validated_data + + +def get_library_page(library, page_url): + actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + auth = signing.get_auth(actor.private_key, actor.private_key_id) + response = session.get_session().get( + page_url, + auth=auth, + timeout=5, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, + headers={ + 'Content-Type': 'application/activity+json' + } + ) + serializer = serializers.CollectionPageSerializer( + data=response.json(), + context={ + 'library': library, + 'item_serializer': serializers.AudioSerializer}) + serializer.is_valid(raise_exception=True) + return serializer.validated_data diff --git a/api/funkwhale_api/federation/management/commands/__init__.py b/api/funkwhale_api/federation/management/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/api/funkwhale_api/federation/management/commands/generate_keys.py b/api/funkwhale_api/federation/management/commands/generate_keys.py deleted file mode 100644 index eafe9aae3..000000000 --- a/api/funkwhale_api/federation/management/commands/generate_keys.py +++ /dev/null @@ -1,53 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction - -from dynamic_preferences.registries import global_preferences_registry - -from funkwhale_api.federation import keys - - -class Command(BaseCommand): - help = ( - 'Generate a public/private key pair for your instance,' - ' for federation purposes. If a key pair already exists, does nothing.' - ) - - def add_arguments(self, parser): - parser.add_argument( - '--replace', - action='store_true', - dest='replace', - default=False, - help='Replace existing key pair, if any', - ) - parser.add_argument( - '--noinput', '--no-input', action='store_false', dest='interactive', - help="Do NOT prompt the user for input of any kind.", - ) - - @transaction.atomic - def handle(self, *args, **options): - preferences = global_preferences_registry.manager() - existing_public = preferences['federation__public_key'] - existing_private = preferences['federation__public_key'] - - if existing_public or existing_private and not options['replace']: - raise CommandError( - 'Keys are already present! ' - 'Replace them with --replace if you know what you are doing.') - - if options['interactive']: - message = ( - 'Are you sure you want to do this?\n\n' - "Type 'yes' to continue, or 'no' to cancel: " - ) - if input(''.join(message)) != 'yes': - raise CommandError("Operation cancelled.") - private, public = keys.get_key_pair() - preferences['federation__public_key'] = public.decode('utf-8') - preferences['federation__private_key'] = private.decode('utf-8') - - self.stdout.write( - 'Your new key pair was generated.' - 'Your public key is now:\n\n{}'.format(public.decode('utf-8')) - ) diff --git a/api/funkwhale_api/federation/migrations/0002_auto_20180403_1620.py b/api/funkwhale_api/federation/migrations/0002_auto_20180403_1620.py new file mode 100644 index 000000000..2200424d8 --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0002_auto_20180403_1620.py @@ -0,0 +1,17 @@ +# Generated by Django 2.0.3 on 2018-04-03 16:20 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0001_initial'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='actor', + unique_together={('domain', 'preferred_username')}, + ), + ] diff --git a/api/funkwhale_api/federation/migrations/0003_auto_20180407_1010.py b/api/funkwhale_api/federation/migrations/0003_auto_20180407_1010.py new file mode 100644 index 000000000..12e3d73fe --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0003_auto_20180407_1010.py @@ -0,0 +1,94 @@ +# Generated by Django 2.0.3 on 2018-04-07 10:10 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import uuid + + +def delete_system_actors(apps, schema_editor): + """Revert site domain and name to default.""" + Actor = apps.get_model("federation", "Actor") + Actor.objects.filter(preferred_username__in=['test', 'library']).delete() + + +def backward(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0002_auto_20180403_1620'), + ] + + operations = [ + migrations.RunPython(delete_system_actors, backward), + migrations.CreateModel( + name='Follow', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('modification_date', models.DateTimeField(auto_now=True)), + ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='emitted_follows', to='federation.Actor')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='received_follows', to='federation.Actor')), + ], + ), + migrations.CreateModel( + name='FollowRequest', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('modification_date', models.DateTimeField(auto_now=True)), + ('approved', models.NullBooleanField(default=None)), + ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='emmited_follow_requests', to='federation.Actor')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='received_follow_requests', to='federation.Actor')), + ], + ), + migrations.CreateModel( + name='Library', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('modification_date', models.DateTimeField(auto_now=True)), + ('fetched_date', models.DateTimeField(blank=True, null=True)), + ('uuid', models.UUIDField(default=uuid.uuid4)), + ('url', models.URLField()), + ('federation_enabled', models.BooleanField()), + ('download_files', models.BooleanField()), + ('autoimport', models.BooleanField()), + ('tracks_count', models.PositiveIntegerField(blank=True, null=True)), + ('actor', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='library', to='federation.Actor')), + ], + ), + migrations.CreateModel( + name='LibraryTrack', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.URLField(unique=True)), + ('audio_url', models.URLField()), + ('audio_mimetype', models.CharField(max_length=200)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('modification_date', models.DateTimeField(auto_now=True)), + ('fetched_date', models.DateTimeField(blank=True, null=True)), + ('published_date', models.DateTimeField(blank=True, null=True)), + ('artist_name', models.CharField(max_length=500)), + ('album_title', models.CharField(max_length=500)), + ('title', models.CharField(max_length=500)), + ('metadata', django.contrib.postgres.fields.jsonb.JSONField(default={}, max_length=10000)), + ('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tracks', to='federation.Library')), + ], + ), + migrations.AddField( + model_name='actor', + name='followers', + field=models.ManyToManyField(related_name='following', through='federation.Follow', to='federation.Actor'), + ), + migrations.AlterUniqueTogether( + name='follow', + unique_together={('actor', 'target')}, + ), + ] diff --git a/api/funkwhale_api/federation/migrations/0004_auto_20180410_2025.py b/api/funkwhale_api/federation/migrations/0004_auto_20180410_2025.py new file mode 100644 index 000000000..bea4d14ae --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0004_auto_20180410_2025.py @@ -0,0 +1,35 @@ +# Generated by Django 2.0.3 on 2018-04-10 20:25 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0003_auto_20180407_1010'), + ] + + operations = [ + migrations.RemoveField( + model_name='followrequest', + name='actor', + ), + migrations.RemoveField( + model_name='followrequest', + name='target', + ), + migrations.AddField( + model_name='follow', + name='approved', + field=models.NullBooleanField(default=None), + ), + migrations.AddField( + model_name='library', + name='follow', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='library', to='federation.Follow'), + ), + migrations.DeleteModel( + name='FollowRequest', + ), + ] diff --git a/api/funkwhale_api/federation/migrations/0005_auto_20180413_1723.py b/api/funkwhale_api/federation/migrations/0005_auto_20180413_1723.py new file mode 100644 index 000000000..00ba5c83d --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0005_auto_20180413_1723.py @@ -0,0 +1,26 @@ +# Generated by Django 2.0.3 on 2018-04-13 17:23 + +import django.contrib.postgres.fields.jsonb +import django.core.serializers.json +from django.db import migrations, models +import funkwhale_api.federation.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0004_auto_20180410_2025'), + ] + + operations = [ + migrations.AddField( + model_name='librarytrack', + name='audio_file', + field=models.FileField(blank=True, null=True, upload_to=funkwhale_api.federation.models.get_file_path), + ), + migrations.AlterField( + model_name='librarytrack', + name='metadata', + field=django.contrib.postgres.fields.jsonb.JSONField(default={}, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=10000), + ), + ] diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index d76ad173b..d91a00c8b 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -1,7 +1,16 @@ +import os +import uuid +import tempfile + from django.conf import settings +from django.contrib.postgres.fields import JSONField +from django.core.serializers.json import DjangoJSONEncoder from django.db import models from django.utils import timezone +from funkwhale_api.common import session +from funkwhale_api.music import utils as music_utils + TYPE_CHOICES = [ ('Person', 'Person'), ('Application', 'Application'), @@ -12,6 +21,8 @@ TYPE_CHOICES = [ class Actor(models.Model): + ap_type = 'Actor' + url = models.URLField(unique=True, max_length=500, db_index=True) outbox_url = models.URLField(max_length=500) inbox_url = models.URLField(max_length=500) @@ -31,6 +42,16 @@ class Actor(models.Model): last_fetch_date = models.DateTimeField( default=timezone.now) manually_approves_followers = models.NullBooleanField(default=None) + followers = models.ManyToManyField( + to='self', + symmetrical=False, + through='Follow', + through_fields=('target', 'actor'), + related_name='following', + ) + + class Meta: + unique_together = ['domain', 'preferred_username'] @property def webfinger_subject(self): @@ -57,3 +78,141 @@ class Actor(models.Model): setattr(self, field, v.lower()) super().save(**kwargs) + + @property + def is_local(self): + return self.domain == settings.FEDERATION_HOSTNAME + + @property + def is_system(self): + from . import actors + return all([ + settings.FEDERATION_HOSTNAME == self.domain, + self.preferred_username in actors.SYSTEM_ACTORS + ]) + + @property + def system_conf(self): + from . import actors + if self.is_system: + return actors.SYSTEM_ACTORS[self.preferred_username] + + def get_approved_followers(self): + follows = self.received_follows.filter(approved=True) + return self.followers.filter( + pk__in=follows.values_list('actor', flat=True)) + + +class Follow(models.Model): + ap_type = 'Follow' + + uuid = models.UUIDField(default=uuid.uuid4, unique=True) + actor = models.ForeignKey( + Actor, + related_name='emitted_follows', + on_delete=models.CASCADE, + ) + target = models.ForeignKey( + Actor, + related_name='received_follows', + on_delete=models.CASCADE, + ) + creation_date = models.DateTimeField(default=timezone.now) + modification_date = models.DateTimeField( + auto_now=True) + approved = models.NullBooleanField(default=None) + + class Meta: + unique_together = ['actor', 'target'] + + def get_federation_url(self): + return '{}#follows/{}'.format(self.actor.url, self.uuid) + + +class Library(models.Model): + creation_date = models.DateTimeField(default=timezone.now) + modification_date = models.DateTimeField( + auto_now=True) + fetched_date = models.DateTimeField(null=True, blank=True) + actor = models.OneToOneField( + Actor, + on_delete=models.CASCADE, + related_name='library') + uuid = models.UUIDField(default=uuid.uuid4) + url = models.URLField() + + # use this flag to disable federation with a library + federation_enabled = models.BooleanField() + # should we mirror files locally or hotlink them? + download_files = models.BooleanField() + # should we automatically import new files from this library? + autoimport = models.BooleanField() + tracks_count = models.PositiveIntegerField(null=True, blank=True) + follow = models.OneToOneField( + Follow, + related_name='library', + null=True, + blank=True, + on_delete=models.SET_NULL, + ) + + +def get_file_path(instance, filename): + uid = str(uuid.uuid4()) + chunk_size = 2 + chunks = [uid[i:i+chunk_size] for i in range(0, len(uid), chunk_size)] + parts = chunks[:3] + [filename] + return os.path.join('federation_cache', *parts) + + +class LibraryTrack(models.Model): + url = models.URLField(unique=True) + audio_url = models.URLField() + audio_mimetype = models.CharField(max_length=200) + audio_file = models.FileField( + upload_to=get_file_path, + null=True, + blank=True) + + creation_date = models.DateTimeField(default=timezone.now) + modification_date = models.DateTimeField( + auto_now=True) + fetched_date = models.DateTimeField(null=True, blank=True) + published_date = models.DateTimeField(null=True, blank=True) + library = models.ForeignKey( + Library, related_name='tracks', on_delete=models.CASCADE) + artist_name = models.CharField(max_length=500) + album_title = models.CharField(max_length=500) + title = models.CharField(max_length=500) + metadata = JSONField( + default={}, max_length=10000, encoder=DjangoJSONEncoder) + + @property + def mbid(self): + try: + return self.metadata['recording']['musicbrainz_id'] + except KeyError: + pass + + def download_audio(self): + from . import actors + auth = actors.SYSTEM_ACTORS['library'].get_request_auth() + remote_response = session.get_session().get( + self.audio_url, + auth=auth, + stream=True, + timeout=20, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, + headers={ + 'Content-Type': 'application/activity+json' + } + ) + with remote_response as r: + remote_response.raise_for_status() + extension = music_utils.get_ext_from_type(self.audio_mimetype) + title = ' - '.join([self.title, self.album_title, self.artist_name]) + filename = '{}.{}'.format(title, extension) + tmp_file = tempfile.TemporaryFile() + for chunk in r.iter_content(chunk_size=512): + tmp_file.write(chunk) + self.audio_file.save(filename, tmp_file) diff --git a/api/funkwhale_api/federation/permissions.py b/api/funkwhale_api/federation/permissions.py new file mode 100644 index 000000000..370328eaa --- /dev/null +++ b/api/funkwhale_api/federation/permissions.py @@ -0,0 +1,19 @@ +from django.conf import settings + +from rest_framework.permissions import BasePermission + +from . import actors + + +class LibraryFollower(BasePermission): + + def has_permission(self, request, view): + if not settings.FEDERATION_MUSIC_NEEDS_APPROVAL: + return True + + actor = getattr(request, 'actor', None) + if actor is None: + return False + + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + return library.followers.filter(url=actor.url).exists() diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 2137e8d91..b56dd3f44 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -2,52 +2,59 @@ import urllib.parse from django.urls import reverse from django.conf import settings +from django.core.paginator import Paginator +from django.db import transaction from rest_framework import serializers from dynamic_preferences.registries import global_preferences_registry +from funkwhale_api.common import utils as funkwhale_utils + from . import activity from . import models from . import utils -class ActorSerializer(serializers.ModelSerializer): - # left maps to activitypub fields, right to our internal models - id = serializers.URLField(source='url') - outbox = serializers.URLField(source='outbox_url') - inbox = serializers.URLField(source='inbox_url') - following = serializers.URLField(source='following_url', required=False) - followers = serializers.URLField(source='followers_url', required=False) - preferredUsername = serializers.CharField( - source='preferred_username', required=False) - publicKey = serializers.JSONField(source='public_key', required=False) - manuallyApprovesFollowers = serializers.NullBooleanField( - source='manually_approves_followers', required=False) - summary = serializers.CharField(max_length=None, required=False) +AP_CONTEXT = [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, +] - class Meta: - model = models.Actor - fields = [ - 'id', - 'type', - 'name', - 'summary', - 'preferredUsername', - 'publicKey', - 'inbox', - 'outbox', - 'following', - 'followers', - 'manuallyApprovesFollowers', - ] + +class ActorSerializer(serializers.Serializer): + id = serializers.URLField() + outbox = serializers.URLField() + inbox = serializers.URLField() + type = serializers.ChoiceField(choices=models.TYPE_CHOICES) + preferredUsername = serializers.CharField() + manuallyApprovesFollowers = serializers.NullBooleanField(required=False) + name = serializers.CharField(required=False, max_length=200) + summary = serializers.CharField(max_length=None, required=False) + followers = serializers.URLField(required=False, allow_null=True) + following = serializers.URLField(required=False, allow_null=True) + publicKey = serializers.JSONField(required=False) def to_representation(self, instance): - ret = super().to_representation(instance) - ret['@context'] = [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - {}, - ] + ret = { + 'id': instance.url, + 'outbox': instance.outbox_url, + 'inbox': instance.inbox_url, + 'preferredUsername': instance.preferred_username, + 'type': instance.type, + } + if instance.name: + ret['name'] = instance.name + if instance.followers_url: + ret['followers'] = instance.followers_url + if instance.following_url: + ret['following'] = instance.following_url + if instance.summary: + ret['summary'] = instance.summary + if instance.manually_approves_followers is not None: + ret['manuallyApprovesFollowers'] = instance.manually_approves_followers + + ret['@context'] = AP_CONTEXT if instance.public_key: ret['publicKey'] = { 'owner': instance.url, @@ -60,8 +67,21 @@ class ActorSerializer(serializers.ModelSerializer): return ret def prepare_missing_fields(self): - kwargs = {} - domain = urllib.parse.urlparse(self.validated_data['url']).netloc + kwargs = { + 'url': self.validated_data['id'], + 'outbox_url': self.validated_data['outbox'], + 'inbox_url': self.validated_data['inbox'], + 'following_url': self.validated_data.get('following'), + 'followers_url': self.validated_data.get('followers'), + 'summary': self.validated_data.get('summary'), + 'type': self.validated_data['type'], + 'name': self.validated_data.get('name'), + 'preferred_username': self.validated_data['preferredUsername'], + } + maf = self.validated_data.get('manuallyApprovesFollowers') + if maf is not None: + kwargs['manually_approves_followers'] = maf + domain = urllib.parse.urlparse(kwargs['url']).netloc kwargs['domain'] = domain for endpoint, url in self.initial_data.get('endpoints', {}).items(): if endpoint == 'sharedInbox': @@ -74,23 +94,386 @@ class ActorSerializer(serializers.ModelSerializer): return kwargs def build(self): - d = self.validated_data.copy() - d.update(self.prepare_missing_fields()) - return self.Meta.model(**d) + d = self.prepare_missing_fields() + return models.Actor(**d) def save(self, **kwargs): - kwargs.update(self.prepare_missing_fields()) - return super().save(**kwargs) + d = self.prepare_missing_fields() + d.update(kwargs) + return models.Actor.objects.create( + **d + ) def validate_summary(self, value): if value: return value[:500] -class ActorWebfingerSerializer(serializers.ModelSerializer): +class APIActorSerializer(serializers.ModelSerializer): class Meta: model = models.Actor - fields = ['url'] + fields = [ + 'id', + 'url', + 'creation_date', + 'summary', + 'preferred_username', + 'name', + 'last_fetch_date', + 'domain', + 'type', + 'manually_approves_followers', + + ] + + +class LibraryActorSerializer(ActorSerializer): + url = serializers.ListField( + child=serializers.JSONField()) + + def validate(self, validated_data): + try: + urls = validated_data['url'] + except KeyError: + raise serializers.ValidationError('Missing URL field') + + for u in urls: + try: + if u['name'] != 'library': + continue + validated_data['library_url'] = u['href'] + break + except KeyError: + continue + + return validated_data + + +class APIFollowSerializer(serializers.ModelSerializer): + class Meta: + model = models.Follow + fields = [ + 'uuid', + 'actor', + 'target', + 'approved', + 'creation_date', + 'modification_date', + ] + + +class APILibrarySerializer(serializers.ModelSerializer): + actor = APIActorSerializer() + follow = APIFollowSerializer() + + class Meta: + model = models.Library + + read_only_fields = [ + 'actor', + 'uuid', + 'url', + 'tracks_count', + 'follow', + 'fetched_date', + 'modification_date', + 'creation_date', + ] + fields = [ + 'autoimport', + 'federation_enabled', + 'download_files', + ] + read_only_fields + + +class APILibraryScanSerializer(serializers.Serializer): + until = serializers.DateTimeField(required=False) + + +class APILibraryFollowUpdateSerializer(serializers.Serializer): + follow = serializers.IntegerField() + approved = serializers.BooleanField() + + def validate_follow(self, value): + from . import actors + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + qs = models.Follow.objects.filter( + pk=value, + target=library_actor, + ) + try: + return qs.get() + except models.Follow.DoesNotExist: + raise serializers.ValidationError('Invalid follow') + + def save(self): + new_status = self.validated_data['approved'] + follow = self.validated_data['follow'] + if new_status == follow.approved: + return follow + + follow.approved = new_status + follow.save(update_fields=['approved', 'modification_date']) + if new_status: + activity.accept_follow(follow) + return follow + + +class APILibraryCreateSerializer(serializers.ModelSerializer): + actor = serializers.URLField() + federation_enabled = serializers.BooleanField() + uuid = serializers.UUIDField(read_only=True) + + class Meta: + model = models.Library + fields = [ + 'uuid', + 'actor', + 'autoimport', + 'federation_enabled', + 'download_files', + ] + + def validate(self, validated_data): + from . import actors + from . import library + + actor_url = validated_data['actor'] + actor_data = actors.get_actor_data(actor_url) + acs = LibraryActorSerializer(data=actor_data) + acs.is_valid(raise_exception=True) + try: + actor = models.Actor.objects.get(url=actor_url) + except models.Actor.DoesNotExist: + actor = acs.save() + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + validated_data['follow'] = models.Follow.objects.get_or_create( + actor=library_actor, + target=actor, + )[0] + if validated_data['follow'].approved is None: + funkwhale_utils.on_commit( + activity.deliver, + FollowSerializer(validated_data['follow']).data, + on_behalf_of=validated_data['follow'].actor, + to=[validated_data['follow'].target.url], + ) + + library_data = library.get_library_data( + acs.validated_data['library_url']) + if 'errors' in library_data: + # we pass silently because it may means we require permission + # before scanning + pass + validated_data['library'] = library_data + validated_data['library'].setdefault( + 'id', acs.validated_data['library_url'] + ) + validated_data['actor'] = actor + return validated_data + + def create(self, validated_data): + library = models.Library.objects.update_or_create( + url=validated_data['library']['id'], + defaults={ + 'actor': validated_data['actor'], + 'follow': validated_data['follow'], + 'tracks_count': validated_data['library'].get('totalItems'), + 'federation_enabled': validated_data['federation_enabled'], + 'autoimport': validated_data['autoimport'], + 'download_files': validated_data['download_files'], + } + )[0] + return library + + +class APILibraryTrackSerializer(serializers.ModelSerializer): + library = APILibrarySerializer() + + class Meta: + model = models.LibraryTrack + fields = [ + 'id', + 'url', + 'audio_url', + 'audio_mimetype', + 'creation_date', + 'modification_date', + 'fetched_date', + 'published_date', + 'metadata', + 'artist_name', + 'album_title', + 'title', + 'library', + 'local_track_file', + ] + + +class FollowSerializer(serializers.Serializer): + id = serializers.URLField() + object = serializers.URLField() + actor = serializers.URLField() + type = serializers.ChoiceField(choices=['Follow']) + + def validate_object(self, v): + expected = self.context.get('follow_target') + if expected and expected.url != v: + raise serializers.ValidationError('Invalid target') + try: + return models.Actor.objects.get(url=v) + except models.Actor.DoesNotExist: + raise serializers.ValidationError('Target not found') + + def validate_actor(self, v): + expected = self.context.get('follow_actor') + if expected and expected.url != v: + raise serializers.ValidationError('Invalid actor') + try: + return models.Actor.objects.get(url=v) + except models.Actor.DoesNotExist: + raise serializers.ValidationError('Actor not found') + + def save(self, **kwargs): + return models.Follow.objects.get_or_create( + actor=self.validated_data['actor'], + target=self.validated_data['object'], + **kwargs, + )[0] + + def to_representation(self, instance): + return { + '@context': AP_CONTEXT, + 'actor': instance.actor.url, + 'id': instance.get_federation_url(), + 'object': instance.target.url, + 'type': 'Follow' + } + return ret + + +class APIFollowSerializer(serializers.ModelSerializer): + actor = APIActorSerializer() + target = APIActorSerializer() + + class Meta: + model = models.Follow + fields = [ + 'uuid', + 'id', + 'approved', + 'creation_date', + 'modification_date', + 'actor', + 'target', + ] + + +class AcceptFollowSerializer(serializers.Serializer): + id = serializers.URLField() + actor = serializers.URLField() + object = FollowSerializer() + type = serializers.ChoiceField(choices=['Accept']) + + def validate_actor(self, v): + expected = self.context.get('follow_target') + if expected and expected.url != v: + raise serializers.ValidationError('Invalid actor') + try: + return models.Actor.objects.get(url=v) + except models.Actor.DoesNotExist: + raise serializers.ValidationError('Actor not found') + + def validate(self, validated_data): + # we ensure the accept actor actually match the follow target + if validated_data['actor'] != validated_data['object']['object']: + raise serializers.ValidationError('Actor mismatch') + try: + validated_data['follow'] = models.Follow.objects.filter( + target=validated_data['actor'], + actor=validated_data['object']['actor'] + ).exclude(approved=True).get() + except models.Follow.DoesNotExist: + raise serializers.ValidationError('No follow to accept') + return validated_data + + def to_representation(self, instance): + return { + "@context": AP_CONTEXT, + "id": instance.get_federation_url() + '/accept', + "type": "Accept", + "actor": instance.target.url, + "object": FollowSerializer(instance).data + } + + def save(self): + self.validated_data['follow'].approved = True + self.validated_data['follow'].save() + return self.validated_data['follow'] + + +class UndoFollowSerializer(serializers.Serializer): + id = serializers.URLField() + actor = serializers.URLField() + object = FollowSerializer() + type = serializers.ChoiceField(choices=['Undo']) + + def validate_actor(self, v): + expected = self.context.get('follow_target') + if expected and expected.url != v: + raise serializers.ValidationError('Invalid actor') + try: + return models.Actor.objects.get(url=v) + except models.Actor.DoesNotExist: + raise serializers.ValidationError('Actor not found') + + def validate(self, validated_data): + # we ensure the accept actor actually match the follow actor + if validated_data['actor'] != validated_data['object']['actor']: + raise serializers.ValidationError('Actor mismatch') + try: + validated_data['follow'] = models.Follow.objects.filter( + actor=validated_data['actor'], + target=validated_data['object']['object'] + ).get() + except models.Follow.DoesNotExist: + raise serializers.ValidationError('No follow to remove') + return validated_data + + def to_representation(self, instance): + return { + "@context": AP_CONTEXT, + "id": instance.get_federation_url() + '/undo', + "type": "Undo", + "actor": instance.actor.url, + "object": FollowSerializer(instance).data + } + + def save(self): + return self.validated_data['follow'].delete() + + +class ActorWebfingerSerializer(serializers.Serializer): + subject = serializers.CharField() + aliases = serializers.ListField(child=serializers.URLField()) + links = serializers.ListField() + actor_url = serializers.URLField(required=False) + + def validate(self, validated_data): + validated_data['actor_url'] = None + for l in validated_data['links']: + try: + if not l['rel'] == 'self': + continue + if not l['type'] == 'application/activity+json': + continue + validated_data['actor_url'] = l['href'] + break + except KeyError: + pass + if validated_data['actor_url'] is None: + raise serializers.ValidationError('No valid actor url found') + return validated_data def to_representation(self, instance): data = {} @@ -110,7 +493,7 @@ class ActorWebfingerSerializer(serializers.ModelSerializer): class ActivitySerializer(serializers.Serializer): actor = serializers.URLField() - id = serializers.URLField() + id = serializers.URLField(required=False) type = serializers.ChoiceField( choices=[(c, c) for c in activity.ACTIVITY_TYPES]) object = serializers.JSONField() @@ -120,7 +503,9 @@ class ActivitySerializer(serializers.Serializer): type = value['type'] except KeyError: raise serializers.ValidationError('Missing object type') - + except TypeError: + # probably a URL + return value try: object_serializer = OBJECT_SERIALIZERS[type] except KeyError: @@ -140,6 +525,14 @@ class ActivitySerializer(serializers.Serializer): ) return value + def to_representation(self, conf): + d = {} + d.update(conf) + + if self.context.get('include_ap_context', True): + d['@context'] = AP_CONTEXT + return d + class ObjectSerializer(serializers.Serializer): id = serializers.URLField() @@ -173,3 +566,242 @@ OBJECT_SERIALIZERS = { t: ObjectSerializer for t in activity.OBJECT_TYPES } + + +class PaginatedCollectionSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=['Collection']) + totalItems = serializers.IntegerField(min_value=0) + actor = serializers.URLField() + id = serializers.URLField() + first = serializers.URLField() + last = serializers.URLField() + + def to_representation(self, conf): + paginator = Paginator( + conf['items'], + conf.get('page_size', 20) + ) + first = funkwhale_utils.set_query_parameter(conf['id'], page=1) + current = first + last = funkwhale_utils.set_query_parameter( + conf['id'], page=paginator.num_pages) + d = { + 'id': conf['id'], + 'actor': conf['actor'].url, + 'totalItems': paginator.count, + 'type': 'Collection', + 'current': current, + 'first': first, + 'last': last, + } + if self.context.get('include_ap_context', True): + d['@context'] = AP_CONTEXT + return d + + +class CollectionPageSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=['CollectionPage']) + totalItems = serializers.IntegerField(min_value=0) + items = serializers.ListField() + actor = serializers.URLField() + id = serializers.URLField() + first = serializers.URLField() + last = serializers.URLField() + next = serializers.URLField(required=False) + prev = serializers.URLField(required=False) + partOf = serializers.URLField() + + def validate_items(self, v): + item_serializer = self.context.get('item_serializer') + if not item_serializer: + return v + raw_items = [item_serializer(data=i, context=self.context) for i in v] + for i in raw_items: + i.is_valid(raise_exception=True) + + return raw_items + + def to_representation(self, conf): + page = conf['page'] + first = funkwhale_utils.set_query_parameter( + conf['id'], page=1) + last = funkwhale_utils.set_query_parameter( + conf['id'], page=page.paginator.num_pages) + id = funkwhale_utils.set_query_parameter( + conf['id'], page=page.number) + d = { + 'id': id, + 'partOf': conf['id'], + 'actor': conf['actor'].url, + 'totalItems': page.paginator.count, + 'type': 'CollectionPage', + 'first': first, + 'last': last, + 'items': [ + conf['item_serializer']( + i, + context={ + 'actor': conf['actor'], + 'include_ap_context': False} + ).data + for i in page.object_list + ] + } + + if page.has_previous(): + d['prev'] = funkwhale_utils.set_query_parameter( + conf['id'], page=page.previous_page_number()) + + if page.has_next(): + d['next'] = funkwhale_utils.set_query_parameter( + conf['id'], page=page.next_page_number()) + + if self.context.get('include_ap_context', True): + d['@context'] = AP_CONTEXT + return d + + +class ArtistMetadataSerializer(serializers.Serializer): + musicbrainz_id = serializers.UUIDField(required=False) + name = serializers.CharField() + + +class ReleaseMetadataSerializer(serializers.Serializer): + musicbrainz_id = serializers.UUIDField(required=False) + title = serializers.CharField() + + +class RecordingMetadataSerializer(serializers.Serializer): + musicbrainz_id = serializers.UUIDField(required=False) + title = serializers.CharField() + + +class AudioMetadataSerializer(serializers.Serializer): + artist = ArtistMetadataSerializer() + release = ReleaseMetadataSerializer() + recording = RecordingMetadataSerializer() + + +class AudioSerializer(serializers.Serializer): + type = serializers.CharField() + id = serializers.URLField() + url = serializers.JSONField() + published = serializers.DateTimeField() + updated = serializers.DateTimeField(required=False) + metadata = AudioMetadataSerializer() + + def validate_type(self, v): + if v != 'Audio': + raise serializers.ValidationError('Invalid type for audio') + return v + + def validate_url(self, v): + try: + url = v['href'] + except (KeyError, TypeError): + raise serializers.ValidationError('Missing href') + + try: + media_type = v['mediaType'] + except (KeyError, TypeError): + raise serializers.ValidationError('Missing mediaType') + + if not media_type.startswith('audio/'): + raise serializers.ValidationError('Invalid mediaType') + + return url + + def validate_url(self, v): + try: + url = v['href'] + except (KeyError, TypeError): + raise serializers.ValidationError('Missing href') + + try: + media_type = v['mediaType'] + except (KeyError, TypeError): + raise serializers.ValidationError('Missing mediaType') + + if not media_type.startswith('audio/'): + raise serializers.ValidationError('Invalid mediaType') + + return v + + def create(self, validated_data): + defaults = { + 'audio_mimetype': validated_data['url']['mediaType'], + 'audio_url': validated_data['url']['href'], + 'metadata': validated_data['metadata'], + 'artist_name': validated_data['metadata']['artist']['name'], + 'album_title': validated_data['metadata']['release']['title'], + 'title': validated_data['metadata']['recording']['title'], + 'published_date': validated_data['published'], + 'modification_date': validated_data.get('updated'), + } + return models.LibraryTrack.objects.get_or_create( + library=self.context['library'], + url=validated_data['id'], + defaults=defaults + )[0] + + def to_representation(self, instance): + track = instance.track + album = instance.track.album + artist = instance.track.artist + + d = { + 'type': 'Audio', + 'id': instance.get_federation_url(), + 'name': instance.track.full_name, + 'published': instance.creation_date.isoformat(), + 'updated': instance.modification_date.isoformat(), + 'metadata': { + 'artist': { + 'musicbrainz_id': str(artist.mbid) if artist.mbid else None, + 'name': artist.name, + }, + 'release': { + 'musicbrainz_id': str(album.mbid) if album.mbid else None, + 'title': album.title, + }, + 'recording': { + 'musicbrainz_id': str(track.mbid) if track.mbid else None, + 'title': track.title, + }, + }, + 'url': { + 'href': utils.full_url(instance.path), + 'type': 'Link', + 'mediaType': instance.mimetype + }, + 'attributedTo': [ + self.context['actor'].url + ] + } + if self.context.get('include_ap_context', True): + d['@context'] = AP_CONTEXT + return d + + +class CollectionSerializer(serializers.Serializer): + + def to_representation(self, conf): + d = { + 'id': conf['id'], + 'actor': conf['actor'].url, + 'totalItems': len(conf['items']), + 'type': 'Collection', + 'items': [ + conf['item_serializer']( + i, + context={ + 'actor': conf['actor'], + 'include_ap_context': False} + ).data + for i in conf['items'] + ] + } + + if self.context.get('include_ap_context', True): + d['@context'] = AP_CONTEXT + return d diff --git a/api/funkwhale_api/federation/signing.py b/api/funkwhale_api/federation/signing.py index 7e4d2aa5a..8d984d3ff 100644 --- a/api/funkwhale_api/federation/signing.py +++ b/api/funkwhale_api/federation/signing.py @@ -53,3 +53,18 @@ def verify_django(django_request, public_key): request.headers[h] = str(v) prepared_request = request.prepare() return verify(request, public_key) + + +def get_auth(private_key, private_key_id): + return requests_http_signature.HTTPSignatureAuth( + use_auth_header=False, + headers=[ + '(request-target)', + 'user-agent', + 'host', + 'date', + 'content-type'], + algorithm='rsa-sha256', + key=private_key.encode('utf-8'), + key_id=private_key_id, + ) diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py new file mode 100644 index 000000000..adc354c4f --- /dev/null +++ b/api/funkwhale_api/federation/tasks.py @@ -0,0 +1,111 @@ +import datetime +import json +import logging + +from django.conf import settings +from django.utils import timezone + +from requests.exceptions import RequestException +from dynamic_preferences.registries import global_preferences_registry + +from funkwhale_api.common import session +from funkwhale_api.history.models import Listening +from funkwhale_api.taskapp import celery + +from . import actors +from . import library as lb +from . import models +from . import signing + + +logger = logging.getLogger(__name__) + + +@celery.app.task( + name='federation.send', + autoretry_for=[RequestException], + retry_backoff=30, + max_retries=5) +@celery.require_instance(models.Actor, 'actor') +def send(activity, actor, to): + logger.info('Preparing activity delivery to %s', to) + auth = signing.get_auth( + actor.private_key, actor.private_key_id) + for url in to: + recipient_actor = actors.get_actor(url) + logger.debug('delivering to %s', recipient_actor.inbox_url) + logger.debug('activity content: %s', json.dumps(activity)) + response = session.get_session().post( + auth=auth, + json=activity, + url=recipient_actor.inbox_url, + timeout=5, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, + headers={ + 'Content-Type': 'application/activity+json' + } + ) + response.raise_for_status() + logger.debug('Remote answered with %s', response.status_code) + + +@celery.app.task( + name='federation.scan_library', + autoretry_for=[RequestException], + retry_backoff=30, + max_retries=5) +@celery.require_instance(models.Library, 'library') +def scan_library(library, until=None): + if not library.federation_enabled: + return + + data = lb.get_library_data(library.url) + scan_library_page.delay( + library_id=library.id, page_url=data['first'], until=until) + library.fetched_date = timezone.now() + library.tracks_count = data['totalItems'] + library.save(update_fields=['fetched_date', 'tracks_count']) + + +@celery.app.task( + name='federation.scan_library_page', + autoretry_for=[RequestException], + retry_backoff=30, + max_retries=5) +@celery.require_instance(models.Library, 'library') +def scan_library_page(library, page_url, until=None): + if not library.federation_enabled: + return + + data = lb.get_library_page(library, page_url) + lts = [] + for item_serializer in data['items']: + item_date = item_serializer.validated_data['published'] + if until and item_date < until: + return + lts.append(item_serializer.save()) + + next_page = data.get('next') + if next_page and next_page != page_url: + scan_library_page.delay(library_id=library.id, page_url=next_page) + + +@celery.app.task(name='federation.clean_music_cache') +def clean_music_cache(): + preferences = global_preferences_registry.manager() + delay = preferences['federation__music_cache_duration'] + if delay < 1: + return # cache clearing disabled + + candidates = models.LibraryTrack.objects.filter( + audio_file__isnull=False + ).values_list('local_track_file__track', flat=True) + listenings = Listening.objects.filter( + creation_date__gte=timezone.now() - datetime.timedelta(minutes=delay), + track__pk__in=candidates).values_list('track', flat=True) + too_old = set(candidates) - set(listenings) + + to_remove = models.LibraryTrack.objects.filter( + local_track_file__track__pk__in=too_old).only('audio_file') + for lt in to_remove: + lt.audio_file.delete() diff --git a/api/funkwhale_api/federation/urls.py b/api/funkwhale_api/federation/urls.py index f2c6f4c78..2c24b5257 100644 --- a/api/funkwhale_api/federation/urls.py +++ b/api/funkwhale_api/federation/urls.py @@ -1,8 +1,10 @@ -from rest_framework import routers +from django.conf.urls import include, url +from rest_framework import routers from . import views router = routers.SimpleRouter(trailing_slash=False) +music_router = routers.SimpleRouter(trailing_slash=False) router.register( r'federation/instance/actors', views.InstanceActorViewSet, @@ -12,4 +14,11 @@ router.register( views.WellKnownViewSet, 'well-known') -urlpatterns = router.urls +music_router.register( + r'files', + views.MusicFilesViewSet, + 'files', +) +urlpatterns = router.urls + [ + url('federation/music/', include((music_router.urls, 'music'), namespace='music')) +] diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 2e3feb8d0..381f87eff 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -1,16 +1,33 @@ from django import forms from django.conf import settings +from django.core import paginator +from django.db import transaction from django.http import HttpResponse +from django.urls import reverse -from rest_framework import viewsets -from rest_framework import views +from rest_framework import mixins +from rest_framework import permissions as rest_permissions from rest_framework import response +from rest_framework import views +from rest_framework import viewsets from rest_framework.decorators import list_route, detail_route +from rest_framework.serializers import ValidationError +from funkwhale_api.common import utils as funkwhale_utils +from funkwhale_api.common.permissions import HasModelPermission +from funkwhale_api.music.models import TrackFile + +from . import activity from . import actors from . import authentication +from . import filters +from . import library +from . import models +from . import permissions from . import renderers from . import serializers +from . import tasks +from . import utils from . import webfinger @@ -38,8 +55,8 @@ class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet): def retrieve(self, request, *args, **kwargs): system_actor = self.get_object() actor = system_actor.get_actor_instance() - serializer = serializers.ActorSerializer(actor) - return response.Response(serializer.data, status=200) + data = actor.system_conf.serialize() + return response.Response(data, status=200) @detail_route(methods=['get', 'post']) def inbox(self, request, *args, **kwargs): @@ -52,7 +69,7 @@ class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet): data = handler(request.data, actor=request.actor) except NotImplementedError: return response.Response(status=405) - return response.Response(data, status=200) + return response.Response({}, status=200) @detail_route(methods=['get', 'post']) def outbox(self, request, *args, **kwargs): @@ -64,7 +81,7 @@ class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet): data = handler(request.data, actor=request.actor) except NotImplementedError: return response.Response(status=405) - return response.Response(data, status=200) + return response.Response({}, status=200) class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet): @@ -101,3 +118,174 @@ class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet): username, hostname = clean_result actor = actors.SYSTEM_ACTORS[username].get_actor_instance() return serializers.ActorWebfingerSerializer(actor).data + + +class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet): + authentication_classes = [ + authentication.SignatureAuthentication] + permission_classes = [permissions.LibraryFollower] + renderer_classes = [renderers.ActivityPubRenderer] + + def list(self, request, *args, **kwargs): + page = request.GET.get('page') + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + qs = TrackFile.objects.order_by('-creation_date').select_related( + 'track__artist', + 'track__album__artist' + ).filter(library_track__isnull=True) + if page is None: + conf = { + 'id': utils.full_url(reverse('federation:music:files-list')), + 'page_size': settings.FEDERATION_COLLECTION_PAGE_SIZE, + 'items': qs, + 'item_serializer': serializers.AudioSerializer, + 'actor': library, + } + serializer = serializers.PaginatedCollectionSerializer(conf) + data = serializer.data + else: + try: + page_number = int(page) + except: + return response.Response( + {'page': ['Invalid page number']}, status=400) + p = paginator.Paginator( + qs, settings.FEDERATION_COLLECTION_PAGE_SIZE) + try: + page = p.page(page_number) + conf = { + 'id': utils.full_url(reverse('federation:music:files-list')), + 'page': page, + 'item_serializer': serializers.AudioSerializer, + 'actor': library, + } + serializer = serializers.CollectionPageSerializer(conf) + data = serializer.data + except paginator.EmptyPage: + return response.Response(status=404) + + return response.Response(data) + + +class LibraryPermission(HasModelPermission): + model = models.Library + + +class LibraryViewSet( + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet): + permission_classes = [LibraryPermission] + queryset = models.Library.objects.all().select_related( + 'actor', + 'follow', + ) + lookup_field = 'uuid' + filter_class = filters.LibraryFilter + serializer_class = serializers.APILibrarySerializer + ordering_fields = ( + 'id', + 'creation_date', + 'fetched_date', + 'actor__domain', + 'tracks_count', + ) + + @list_route(methods=['get']) + def fetch(self, request, *args, **kwargs): + account = request.GET.get('account') + if not account: + return response.Response( + {'account': 'This field is mandatory'}, status=400) + + data = library.scan_from_account_name(account) + return response.Response(data) + + @detail_route(methods=['post']) + def scan(self, request, *args, **kwargs): + library = self.get_object() + serializer = serializers.APILibraryScanSerializer( + data=request.data + ) + serializer.is_valid(raise_exception=True) + result = tasks.scan_library.delay( + library_id=library.pk, + until=serializer.validated_data.get('until') + ) + return response.Response({'task': result.id}) + + @list_route(methods=['get']) + def following(self, request, *args, **kwargs): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + queryset = models.Follow.objects.filter( + actor=library_actor + ).select_related( + 'actor', + 'target', + ).order_by('-creation_date') + filterset = filters.FollowFilter(request.GET, queryset=queryset) + final_qs = filterset.qs + serializer = serializers.APIFollowSerializer(final_qs, many=True) + data = { + 'results': serializer.data, + 'count': len(final_qs), + } + return response.Response(data) + + @list_route(methods=['get', 'patch']) + def followers(self, request, *args, **kwargs): + if request.method.lower() == 'patch': + serializer = serializers.APILibraryFollowUpdateSerializer( + data=request.data) + serializer.is_valid(raise_exception=True) + follow = serializer.save() + return response.Response( + serializers.APIFollowSerializer(follow).data + ) + + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + queryset = models.Follow.objects.filter( + target=library_actor + ).select_related( + 'actor', + 'target', + ).order_by('-creation_date') + filterset = filters.FollowFilter(request.GET, queryset=queryset) + final_qs = filterset.qs + serializer = serializers.APIFollowSerializer(final_qs, many=True) + data = { + 'results': serializer.data, + 'count': len(final_qs), + } + return response.Response(data) + + @transaction.atomic + def create(self, request, *args, **kwargs): + serializer = serializers.APILibraryCreateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + library = serializer.save() + return response.Response(serializer.data, status=201) + + +class LibraryTrackViewSet( + mixins.ListModelMixin, + viewsets.GenericViewSet): + permission_classes = [LibraryPermission] + queryset = models.LibraryTrack.objects.all().select_related( + 'library__actor', + 'library__follow', + 'local_track_file', + ) + filter_class = filters.LibraryTrackFilter + serializer_class = serializers.APILibraryTrackSerializer + ordering_fields = ( + 'id', + 'artist_name', + 'title', + 'album_title', + 'creation_date', + 'modification_date', + 'fetched_date', + 'published_date', + ) diff --git a/api/funkwhale_api/federation/webfinger.py b/api/funkwhale_api/federation/webfinger.py index 4e9753385..f5cb99635 100644 --- a/api/funkwhale_api/federation/webfinger.py +++ b/api/funkwhale_api/federation/webfinger.py @@ -2,8 +2,11 @@ from django import forms from django.conf import settings from django.urls import reverse +from funkwhale_api.common import session + from . import actors from . import utils +from . import serializers VALID_RESOURCE_TYPES = ['acct'] @@ -23,17 +26,32 @@ def clean_resource(resource_string): return resource_type, resource -def clean_acct(acct_string): +def clean_acct(acct_string, ensure_local=True): try: username, hostname = acct_string.split('@') except ValueError: raise forms.ValidationError('Invalid format') - if hostname.lower() != settings.FEDERATION_HOSTNAME: + if ensure_local and hostname.lower() != settings.FEDERATION_HOSTNAME: raise forms.ValidationError( 'Invalid hostname {}'.format(hostname)) - if username not in actors.SYSTEM_ACTORS: + if ensure_local and username not in actors.SYSTEM_ACTORS: raise forms.ValidationError('Invalid username') return username, hostname + + +def get_resource(resource_string): + resource_type, resource = clean_resource(resource_string) + username, hostname = clean_acct(resource, ensure_local=False) + url = 'https://{}/.well-known/webfinger?resource={}'.format( + hostname, resource_string) + response = session.get_session().get( + url, + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, + timeout=5) + response.raise_for_status() + serializer = serializers.ActorWebfingerSerializer(data=response.json()) + serializer.is_valid(raise_exception=True) + return serializer.validated_data diff --git a/api/funkwhale_api/music/factories.py b/api/funkwhale_api/music/factories.py index 303e45228..ea7ff64df 100644 --- a/api/funkwhale_api/music/factories.py +++ b/api/funkwhale_api/music/factories.py @@ -2,6 +2,9 @@ import factory import os from funkwhale_api.factories import registry, ManyToManyFromList +from funkwhale_api.federation.factories import ( + LibraryTrackFactory, +) from funkwhale_api.users.factories import UserFactory SAMPLES_PATH = os.path.join( @@ -53,6 +56,18 @@ class TrackFileFactory(factory.django.DjangoModelFactory): class Meta: model = 'music.TrackFile' + class Params: + federation = factory.Trait( + audio_file=None, + library_track=factory.SubFactory(LibraryTrackFactory), + mimetype=factory.LazyAttribute( + lambda o: o.library_track.audio_mimetype + ), + source=factory.LazyAttribute( + lambda o: o.library_track.audio_url + ), + ) + @registry.register class ImportBatchFactory(factory.django.DjangoModelFactory): @@ -61,6 +76,15 @@ class ImportBatchFactory(factory.django.DjangoModelFactory): class Meta: model = 'music.ImportBatch' + class Params: + federation = factory.Trait( + submitted_by=None, + source='federation', + ) + finished = factory.Trait( + status='finished', + ) + @registry.register class ImportJobFactory(factory.django.DjangoModelFactory): @@ -71,6 +95,17 @@ class ImportJobFactory(factory.django.DjangoModelFactory): class Meta: model = 'music.ImportJob' + class Params: + federation = factory.Trait( + mbid=None, + library_track=factory.SubFactory(LibraryTrackFactory), + batch=factory.SubFactory(ImportBatchFactory, federation=True), + ) + finished = factory.Trait( + status='finished', + track_file=factory.SubFactory(TrackFileFactory), + ) + @registry.register(name='music.FileImportJob') class FileImportJobFactory(ImportJobFactory): diff --git a/api/funkwhale_api/music/forms.py b/api/funkwhale_api/music/forms.py index 04e4bfe05..e68ab73cc 100644 --- a/api/funkwhale_api/music/forms.py +++ b/api/funkwhale_api/music/forms.py @@ -19,5 +19,5 @@ class TranscodeForm(forms.Form): choices=BITRATE_CHOICES, required=False) track_file = forms.ModelChoiceField( - queryset=models.TrackFile.objects.all() + queryset=models.TrackFile.objects.exclude(audio_file__isnull=True) ) diff --git a/api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py b/api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py new file mode 100644 index 000000000..ed7404ac4 --- /dev/null +++ b/api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py @@ -0,0 +1,87 @@ +# Generated by Django 2.0.3 on 2018-04-07 10:10 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0003_auto_20180407_1010'), + ('music', '0022_importbatch_import_request'), + ] + + operations = [ + migrations.AddField( + model_name='album', + name='uuid', + field=models.UUIDField(db_index=True, null=True, unique=True), + ), + migrations.AddField( + model_name='artist', + name='uuid', + field=models.UUIDField(db_index=True, null=True, unique=True), + ), + migrations.AddField( + model_name='importbatch', + name='uuid', + field=models.UUIDField(db_index=True, null=True, unique=True), + ), + migrations.AddField( + model_name='importjob', + name='library_track', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='import_jobs', to='federation.LibraryTrack'), + ), + migrations.AddField( + model_name='importjob', + name='uuid', + field=models.UUIDField(db_index=True, null=True, unique=True), + ), + migrations.AddField( + model_name='lyrics', + name='uuid', + field=models.UUIDField(db_index=True, null=True, unique=True), + ), + migrations.AddField( + model_name='track', + name='uuid', + field=models.UUIDField(db_index=True, null=True, unique=True), + ), + migrations.AddField( + model_name='trackfile', + name='creation_date', + field=models.DateTimeField(default=django.utils.timezone.now), + ), + migrations.AddField( + model_name='trackfile', + name='library_track', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='local_track_file', to='federation.LibraryTrack'), + ), + migrations.AddField( + model_name='trackfile', + name='modification_date', + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name='trackfile', + name='uuid', + field=models.UUIDField(db_index=True, null=True, unique=True), + ), + migrations.AddField( + model_name='work', + name='uuid', + field=models.UUIDField(db_index=True, null=True, unique=True), + ), + migrations.AlterField( + model_name='importbatch', + name='source', + field=models.CharField(choices=[('api', 'api'), ('shell', 'shell'), ('federation', 'federation')], default='api', max_length=30), + ), + migrations.AlterField( + model_name='importbatch', + name='submitted_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='imports', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/api/funkwhale_api/music/migrations/0024_populate_uuid.py b/api/funkwhale_api/music/migrations/0024_populate_uuid.py new file mode 100644 index 000000000..10c78a3db --- /dev/null +++ b/api/funkwhale_api/music/migrations/0024_populate_uuid.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +import os +import uuid +from django.db import migrations, models + + +def populate_uuids(apps, schema_editor): + models = [ + 'Album', + 'Artist', + 'Importbatch', + 'Importjob', + 'Lyrics', + 'Track', + 'Trackfile', + 'Work', + ] + for m in models: + kls = apps.get_model('music', m) + qs = kls.objects.filter(uuid__isnull=True).only('id') + print('Setting uuids for {} ({} objects)'.format(m, len(qs))) + for o in qs: + o.uuid = uuid.uuid4() + o.save(update_fields=['uuid']) + + +def rewind(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('music', '0023_auto_20180407_1010'), + ] + + operations = [ + migrations.RunPython(populate_uuids, rewind), + migrations.AlterField( + model_name='album', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='artist', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='importbatch', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='importjob', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='lyrics', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='track', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='trackfile', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='work', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True), + ), + ] diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index 7138dcdd6..4ec3ff427 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -5,6 +5,7 @@ import datetime import tempfile import shutil import markdown +import uuid from django.conf import settings from django.db import models @@ -20,12 +21,15 @@ from versatileimagefield.fields import VersatileImageField from funkwhale_api import downloader from funkwhale_api import musicbrainz +from funkwhale_api.federation import utils as federation_utils from . import importers from . import utils class APIModelMixin(models.Model): mbid = models.UUIDField(unique=True, db_index=True, null=True, blank=True) + uuid = models.UUIDField( + unique=True, db_index=True, default=uuid.uuid4) api_includes = [] creation_date = models.DateTimeField(default=timezone.now) import_hooks = [] @@ -65,6 +69,13 @@ class APIModelMixin(models.Model): pass return cleaned_data + @property + def musicbrainz_url(self): + if self.mbid: + return 'https://musicbrainz.org/{}/{}'.format( + self.musicbrainz_model, self.mbid) + + class Artist(APIModelMixin): name = models.CharField(max_length=255) @@ -90,10 +101,19 @@ class Artist(APIModelMixin): t.append(tag) return set(t) + @classmethod + def get_or_create_from_name(cls, name, **kwargs): + kwargs.update({'name': name}) + return cls.objects.get_or_create( + name__iexact=name, + defaults=kwargs)[0] + + def import_artist(v): a = Artist.get_or_create_from_api(mbid=v[0]['artist']['id'])[0] return a + def parse_date(v): if len(v) == 4: return datetime.date(int(v), 1, 1) @@ -108,6 +128,7 @@ def import_tracks(instance, cleaned_data, raw_data): track_cleaned_data['position'] = int(track_data['position']) track = importers.load(Track, track_cleaned_data, track_data, Track.import_hooks) + class Album(APIModelMixin): title = models.CharField(max_length=255) artist = models.ForeignKey( @@ -170,6 +191,14 @@ class Album(APIModelMixin): t.append(tag) return set(t) + @classmethod + def get_or_create_from_title(cls, title, **kwargs): + kwargs.update({'title': title}) + return cls.objects.get_or_create( + title__iexact=title, + defaults=kwargs)[0] + + def import_tags(instance, cleaned_data, raw_data): MINIMUM_COUNT = 2 tags_to_add = [] @@ -182,6 +211,7 @@ def import_tags(instance, cleaned_data, raw_data): tags_to_add.append(tag_data['name']) instance.tags.add(*tags_to_add) + def import_album(v): a = Album.get_or_create_from_api(mbid=v[0]['id'])[0] return a @@ -248,6 +278,8 @@ class Work(APIModelMixin): class Lyrics(models.Model): + uuid = models.UUIDField( + unique=True, db_index=True, default=uuid.uuid4) work = models.ForeignKey( Work, related_name='lyrics', @@ -328,7 +360,7 @@ class Track(APIModelMixin): def save(self, **kwargs): try: self.artist - except Artist.DoesNotExist: + except Artist.DoesNotExist: self.artist = self.album.artist super().save(**kwargs) @@ -366,16 +398,35 @@ class Track(APIModelMixin): self.mbid) return settings.FUNKWHALE_URL + '/tracks/{}'.format(self.pk) + @classmethod + def get_or_create_from_title(cls, title, **kwargs): + kwargs.update({'title': title}) + return cls.objects.get_or_create( + title__iexact=title, + defaults=kwargs)[0] + class TrackFile(models.Model): + uuid = models.UUIDField( + unique=True, db_index=True, default=uuid.uuid4) track = models.ForeignKey( Track, related_name='files', on_delete=models.CASCADE) audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255) source = models.URLField(null=True, blank=True) + creation_date = models.DateTimeField(default=timezone.now) + modification_date = models.DateTimeField(auto_now=True) duration = models.IntegerField(null=True, blank=True) acoustid_track_id = models.UUIDField(null=True, blank=True) mimetype = models.CharField(null=True, blank=True, max_length=200) + library_track = models.OneToOneField( + 'federation.LibraryTrack', + related_name='local_track_file', + on_delete=models.CASCADE, + null=True, + blank=True, + ) + def download_file(self): # import the track file, since there is not any # we create a tmp dir for the download @@ -391,12 +442,15 @@ class TrackFile(models.Model): shutil.rmtree(tmp_dir) return self.audio_file + def get_federation_url(self): + return federation_utils.full_url( + '/federation/music/file/{}'.format(self.uuid) + ) + @property def path(self): - if settings.PROTECT_AUDIO_FILES: - return reverse( - 'api:v1:trackfiles-serve', kwargs={'pk': self.pk}) - return self.audio_file.url + return reverse( + 'api:v1:trackfiles-serve', kwargs={'pk': self.pk}) @property def filename(self): @@ -417,10 +471,14 @@ IMPORT_STATUS_CHOICES = ( ('skipped', 'Skipped'), ) + class ImportBatch(models.Model): + uuid = models.UUIDField( + unique=True, db_index=True, default=uuid.uuid4) IMPORT_BATCH_SOURCES = [ ('api', 'api'), - ('shell', 'shell') + ('shell', 'shell'), + ('federation', 'federation'), ] source = models.CharField( max_length=30, default='api', choices=IMPORT_BATCH_SOURCES) @@ -428,6 +486,8 @@ class ImportBatch(models.Model): submitted_by = models.ForeignKey( 'users.User', related_name='imports', + null=True, + blank=True, on_delete=models.CASCADE) status = models.CharField( choices=IMPORT_STATUS_CHOICES, default='pending', max_length=30) @@ -437,6 +497,7 @@ class ImportBatch(models.Model): null=True, blank=True, on_delete=models.CASCADE) + class Meta: ordering = ['-creation_date'] @@ -444,11 +505,22 @@ class ImportBatch(models.Model): return str(self.pk) def update_status(self): + old_status = self.status self.status = utils.compute_status(self.jobs.all()) self.save(update_fields=['status']) + if self.status != old_status and self.status == 'finished': + from . import tasks + tasks.import_batch_notify_followers.delay(import_batch_id=self.pk) + + def get_federation_url(self): + return federation_utils.full_url( + '/federation/music/import/batch/{}'.format(self.uuid) + ) class ImportJob(models.Model): + uuid = models.UUIDField( + unique=True, db_index=True, default=uuid.uuid4) batch = models.ForeignKey( ImportBatch, related_name='jobs', on_delete=models.CASCADE) track_file = models.ForeignKey( @@ -465,6 +537,14 @@ class ImportJob(models.Model): audio_file = models.FileField( upload_to='imports/%Y/%m/%d', max_length=255, null=True, blank=True) + library_track = models.ForeignKey( + 'federation.LibraryTrack', + related_name='import_jobs', + on_delete=models.SET_NULL, + null=True, + blank=True + ) + class Meta: ordering = ('id', ) diff --git a/api/funkwhale_api/music/permissions.py b/api/funkwhale_api/music/permissions.py new file mode 100644 index 000000000..61fc65beb --- /dev/null +++ b/api/funkwhale_api/music/permissions.py @@ -0,0 +1,28 @@ +from django.conf import settings + +from rest_framework.permissions import BasePermission + +from funkwhale_api.federation import actors +from funkwhale_api.federation import models + + +class Listen(BasePermission): + + def has_permission(self, request, view): + if not settings.PROTECT_AUDIO_FILES: + return True + + user = getattr(request, 'user', None) + if user and user.is_authenticated: + return True + + actor = getattr(request, 'actor', None) + if actor is None: + return False + + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + return models.Follow.objects.filter( + target=library, + actor=actor, + approved=True + ).exists() diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index 48419bbe4..b5f69eb1d 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -1,7 +1,11 @@ +from django.db import transaction from rest_framework import serializers from taggit.models import Tag from funkwhale_api.activity import serializers as activity_serializers +from funkwhale_api.federation import utils as federation_utils +from funkwhale_api.federation.models import LibraryTrack +from funkwhale_api.federation.serializers import AP_CONTEXT from . import models @@ -150,3 +154,25 @@ class TrackActivitySerializer(activity_serializers.ModelSerializer): def get_type(self, obj): return 'Audio' + + +class SubmitFederationTracksSerializer(serializers.Serializer): + library_tracks = serializers.PrimaryKeyRelatedField( + many=True, + queryset=LibraryTrack.objects.filter(local_track_file__isnull=True), + ) + + @transaction.atomic + def save(self, **kwargs): + batch = models.ImportBatch.objects.create( + source='federation', + **kwargs + ) + for lt in self.validated_data['library_tracks']: + models.ImportJob.objects.create( + batch=batch, + library_track=lt, + mbid=lt.mbid, + source=lt.url, + ) + return batch diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py index bf7a847d0..bc5ab94f0 100644 --- a/api/funkwhale_api/music/tasks.py +++ b/api/funkwhale_api/music/tasks.py @@ -2,6 +2,10 @@ from django.core.files.base import ContentFile from dynamic_preferences.registries import global_preferences_registry +from funkwhale_api.federation import activity +from funkwhale_api.federation import actors +from funkwhale_api.federation import models as federation_models +from funkwhale_api.federation import serializers as federation_serializers from funkwhale_api.taskapp import celery from funkwhale_api.providers.acoustid import get_acoustid_client from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path @@ -25,6 +29,48 @@ def set_acoustid_on_track_file(track_file): return update(result['id']) +def import_track_from_remote(library_track): + metadata = library_track.metadata + try: + track_mbid = metadata['recording']['musicbrainz_id'] + assert track_mbid # for null/empty values + except (KeyError, AssertionError): + pass + else: + return models.Track.get_or_create_from_api(mbid=track_mbid) + + try: + album_mbid = metadata['release']['musicbrainz_id'] + assert album_mbid # for null/empty values + except (KeyError, AssertionError): + pass + else: + album = models.Album.get_or_create_from_api(mbid=album_mbid) + return models.Track.get_or_create_from_title( + library_track.title, artist=album.artist, album=album) + + try: + artist_mbid = metadata['artist']['musicbrainz_id'] + assert artist_mbid # for null/empty values + except (KeyError, AssertionError): + pass + else: + artist = models.Artist.get_or_create_from_api(mbid=artist_mbid) + album = models.Album.get_or_create_from_title( + library_track.album_title, artist=artist) + return models.Track.get_or_create_from_title( + library_track.title, artist=artist, album=album) + + # worst case scenario, we have absolutely no way to link to a + # musicbrainz resource, we rely on the name/titles + artist = models.Artist.get_or_create_from_name( + library_track.artist_name) + album = models.Album.get_or_create_from_title( + library_track.album_title, artist=artist) + return models.Track.get_or_create_from_title( + library_track.title, artist=artist, album=album) + + def _do_import(import_job, replace, use_acoustid=True): from_file = bool(import_job.audio_file) mbid = import_job.mbid @@ -43,8 +89,14 @@ def _do_import(import_job, replace, use_acoustid=True): acoustid_track_id = match['id'] if mbid: track, _ = models.Track.get_or_create_from_api(mbid=mbid) - else: + elif import_job.audio_file: track = import_track_data_from_path(import_job.audio_file.path) + elif import_job.library_track: + track = import_track_from_remote(import_job.library_track) + else: + raise ValueError( + 'Not enough data to process import, ' + 'add a mbid, an audio file or a library track') track_file = None if replace: @@ -63,6 +115,14 @@ def _do_import(import_job, replace, use_acoustid=True): track_file.audio_file = ContentFile(import_job.audio_file.read()) track_file.audio_file.name = import_job.audio_file.name track_file.duration = duration + elif import_job.library_track: + track_file.library_track = import_job.library_track + track_file.mimetype = import_job.library_track.audio_mimetype + if import_job.library_track.library.download_files: + raise NotImplementedError() + else: + # no downloading, we hotlink + pass else: track_file.download_file() track_file.save() @@ -72,6 +132,7 @@ def _do_import(import_job, replace, use_acoustid=True): # it's imported on the track, we don't need it anymore import_job.audio_file.delete() import_job.save() + return track.pk @@ -106,3 +167,44 @@ def fetch_content(lyrics): cleaned_content = lyrics_utils.clean_content(content) lyrics.content = cleaned_content lyrics.save(update_fields=['content']) + + +@celery.app.task(name='music.import_batch_notify_followers') +@celery.require_instance( + models.ImportBatch.objects.filter(status='finished'), 'import_batch') +def import_batch_notify_followers(import_batch): + if not settings.FEDERATION_ENABLED: + return + + if import_batch.source == 'federation': + return + + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + followers = library_actor.get_approved_followers() + jobs = import_batch.jobs.filter( + status='finished', + library_track__isnull=True, + track_file__isnull=False, + ).select_related( + 'track_file__track__artist', + 'track_file__track__album__artist', + ) + track_files = [job.track_file for job in jobs] + collection = federation_serializers.CollectionSerializer({ + 'actor': library_actor, + 'id': import_batch.get_federation_url(), + 'items': track_files, + 'item_serializer': federation_serializers.AudioSerializer + }).data + for f in followers: + create = federation_serializers.ActivitySerializer( + { + 'type': 'Create', + 'id': collection['id'], + 'object': collection, + 'actor': library_actor.url, + 'to': [f.url], + } + ).data + + activity.deliver(create, on_behalf_of=library_actor, to=[f.url]) diff --git a/api/funkwhale_api/music/utils.py b/api/funkwhale_api/music/utils.py index df659cb80..af0e59ab4 100644 --- a/api/funkwhale_api/music/utils.py +++ b/api/funkwhale_api/music/utils.py @@ -60,3 +60,10 @@ def compute_status(jobs): if pending: return 'pending' return 'finished' + + +def get_ext_from_type(mimetype): + mapping = { + 'audio/ogg': 'ogg', + 'audio/mpeg': 'mp3', + } diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index 5ac3143f9..e8ace1b3a 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -1,39 +1,47 @@ import ffmpeg import os import json +import logging import subprocess import unicodedata import urllib -from django.urls import reverse +from django.contrib.auth.decorators import login_required +from django.core.exceptions import ObjectDoesNotExist +from django.conf import settings from django.db import models, transaction from django.db.models.functions import Length -from django.conf import settings from django.http import StreamingHttpResponse +from django.urls import reverse +from django.utils.decorators import method_decorator from rest_framework import viewsets, views, mixins from rest_framework.decorators import detail_route, list_route from rest_framework.response import Response +from rest_framework import settings as rest_settings from rest_framework import permissions from musicbrainzngs import ResponseError -from django.contrib.auth.decorators import login_required -from django.utils.decorators import method_decorator from funkwhale_api.common import utils as funkwhale_utils +from funkwhale_api.federation import actors from funkwhale_api.requests.models import ImportRequest from funkwhale_api.musicbrainz import api from funkwhale_api.common.permissions import ( ConditionalAuthentication, HasModelPermission) from taggit.models import Tag +from funkwhale_api.federation.authentication import SignatureAuthentication -from . import forms -from . import models -from . import serializers -from . import importers from . import filters +from . import forms +from . import importers +from . import models +from . import permissions as music_permissions +from . import serializers from . import tasks from . import utils +logger = logging.getLogger(__name__) + class SearchMixin(object): search_fields = [] @@ -45,6 +53,7 @@ class SearchMixin(object): serializer = self.serializer_class(queryset, many=True) return Response(serializer.data) + class TagViewSetMixin(object): def get_queryset(self): @@ -179,22 +188,45 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet): class TrackFileViewSet(viewsets.ReadOnlyModelViewSet): queryset = (models.TrackFile.objects.all().order_by('-id')) serializer_class = serializers.TrackFileSerializer - permission_classes = [ConditionalAuthentication] + authentication_classes = rest_settings.api_settings.DEFAULT_AUTHENTICATION_CLASSES + [ + SignatureAuthentication + ] + permission_classes = [music_permissions.Listen] @detail_route(methods=['get']) def serve(self, request, *args, **kwargs): try: - f = models.TrackFile.objects.get(pk=kwargs['pk']) + f = models.TrackFile.objects.select_related( + 'library_track', + 'track__album__artist', + 'track__artist', + ).get(pk=kwargs['pk']) except models.TrackFile.DoesNotExist: return Response(status=404) + mt = f.mimetype + audio_file = f.audio_file + try: + library_track = f.library_track + except ObjectDoesNotExist: + library_track = None + if library_track and not audio_file: + if not library_track.audio_file: + # we need to populate from cache + library_track.download_audio() + audio_file = library_track.audio_file + mt = library_track.audio_mimetype response = Response() - filename = "filename*=UTF-8''{}".format( - urllib.parse.quote(f.filename)) - response["Content-Disposition"] = "attachment; {}".format(filename) + filename = f.filename response['X-Accel-Redirect'] = "{}{}".format( settings.PROTECT_FILES_PATH, - f.audio_file.url) + audio_file.url) + filename = "filename*=UTF-8''{}".format( + urllib.parse.quote(filename)) + response["Content-Disposition"] = "attachment; {}".format(filename) + if mt: + response["Content-Type"] = mt + return response @list_route(methods=['get']) @@ -208,6 +240,8 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet): return Response(form.errors, status=400) f = form.cleaned_data['track_file'] + if not f.audio_file: + return Response(status=400) output_kwargs = { 'format': form.cleaned_data['to'] } @@ -351,6 +385,22 @@ class SubmitViewSet(viewsets.ViewSet): data, request, batch=None, import_request=import_request) return Response(import_data) + @list_route(methods=['post']) + @transaction.non_atomic_requests + def federation(self, request, *args, **kwargs): + serializer = serializers.SubmitFederationTracksSerializer( + data=request.data) + serializer.is_valid(raise_exception=True) + batch = serializer.save(submitted_by=request.user) + for job in batch.jobs.all(): + funkwhale_utils.on_commit( + tasks.import_job_run.delay, + import_job_id=job.pk, + use_acoustid=False, + ) + + return Response({'id': batch.id}, status=201) + @transaction.atomic def _import_album(self, data, request, batch=None, import_request=None): # we import the whole album here to prevent race conditions that occurs diff --git a/api/funkwhale_api/musicbrainz/client.py b/api/funkwhale_api/musicbrainz/client.py index 049ed298c..8e7076a78 100644 --- a/api/funkwhale_api/musicbrainz/client.py +++ b/api/funkwhale_api/musicbrainz/client.py @@ -5,7 +5,7 @@ from django.conf import settings from funkwhale_api import __version__ _api = musicbrainzngs -_api.set_useragent('funkwhale', str(__version__), 'contact@eliotberriot.com') +_api.set_useragent('funkwhale', str(__version__), settings.FUNKWHALE_URL) store = memoize.djangocache.Cache('default') diff --git a/api/funkwhale_api/radios/models.py b/api/funkwhale_api/radios/models.py index d9c12534c..0273b5387 100644 --- a/api/funkwhale_api/radios/models.py +++ b/api/funkwhale_api/radios/models.py @@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError from django.contrib.postgres.fields import JSONField from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType +from django.core.serializers.json import DjangoJSONEncoder from funkwhale_api.music.models import Track @@ -23,7 +24,7 @@ class Radio(models.Model): creation_date = models.DateTimeField(default=timezone.now) is_public = models.BooleanField(default=False) version = models.PositiveIntegerField(default=0) - config = JSONField() + config = JSONField(encoder=DjangoJSONEncoder) def get_candidates(self): return filters.run(self.config) diff --git a/api/funkwhale_api/users/admin.py b/api/funkwhale_api/users/admin.py index ac3712015..89b67d3df 100644 --- a/api/funkwhale_api/users/admin.py +++ b/api/funkwhale_api/users/admin.py @@ -36,3 +36,13 @@ class MyUserCreationForm(UserCreationForm): class UserAdmin(AuthUserAdmin): form = MyUserChangeForm add_form = MyUserCreationForm + list_display = [ + 'username', + 'email', + 'date_joined', + 'last_login', + 'privacy_level', + ] + list_filter = [ + 'privacy_level', + ] diff --git a/api/funkwhale_api/users/models.py b/api/funkwhale_api/users/models.py index 9516c108f..572fa9ddc 100644 --- a/api/funkwhale_api/users/models.py +++ b/api/funkwhale_api/users/models.py @@ -31,6 +31,9 @@ class User(AbstractUser): 'dynamic_preferences.change_globalpreferencemodel': { 'external_codename': 'settings.change', }, + 'federation.change_library': { + 'external_codename': 'federation.manage', + }, } privacy_level = fields.get_privacy_field() diff --git a/api/tests/common/test_session.py b/api/tests/common/test_session.py new file mode 100644 index 000000000..7ff1e660b --- /dev/null +++ b/api/tests/common/test_session.py @@ -0,0 +1,18 @@ +import funkwhale_api + +from funkwhale_api.common import session + + +def test_get_user_agent(settings): + settings.FUNKWHALE_URL = 'https://test.com' + 'http.rb/3.0.0 (Mastodon/2.2.0; +https://mastodon.eliotberriot.com/)' + expected = 'python-requests (funkwhale/{}; +{})'.format( + funkwhale_api.__version__, + settings.FUNKWHALE_URL + ) + assert session.get_user_agent() == expected + + +def test_get_session(): + expected = session.get_user_agent() + assert session.get_session().headers['User-Agent'] == expected diff --git a/api/tests/conftest.py b/api/tests/conftest.py index d5bb56565..64dc394e7 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -69,6 +69,11 @@ def tmpdir(): shutil.rmtree(d) +@pytest.fixture +def tmpfile(): + yield tempfile.NamedTemporaryFile() + + @pytest.fixture def logged_in_client(db, factories, client): user = factories['users.User']() @@ -162,3 +167,12 @@ def media_root(settings): def r_mock(): with requests_mock.mock() as m: yield m + + +@pytest.fixture +def authenticated_actor(factories, mocker): + actor = factories['federation.Actor']() + mocker.patch( + 'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor', + return_value=actor) + yield actor diff --git a/api/tests/federation/conftest.py b/api/tests/federation/conftest.py deleted file mode 100644 index c5831914b..000000000 --- a/api/tests/federation/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -import pytest - - -@pytest.fixture -def authenticated_actor(nodb_factories, mocker): - actor = nodb_factories['federation.Actor']() - mocker.patch( - 'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor', - return_value=actor) - yield actor diff --git a/api/tests/federation/test_activity.py b/api/tests/federation/test_activity.py index a6e1d28aa..c2673ff3b 100644 --- a/api/tests/federation/test_activity.py +++ b/api/tests/federation/test_activity.py @@ -1,11 +1,16 @@ -from funkwhale_api.federation import activity +import uuid -def test_deliver(nodb_factories, r_mock, mocker): - to = nodb_factories['federation.Actor']() +from funkwhale_api.federation import activity +from funkwhale_api.federation import serializers + + +def test_deliver(factories, r_mock, mocker, settings): + settings.CELERY_TASK_ALWAYS_EAGER = True + to = factories['federation.Actor']() mocker.patch( 'funkwhale_api.federation.actors.get_actor', return_value=to) - sender = nodb_factories['federation.Actor']() + sender = factories['federation.Actor']() ac = { 'id': 'http://test.federation/activity', 'type': 'Create', @@ -30,3 +35,14 @@ def test_deliver(nodb_factories, r_mock, mocker): assert r_mock.call_count == 1 assert request.url == to.inbox_url assert request.headers['content-type'] == 'application/activity+json' + + +def test_accept_follow(mocker, factories): + deliver = mocker.patch( + 'funkwhale_api.federation.activity.deliver') + follow = factories['federation.Follow'](approved=None) + expected_accept = serializers.AcceptFollowSerializer(follow).data + activity.accept_follow(follow) + deliver.assert_called_once_with( + expected_accept, to=[follow.actor.url], on_behalf_of=follow.target + ) diff --git a/api/tests/federation/test_actors.py b/api/tests/federation/test_actors.py index b3b0f8df0..7281147a1 100644 --- a/api/tests/federation/test_actors.py +++ b/api/tests/federation/test_actors.py @@ -1,13 +1,19 @@ +import arrow import pytest +import uuid from django.urls import reverse from django.utils import timezone from rest_framework import exceptions +from funkwhale_api.federation import activity from funkwhale_api.federation import actors +from funkwhale_api.federation import models from funkwhale_api.federation import serializers from funkwhale_api.federation import utils +from funkwhale_api.music import models as music_models +from funkwhale_api.music import tasks as music_tasks def test_actor_fetching(r_mock): @@ -23,14 +29,17 @@ def test_actor_fetching(r_mock): assert r == payload -def test_get_library(settings, preferences): - preferences['federation__public_key'] = 'public_key' +def test_get_library(db, settings, mocker): + get_key_pair = mocker.patch( + 'funkwhale_api.federation.keys.get_key_pair', + return_value=(b'private', b'public')) expected = { 'preferred_username': 'library', 'domain': settings.FEDERATION_HOSTNAME, 'type': 'Person', 'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME), 'manually_approves_followers': True, + 'public_key': 'public', 'url': utils.full_url( reverse( 'federation:instance-actors-detail', @@ -47,7 +56,6 @@ def test_get_library(settings, preferences): reverse( 'federation:instance-actors-outbox', kwargs={'actor': 'library'})), - 'public_key': 'public_key', 'summary': 'Bot account to federate with {}\'s library'.format( settings.FEDERATION_HOSTNAME), } @@ -56,14 +64,17 @@ def test_get_library(settings, preferences): assert getattr(actor, key) == value -def test_get_test(settings, preferences): - preferences['federation__public_key'] = 'public_key' +def test_get_test(db, mocker, settings): + get_key_pair = mocker.patch( + 'funkwhale_api.federation.keys.get_key_pair', + return_value=(b'private', b'public')) expected = { 'preferred_username': 'test', 'domain': settings.FEDERATION_HOSTNAME, 'type': 'Person', 'name': '{}\'s test account'.format(settings.FEDERATION_HOSTNAME), 'manually_approves_followers': False, + 'public_key': 'public', 'url': utils.full_url( reverse( 'federation:instance-actors-detail', @@ -80,7 +91,6 @@ def test_get_test(settings, preferences): reverse( 'federation:instance-actors-outbox', kwargs={'actor': 'test'})), - 'public_key': 'public_key', 'summary': 'Bot account to test federation with {}. Send me /ping and I\'ll answer you.'.format( settings.FEDERATION_HOSTNAME), } @@ -91,18 +101,18 @@ def test_get_test(settings, preferences): def test_test_get_outbox(): expected = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {} - ], - "id": utils.full_url( + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + "id": utils.full_url( reverse( 'federation:instance-actors-outbox', kwargs={'actor': 'test'})), - "type": "OrderedCollection", - "totalItems": 0, - "orderedItems": [] + "type": "OrderedCollection", + "totalItems": 0, + "orderedItems": [] } data = actors.SYSTEM_ACTORS['test'].get_outbox({}, actor=None) @@ -126,7 +136,7 @@ def test_test_post_outbox_validates_actor(nodb_factories): assert msg in exc_info.value -def test_test_post_outbox_handles_create_note( +def test_test_post_inbox_handles_create_note( settings, mocker, factories): deliver = mocker.patch( 'funkwhale_api.federation.activity.deliver') @@ -167,11 +177,7 @@ def test_test_post_outbox_handles_create_note( }] ) expected_activity = { - '@context': [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - {} - ], + '@context': serializers.AP_CONTEXT, 'actor': test_actor.url, 'id': 'https://{}/activities/note/{}/activity'.format( settings.FEDERATION_HOSTNAME, now.timestamp() @@ -188,3 +194,335 @@ def test_test_post_outbox_handles_create_note( to=[actor.url], on_behalf_of=actors.SYSTEM_ACTORS['test'].get_actor_instance() ) + + +def test_getting_actor_instance_persists_in_db(db): + test = actors.SYSTEM_ACTORS['test'].get_actor_instance() + from_db = models.Actor.objects.get(url=test.url) + + for f in test._meta.fields: + assert getattr(from_db, f.name) == getattr(test, f.name) + + +@pytest.mark.parametrize('username,domain,expected', [ + ('test', 'wrongdomain.com', False), + ('notsystem', '', False), + ('test', '', True), +]) +def test_actor_is_system( + username, domain, expected, nodb_factories, settings): + if not domain: + domain = settings.FEDERATION_HOSTNAME + + actor = nodb_factories['federation.Actor']( + preferred_username=username, + domain=domain, + ) + assert actor.is_system is expected + + +@pytest.mark.parametrize('username,domain,expected', [ + ('test', 'wrongdomain.com', None), + ('notsystem', '', None), + ('test', '', actors.SYSTEM_ACTORS['test']), +]) +def test_actor_is_system( + username, domain, expected, nodb_factories, settings): + if not domain: + domain = settings.FEDERATION_HOSTNAME + actor = nodb_factories['federation.Actor']( + preferred_username=username, + domain=domain, + ) + assert actor.system_conf == expected + + +@pytest.mark.parametrize('value', [False, True]) +def test_library_actor_manually_approves_based_on_setting( + value, settings): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = value + library_conf = actors.SYSTEM_ACTORS['library'] + assert library_conf.manually_approves_followers is value + + +def test_system_actor_handle(mocker, nodb_factories): + handler = mocker.patch( + 'funkwhale_api.federation.actors.TestActor.handle_create') + actor = nodb_factories['federation.Actor']() + activity = nodb_factories['federation.Activity']( + type='Create', actor=actor.url) + serializer = serializers.ActivitySerializer( + data=activity + ) + assert serializer.is_valid() + actors.SYSTEM_ACTORS['test'].handle(activity, actor) + handler.assert_called_once_with(activity, actor) + + +def test_test_actor_handles_follow( + settings, mocker, factories): + deliver = mocker.patch( + 'funkwhale_api.federation.activity.deliver') + actor = factories['federation.Actor']() + accept_follow = mocker.patch( + 'funkwhale_api.federation.activity.accept_follow') + test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() + data = { + 'actor': actor.url, + 'type': 'Follow', + 'id': 'http://test.federation/user#follows/267', + 'object': test_actor.url, + } + actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor) + follow = models.Follow.objects.get(target=test_actor, approved=True) + follow_back = models.Follow.objects.get(actor=test_actor, approved=None) + accept_follow.assert_called_once_with(follow) + deliver.assert_called_once_with( + serializers.FollowSerializer(follow_back).data, + on_behalf_of=test_actor, + to=[actor.url] + ) + + +def test_test_actor_handles_undo_follow( + settings, mocker, factories): + deliver = mocker.patch( + 'funkwhale_api.federation.activity.deliver') + test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() + follow = factories['federation.Follow'](target=test_actor) + reverse_follow = factories['federation.Follow']( + actor=test_actor, target=follow.actor) + follow_serializer = serializers.FollowSerializer(follow) + reverse_follow_serializer = serializers.FollowSerializer( + reverse_follow) + undo = { + '@context': serializers.AP_CONTEXT, + 'type': 'Undo', + 'id': follow_serializer.data['id'] + '/undo', + 'actor': follow.actor.url, + 'object': follow_serializer.data, + } + expected_undo = { + '@context': serializers.AP_CONTEXT, + 'type': 'Undo', + 'id': reverse_follow_serializer.data['id'] + '/undo', + 'actor': reverse_follow.actor.url, + 'object': reverse_follow_serializer.data, + } + + actors.SYSTEM_ACTORS['test'].post_inbox(undo, actor=follow.actor) + deliver.assert_called_once_with( + expected_undo, + to=[follow.actor.url], + on_behalf_of=test_actor,) + + assert models.Follow.objects.count() == 0 + + +def test_library_actor_handles_follow_manual_approval( + settings, mocker, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True + actor = factories['federation.Actor']() + now = timezone.now() + mocker.patch('django.utils.timezone.now', return_value=now) + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + data = { + 'actor': actor.url, + 'type': 'Follow', + 'id': 'http://test.federation/user#follows/267', + 'object': library_actor.url, + } + + library_actor.system_conf.post_inbox(data, actor=actor) + follow = library_actor.received_follows.first() + + assert follow.actor == actor + assert follow.approved is None + + +def test_library_actor_handles_follow_auto_approval( + settings, mocker, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + actor = factories['federation.Actor']() + accept_follow = mocker.patch( + 'funkwhale_api.federation.activity.accept_follow') + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + data = { + 'actor': actor.url, + 'type': 'Follow', + 'id': 'http://test.federation/user#follows/267', + 'object': library_actor.url, + } + library_actor.system_conf.post_inbox(data, actor=actor) + + follow = library_actor.received_follows.first() + + assert follow.actor == actor + assert follow.approved is True + + +def test_library_actor_handles_accept( + mocker, factories): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + actor = factories['federation.Actor']() + pending_follow = factories['federation.Follow']( + actor=library_actor, + target=actor, + approved=None, + ) + serializer = serializers.AcceptFollowSerializer(pending_follow) + library_actor.system_conf.post_inbox(serializer.data, actor=actor) + + pending_follow.refresh_from_db() + + assert pending_follow.approved is True + + +def test_library_actor_handle_create_audio_no_library(mocker, factories): + # when we receive inbox create audio, we should not do anything + # if we don't have a configured library matching the sender + mocked_create = mocker.patch( + 'funkwhale_api.federation.serializers.AudioSerializer.create' + ) + actor = factories['federation.Actor']() + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + data = { + 'actor': actor.url, + 'type': 'Create', + 'id': 'http://test.federation/audio/create', + 'object': { + 'id': 'https://batch.import', + 'type': 'Collection', + 'totalItems': 2, + 'items': factories['federation.Audio'].create_batch(size=2) + }, + } + library_actor.system_conf.post_inbox(data, actor=actor) + + mocked_create.assert_not_called() + models.LibraryTrack.objects.count() == 0 + + +def test_library_actor_handle_create_audio_no_library_enabled( + mocker, factories): + # when we receive inbox create audio, we should not do anything + # if we don't have an enabled library + mocked_create = mocker.patch( + 'funkwhale_api.federation.serializers.AudioSerializer.create' + ) + disabled_library = factories['federation.Library']( + federation_enabled=False) + actor = disabled_library.actor + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + data = { + 'actor': actor.url, + 'type': 'Create', + 'id': 'http://test.federation/audio/create', + 'object': { + 'id': 'https://batch.import', + 'type': 'Collection', + 'totalItems': 2, + 'items': factories['federation.Audio'].create_batch(size=2) + }, + } + library_actor.system_conf.post_inbox(data, actor=actor) + + mocked_create.assert_not_called() + models.LibraryTrack.objects.count() == 0 + + +def test_library_actor_handle_create_audio(mocker, factories): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + remote_library = factories['federation.Library']( + federation_enabled=True + ) + + data = { + 'actor': remote_library.actor.url, + 'type': 'Create', + 'id': 'http://test.federation/audio/create', + 'object': { + 'id': 'https://batch.import', + 'type': 'Collection', + 'totalItems': 2, + 'items': factories['federation.Audio'].create_batch(size=2) + }, + } + + library_actor.system_conf.post_inbox(data, actor=remote_library.actor) + + lts = list(remote_library.tracks.order_by('id')) + + assert len(lts) == 2 + + for i, a in enumerate(data['object']['items']): + lt = lts[i] + assert lt.pk is not None + assert lt.url == a['id'] + assert lt.library == remote_library + assert lt.audio_url == a['url']['href'] + assert lt.audio_mimetype == a['url']['mediaType'] + assert lt.metadata == a['metadata'] + assert lt.title == a['metadata']['recording']['title'] + assert lt.artist_name == a['metadata']['artist']['name'] + assert lt.album_title == a['metadata']['release']['title'] + assert lt.published_date == arrow.get(a['published']) + + +def test_library_actor_handle_create_audio_autoimport(mocker, factories): + mocked_import = mocker.patch( + 'funkwhale_api.common.utils.on_commit') + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + remote_library = factories['federation.Library']( + federation_enabled=True, + autoimport=True, + ) + + data = { + 'actor': remote_library.actor.url, + 'type': 'Create', + 'id': 'http://test.federation/audio/create', + 'object': { + 'id': 'https://batch.import', + 'type': 'Collection', + 'totalItems': 2, + 'items': factories['federation.Audio'].create_batch(size=2) + }, + } + + library_actor.system_conf.post_inbox(data, actor=remote_library.actor) + + lts = list(remote_library.tracks.order_by('id')) + + assert len(lts) == 2 + + for i, a in enumerate(data['object']['items']): + lt = lts[i] + assert lt.pk is not None + assert lt.url == a['id'] + assert lt.library == remote_library + assert lt.audio_url == a['url']['href'] + assert lt.audio_mimetype == a['url']['mediaType'] + assert lt.metadata == a['metadata'] + assert lt.title == a['metadata']['recording']['title'] + assert lt.artist_name == a['metadata']['artist']['name'] + assert lt.album_title == a['metadata']['release']['title'] + assert lt.published_date == arrow.get(a['published']) + + batch = music_models.ImportBatch.objects.latest('id') + + assert batch.jobs.count() == len(lts) + assert batch.source == 'federation' + assert batch.submitted_by is None + + for i, job in enumerate(batch.jobs.order_by('id')): + lt = lts[i] + assert job.library_track == lt + assert job.mbid == lt.mbid + assert job.source == lt.url + + mocked_import.assert_any_call( + music_tasks.import_job_run.delay, + import_job_id=job.pk, + use_acoustid=False, + ) diff --git a/api/tests/federation/test_authentication.py b/api/tests/federation/test_authentication.py index 1837b3950..2f69e4d4f 100644 --- a/api/tests/federation/test_authentication.py +++ b/api/tests/federation/test_authentication.py @@ -3,22 +3,24 @@ from funkwhale_api.federation import keys from funkwhale_api.federation import signing -def test_authenticate(nodb_factories, mocker, api_request): +def test_authenticate(factories, mocker, api_request): private, public = keys.get_key_pair() actor_url = 'https://test.federation/actor' mocker.patch( 'funkwhale_api.federation.actors.get_actor_data', return_value={ 'id': actor_url, + 'type': 'Person', 'outbox': 'https://test.com', 'inbox': 'https://test.com', + 'preferredUsername': 'test', 'publicKey': { 'publicKeyPem': public.decode('utf-8'), 'owner': actor_url, 'id': actor_url + '#main-key', } }) - signed_request = nodb_factories['federation.SignedRequest']( + signed_request = factories['federation.SignedRequest']( auth__key=private, auth__key_id=actor_url + '#main-key', auth__headers=[ diff --git a/api/tests/federation/test_commands.py b/api/tests/federation/test_commands.py deleted file mode 100644 index 7c5333068..000000000 --- a/api/tests/federation/test_commands.py +++ /dev/null @@ -1,14 +0,0 @@ -from django.core.management import call_command - - -def test_generate_instance_key_pair(preferences, mocker): - mocker.patch( - 'funkwhale_api.federation.keys.get_key_pair', - return_value=(b'private', b'public')) - assert preferences['federation__public_key'] == '' - assert preferences['federation__private_key'] == '' - - call_command('generate_keys', interactive=False) - - assert preferences['federation__private_key'] == 'private' - assert preferences['federation__public_key'] == 'public' diff --git a/api/tests/federation/test_library.py b/api/tests/federation/test_library.py new file mode 100644 index 000000000..7a3abf5d8 --- /dev/null +++ b/api/tests/federation/test_library.py @@ -0,0 +1,70 @@ +from funkwhale_api.federation import library +from funkwhale_api.federation import serializers + + +def test_library_scan_from_account_name(mocker, factories): + actor = factories['federation.Actor']( + preferred_username='library', + domain='test.library' + ) + get_resource_result = {'actor_url': actor.url} + get_resource = mocker.patch( + 'funkwhale_api.federation.webfinger.get_resource', + return_value=get_resource_result) + + actor_data = serializers.ActorSerializer(actor).data + actor_data['manuallyApprovesFollowers'] = False + actor_data['url'] = [{ + 'type': 'Link', + 'name': 'library', + 'mediaType': 'application/activity+json', + 'href': 'https://test.library' + }] + get_actor_data = mocker.patch( + 'funkwhale_api.federation.actors.get_actor_data', + return_value=actor_data) + + get_library_data_result = {'test': 'test'} + get_library_data = mocker.patch( + 'funkwhale_api.federation.library.get_library_data', + return_value=get_library_data_result) + + result = library.scan_from_account_name('library@test.actor') + + get_resource.assert_called_once_with('acct:library@test.actor') + get_actor_data.assert_called_once_with(actor.url) + get_library_data.assert_called_once_with(actor_data['url'][0]['href']) + + assert result == { + 'webfinger': get_resource_result, + 'actor': actor_data, + 'library': get_library_data_result, + 'local': { + 'following': False, + 'awaiting_approval': False, + }, + } + + +def test_get_library_data(r_mock, factories): + actor = factories['federation.Actor']() + url = 'https://test.library' + conf = { + 'id': url, + 'items': [], + 'actor': actor, + 'page_size': 5, + } + data = serializers.PaginatedCollectionSerializer(conf).data + r_mock.get(url, json=data) + + result = library.get_library_data(url) + for f in ['totalItems', 'actor', 'id', 'type']: + assert result[f] == data[f] + + +def test_get_library_data_requires_authentication(r_mock, factories): + url = 'https://test.library' + r_mock.get(url, status_code=403) + result = library.get_library_data(url) + assert result['errors'] == ['Permission denied while scanning library'] diff --git a/api/tests/federation/test_models.py b/api/tests/federation/test_models.py new file mode 100644 index 000000000..ae158e659 --- /dev/null +++ b/api/tests/federation/test_models.py @@ -0,0 +1,41 @@ +import pytest +import uuid + +from django import db + +from funkwhale_api.federation import models +from funkwhale_api.federation import serializers + + +def test_cannot_duplicate_actor(factories): + actor = factories['federation.Actor']() + + with pytest.raises(db.IntegrityError): + factories['federation.Actor']( + domain=actor.domain, + preferred_username=actor.preferred_username, + ) + + +def test_cannot_duplicate_follow(factories): + follow = factories['federation.Follow']() + + with pytest.raises(db.IntegrityError): + factories['federation.Follow']( + target=follow.target, + actor=follow.actor, + ) + + +def test_follow_federation_url(factories): + follow = factories['federation.Follow'](local=True) + expected = '{}#follows/{}'.format( + follow.actor.url, follow.uuid) + + assert follow.get_federation_url() == expected + + +def test_library_model_unique_per_actor(factories): + library = factories['federation.Library']() + with pytest.raises(db.IntegrityError): + factories['federation.Library'](actor=library.actor) diff --git a/api/tests/federation/test_permissions.py b/api/tests/federation/test_permissions.py new file mode 100644 index 000000000..1a6977542 --- /dev/null +++ b/api/tests/federation/test_permissions.py @@ -0,0 +1,45 @@ +from rest_framework.views import APIView + +from funkwhale_api.federation import actors +from funkwhale_api.federation import permissions + + +def test_library_follower( + factories, api_request, anonymous_user, settings): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True + view = APIView.as_view() + permission = permissions.LibraryFollower() + request = api_request.get('/') + setattr(request, 'user', anonymous_user) + check = permission.has_permission(request, view) + + assert check is False + + +def test_library_follower_actor_non_follower( + factories, api_request, anonymous_user, settings): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True + actor = factories['federation.Actor']() + view = APIView.as_view() + permission = permissions.LibraryFollower() + request = api_request.get('/') + setattr(request, 'user', anonymous_user) + setattr(request, 'actor', actor) + check = permission.has_permission(request, view) + + assert check is False + + +def test_library_follower_actor_follower( + factories, api_request, anonymous_user, settings): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow = factories['federation.Follow'](target=library) + view = APIView.as_view() + permission = permissions.LibraryFollower() + request = api_request.get('/') + setattr(request, 'user', anonymous_user) + setattr(request, 'actor', follow.actor) + check = permission.has_permission(request, view) + + assert check is True diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index efa92b16a..6d33a529d 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -1,35 +1,41 @@ -from django.urls import reverse +import arrow +import pytest +from django.urls import reverse +from django.core.paginator import Paginator + +from funkwhale_api.federation import actors from funkwhale_api.federation import keys from funkwhale_api.federation import models from funkwhale_api.federation import serializers +from funkwhale_api.federation import utils def test_actor_serializer_from_ap(db): payload = { - 'id': 'https://test.federation/user', - 'type': 'Person', - 'following': 'https://test.federation/user/following', - 'followers': 'https://test.federation/user/followers', - 'inbox': 'https://test.federation/user/inbox', - 'outbox': 'https://test.federation/user/outbox', - 'preferredUsername': 'user', - 'name': 'Real User', - 'summary': 'Hello world', - 'url': 'https://test.federation/@user', - 'manuallyApprovesFollowers': False, - 'publicKey': { - 'id': 'https://test.federation/user#main-key', - 'owner': 'https://test.federation/user', - 'publicKeyPem': 'yolo' - }, - 'endpoints': { - 'sharedInbox': 'https://test.federation/inbox' - }, + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', + 'name': 'Real User', + 'summary': 'Hello world', + 'url': 'https://test.federation/@user', + 'manuallyApprovesFollowers': False, + 'publicKey': { + 'id': 'https://test.federation/user#main-key', + 'owner': 'https://test.federation/user', + 'publicKeyPem': 'yolo' + }, + 'endpoints': { + 'sharedInbox': 'https://test.federation/inbox' + }, } serializer = serializers.ActorSerializer(data=payload) - assert serializer.is_valid() + assert serializer.is_valid(raise_exception=True) actor = serializer.build() @@ -50,17 +56,17 @@ def test_actor_serializer_from_ap(db): def test_actor_serializer_only_mandatory_field_from_ap(db): payload = { - 'id': 'https://test.federation/user', - 'type': 'Person', - 'following': 'https://test.federation/user/following', - 'followers': 'https://test.federation/user/followers', - 'inbox': 'https://test.federation/user/inbox', - 'outbox': 'https://test.federation/user/outbox', - 'preferredUsername': 'user', + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', } serializer = serializers.ActorSerializer(data=payload) - assert serializer.is_valid() + assert serializer.is_valid(raise_exception=True) actor = serializer.build() @@ -82,24 +88,24 @@ def test_actor_serializer_to_ap(): 'https://w3id.org/security/v1', {}, ], - 'id': 'https://test.federation/user', - 'type': 'Person', - 'following': 'https://test.federation/user/following', - 'followers': 'https://test.federation/user/followers', - 'inbox': 'https://test.federation/user/inbox', - 'outbox': 'https://test.federation/user/outbox', - 'preferredUsername': 'user', - 'name': 'Real User', - 'summary': 'Hello world', - 'manuallyApprovesFollowers': False, - 'publicKey': { - 'id': 'https://test.federation/user#main-key', - 'owner': 'https://test.federation/user', - 'publicKeyPem': 'yolo' - }, - 'endpoints': { - 'sharedInbox': 'https://test.federation/inbox' - }, + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', + 'name': 'Real User', + 'summary': 'Hello world', + 'manuallyApprovesFollowers': False, + 'publicKey': { + 'id': 'https://test.federation/user#main-key', + 'owner': 'https://test.federation/user', + 'publicKeyPem': 'yolo' + }, + 'endpoints': { + 'sharedInbox': 'https://test.federation/inbox' + }, } ac = models.Actor( url=expected['id'], @@ -144,3 +150,534 @@ def test_webfinger_serializer(): serializer = serializers.ActorWebfingerSerializer(actor) assert serializer.data == expected + + +def test_follow_serializer_to_ap(factories): + follow = factories['federation.Follow'](local=True) + serializer = serializers.FollowSerializer(follow) + + expected = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url(), + 'type': 'Follow', + 'actor': follow.actor.url, + 'object': follow.target.url, + } + + assert serializer.data == expected + + +def test_follow_serializer_save(factories): + actor = factories['federation.Actor']() + target = factories['federation.Actor']() + + data = expected = { + 'id': 'https://test.follow', + 'type': 'Follow', + 'actor': actor.url, + 'object': target.url, + } + serializer = serializers.FollowSerializer(data=data) + + assert serializer.is_valid(raise_exception=True) + + follow = serializer.save() + + assert follow.pk is not None + assert follow.actor == actor + assert follow.target == target + assert follow.approved is None + + +def test_follow_serializer_save_validates_on_context(factories): + actor = factories['federation.Actor']() + target = factories['federation.Actor']() + impostor = factories['federation.Actor']() + + data = expected = { + 'id': 'https://test.follow', + 'type': 'Follow', + 'actor': actor.url, + 'object': target.url, + } + serializer = serializers.FollowSerializer( + data=data, + context={'follow_actor': impostor, 'follow_target': impostor}) + + assert serializer.is_valid() is False + + assert 'actor' in serializer.errors + assert 'object' in serializer.errors + + +def test_accept_follow_serializer_representation(factories): + follow = factories['federation.Follow'](approved=None) + + expected = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/accept', + 'type': 'Accept', + 'actor': follow.target.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.AcceptFollowSerializer(follow) + + assert serializer.data == expected + + +def test_accept_follow_serializer_save(factories): + follow = factories['federation.Follow'](approved=None) + + data = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/accept', + 'type': 'Accept', + 'actor': follow.target.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.AcceptFollowSerializer(data=data) + assert serializer.is_valid(raise_exception=True) + serializer.save() + + follow.refresh_from_db() + + assert follow.approved is True + + +def test_accept_follow_serializer_validates_on_context(factories): + follow = factories['federation.Follow'](approved=None) + impostor = factories['federation.Actor']() + data = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/accept', + 'type': 'Accept', + 'actor': impostor.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.AcceptFollowSerializer( + data=data, + context={'follow_actor': impostor, 'follow_target': impostor}) + + assert serializer.is_valid() is False + assert 'actor' in serializer.errors['object'] + assert 'object' in serializer.errors['object'] + + +def test_undo_follow_serializer_representation(factories): + follow = factories['federation.Follow'](approved=True) + + expected = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/undo', + 'type': 'Undo', + 'actor': follow.actor.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.UndoFollowSerializer(follow) + + assert serializer.data == expected + + +def test_undo_follow_serializer_save(factories): + follow = factories['federation.Follow'](approved=True) + + data = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/undo', + 'type': 'Undo', + 'actor': follow.actor.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.UndoFollowSerializer(data=data) + assert serializer.is_valid(raise_exception=True) + serializer.save() + + with pytest.raises(models.Follow.DoesNotExist): + follow.refresh_from_db() + + +def test_undo_follow_serializer_validates_on_context(factories): + follow = factories['federation.Follow'](approved=True) + impostor = factories['federation.Actor']() + data = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'id': follow.get_federation_url() + '/undo', + 'type': 'Undo', + 'actor': impostor.url, + 'object': serializers.FollowSerializer(follow).data, + } + + serializer = serializers.UndoFollowSerializer( + data=data, + context={'follow_actor': impostor, 'follow_target': impostor}) + + assert serializer.is_valid() is False + assert 'actor' in serializer.errors['object'] + assert 'object' in serializer.errors['object'] + + +def test_paginated_collection_serializer(factories): + tfs = factories['music.TrackFile'].create_batch(size=5) + actor = factories['federation.Actor'](local=True) + + conf = { + 'id': 'https://test.federation/test', + 'items': tfs, + 'item_serializer': serializers.AudioSerializer, + 'actor': actor, + 'page_size': 2, + } + expected = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'type': 'Collection', + 'id': conf['id'], + 'actor': actor.url, + 'totalItems': len(tfs), + 'current': conf['id'] + '?page=1', + 'last': conf['id'] + '?page=3', + 'first': conf['id'] + '?page=1', + } + + serializer = serializers.PaginatedCollectionSerializer(conf) + + assert serializer.data == expected + + +def test_paginated_collection_serializer_validation(): + data = { + 'type': 'Collection', + 'id': 'https://test.federation/test', + 'totalItems': 5, + 'actor': 'http://test.actor', + 'first': 'https://test.federation/test?page=1', + 'last': 'https://test.federation/test?page=1', + 'items': [] + } + + serializer = serializers.PaginatedCollectionSerializer( + data=data + ) + + assert serializer.is_valid(raise_exception=True) is True + assert serializer.validated_data['totalItems'] == 5 + assert serializer.validated_data['id'] == data['id'] + assert serializer.validated_data['actor'] == data['actor'] + + +def test_collection_page_serializer_validation(): + base = 'https://test.federation/test' + data = { + 'type': 'CollectionPage', + 'id': base + '?page=2', + 'totalItems': 5, + 'actor': 'https://test.actor', + 'items': [], + 'first': 'https://test.federation/test?page=1', + 'last': 'https://test.federation/test?page=3', + 'prev': base + '?page=1', + 'next': base + '?page=3', + 'partOf': base, + } + + serializer = serializers.CollectionPageSerializer( + data=data + ) + + assert serializer.is_valid(raise_exception=True) is True + assert serializer.validated_data['totalItems'] == 5 + assert serializer.validated_data['id'] == data['id'] + assert serializer.validated_data['actor'] == data['actor'] + assert serializer.validated_data['items'] == [] + assert serializer.validated_data['prev'] == data['prev'] + assert serializer.validated_data['next'] == data['next'] + assert serializer.validated_data['partOf'] == data['partOf'] + + +def test_collection_page_serializer_can_validate_child(): + base = 'https://test.federation/test' + data = { + 'items': [{'in': 'valid'}], + } + + serializer = serializers.CollectionPageSerializer( + data=data, + context={'item_serializer': serializers.AudioSerializer} + ) + + assert serializer.is_valid() is False + assert 'items' in serializer.errors + + +def test_collection_page_serializer(factories): + tfs = factories['music.TrackFile'].create_batch(size=5) + actor = factories['federation.Actor'](local=True) + + conf = { + 'id': 'https://test.federation/test', + 'item_serializer': serializers.AudioSerializer, + 'actor': actor, + 'page': Paginator(tfs, 2).page(2), + } + expected = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {}, + ], + 'type': 'CollectionPage', + 'id': conf['id'] + '?page=2', + 'actor': actor.url, + 'totalItems': len(tfs), + 'partOf': conf['id'], + 'prev': conf['id'] + '?page=1', + 'next': conf['id'] + '?page=3', + 'first': conf['id'] + '?page=1', + 'last': conf['id'] + '?page=3', + 'items': [ + conf['item_serializer']( + i, + context={'actor': actor, 'include_ap_context': False} + ).data + for i in conf['page'].object_list + ] + } + + serializer = serializers.CollectionPageSerializer(conf) + + assert serializer.data == expected + + +def test_activity_pub_audio_serializer_to_library_track(factories): + remote_library = factories['federation.Library']() + audio = factories['federation.Audio']() + serializer = serializers.AudioSerializer( + data=audio, context={'library': remote_library}) + + assert serializer.is_valid(raise_exception=True) + + lt = serializer.save() + + assert lt.pk is not None + assert lt.url == audio['id'] + assert lt.library == remote_library + assert lt.audio_url == audio['url']['href'] + assert lt.audio_mimetype == audio['url']['mediaType'] + assert lt.metadata == audio['metadata'] + assert lt.title == audio['metadata']['recording']['title'] + assert lt.artist_name == audio['metadata']['artist']['name'] + assert lt.album_title == audio['metadata']['release']['title'] + assert lt.published_date == arrow.get(audio['published']) + + +def test_activity_pub_audio_serializer_to_library_track_no_duplicate( + factories): + remote_library = factories['federation.Library']() + audio = factories['federation.Audio']() + serializer1 = serializers.AudioSerializer( + data=audio, context={'library': remote_library}) + serializer2 = serializers.AudioSerializer( + data=audio, context={'library': remote_library}) + + assert serializer1.is_valid() is True + assert serializer2.is_valid() is True + + lt1 = serializer1.save() + lt2 = serializer2.save() + + assert lt1 == lt2 + assert models.LibraryTrack.objects.count() == 1 + + +def test_activity_pub_audio_serializer_to_ap(factories): + tf = factories['music.TrackFile'](mimetype='audio/mp3') + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + expected = { + '@context': serializers.AP_CONTEXT, + 'type': 'Audio', + 'id': tf.get_federation_url(), + 'name': tf.track.full_name, + 'published': tf.creation_date.isoformat(), + 'updated': tf.modification_date.isoformat(), + 'metadata': { + 'artist': { + 'musicbrainz_id': tf.track.artist.mbid, + 'name': tf.track.artist.name, + }, + 'release': { + 'musicbrainz_id': tf.track.album.mbid, + 'title': tf.track.album.title, + }, + 'recording': { + 'musicbrainz_id': tf.track.mbid, + 'title': tf.track.title, + }, + }, + 'url': { + 'href': utils.full_url(tf.path), + 'type': 'Link', + 'mediaType': 'audio/mp3' + }, + 'attributedTo': [ + library.url + ] + } + + serializer = serializers.AudioSerializer(tf, context={'actor': library}) + + assert serializer.data == expected + + +def test_activity_pub_audio_serializer_to_ap_no_mbid(factories): + tf = factories['music.TrackFile']( + mimetype='audio/mp3', + track__mbid=None, + track__album__mbid=None, + track__album__artist__mbid=None, + ) + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + expected = { + '@context': serializers.AP_CONTEXT, + 'type': 'Audio', + 'id': tf.get_federation_url(), + 'name': tf.track.full_name, + 'published': tf.creation_date.isoformat(), + 'updated': tf.modification_date.isoformat(), + 'metadata': { + 'artist': { + 'name': tf.track.artist.name, + 'musicbrainz_id': None, + }, + 'release': { + 'title': tf.track.album.title, + 'musicbrainz_id': None, + }, + 'recording': { + 'title': tf.track.title, + 'musicbrainz_id': None, + }, + }, + 'url': { + 'href': utils.full_url(tf.path), + 'type': 'Link', + 'mediaType': 'audio/mp3' + }, + 'attributedTo': [ + library.url + ] + } + + serializer = serializers.AudioSerializer(tf, context={'actor': library}) + + assert serializer.data == expected + + +def test_collection_serializer_to_ap(factories): + tf1 = factories['music.TrackFile'](mimetype='audio/mp3') + tf2 = factories['music.TrackFile'](mimetype='audio/ogg') + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + expected = { + '@context': serializers.AP_CONTEXT, + 'id': 'https://test.id', + 'actor': library.url, + 'totalItems': 2, + 'type': 'Collection', + 'items': [ + serializers.AudioSerializer( + tf1, context={'actor': library, 'include_ap_context': False} + ).data, + serializers.AudioSerializer( + tf2, context={'actor': library, 'include_ap_context': False} + ).data, + ] + } + + collection = { + 'id': expected['id'], + 'actor': library, + 'items': [tf1, tf2], + 'item_serializer': serializers.AudioSerializer + } + serializer = serializers.CollectionSerializer( + collection, context={'actor': library, 'id': 'https://test.id'}) + + assert serializer.data == expected + + +def test_api_library_create_serializer_save(factories, r_mock): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + actor = factories['federation.Actor']() + follow = factories['federation.Follow']( + target=actor, + actor=library_actor, + ) + actor_data = serializers.ActorSerializer(actor).data + actor_data['url'] = [{ + 'href': 'https://test.library', + 'name': 'library', + 'type': 'Link', + }] + library_conf = { + 'id': 'https://test.library', + 'items': range(10), + 'actor': actor, + 'page_size': 5, + } + library_data = serializers.PaginatedCollectionSerializer(library_conf).data + r_mock.get(actor.url, json=actor_data) + r_mock.get('https://test.library', json=library_data) + data = { + 'actor': actor.url, + 'autoimport': False, + 'federation_enabled': True, + 'download_files': False, + } + + serializer = serializers.APILibraryCreateSerializer(data=data) + assert serializer.is_valid(raise_exception=True) is True + library = serializer.save() + follow = models.Follow.objects.get( + target=actor, actor=library_actor, approved=None) + + assert library.autoimport is data['autoimport'] + assert library.federation_enabled is data['federation_enabled'] + assert library.download_files is data['download_files'] + assert library.tracks_count == 10 + assert library.actor == actor + assert library.follow == follow diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py new file mode 100644 index 000000000..506fbc1fe --- /dev/null +++ b/api/tests/federation/test_tasks.py @@ -0,0 +1,140 @@ +import datetime + +from django.core.paginator import Paginator +from django.utils import timezone + +from funkwhale_api.federation import serializers +from funkwhale_api.federation import tasks + + +def test_scan_library_does_nothing_if_federation_disabled(mocker, factories): + library = factories['federation.Library'](federation_enabled=False) + tasks.scan_library(library_id=library.pk) + + assert library.tracks.count() == 0 + + +def test_scan_library_page_does_nothing_if_federation_disabled( + mocker, factories): + library = factories['federation.Library'](federation_enabled=False) + tasks.scan_library_page(library_id=library.pk, page_url=None) + + assert library.tracks.count() == 0 + + +def test_scan_library_fetches_page_and_calls_scan_page( + mocker, factories, r_mock): + now = timezone.now() + library = factories['federation.Library'](federation_enabled=True) + collection_conf = { + 'actor': library.actor, + 'id': library.url, + 'page_size': 10, + 'items': range(10), + } + collection = serializers.PaginatedCollectionSerializer(collection_conf) + scan_page = mocker.patch( + 'funkwhale_api.federation.tasks.scan_library_page.delay') + r_mock.get(collection_conf['id'], json=collection.data) + tasks.scan_library(library_id=library.pk) + + scan_page.assert_called_once_with( + library_id=library.id, + page_url=collection.data['first'], + until=None, + ) + library.refresh_from_db() + assert library.fetched_date > now + + +def test_scan_page_fetches_page_and_creates_tracks( + mocker, factories, r_mock): + library = factories['federation.Library'](federation_enabled=True) + tfs = factories['music.TrackFile'].create_batch(size=5) + page_conf = { + 'actor': library.actor, + 'id': library.url, + 'page': Paginator(tfs, 5).page(1), + 'item_serializer': serializers.AudioSerializer, + } + page = serializers.CollectionPageSerializer(page_conf) + r_mock.get(page.data['id'], json=page.data) + + tasks.scan_library_page(library_id=library.pk, page_url=page.data['id']) + + lts = list(library.tracks.all().order_by('-published_date')) + assert len(lts) == 5 + + +def test_scan_page_trigger_next_page_scan_skip_if_same( + mocker, factories, r_mock): + patched_scan = mocker.patch( + 'funkwhale_api.federation.tasks.scan_library_page.delay' + ) + library = factories['federation.Library'](federation_enabled=True) + tfs = factories['music.TrackFile'].create_batch(size=1) + page_conf = { + 'actor': library.actor, + 'id': library.url, + 'page': Paginator(tfs, 3).page(1), + 'item_serializer': serializers.AudioSerializer, + } + page = serializers.CollectionPageSerializer(page_conf) + data = page.data + data['next'] = data['id'] + r_mock.get(page.data['id'], json=data) + + tasks.scan_library_page(library_id=library.pk, page_url=data['id']) + patched_scan.assert_not_called() + + +def test_scan_page_stops_once_until_is_reached( + mocker, factories, r_mock): + library = factories['federation.Library'](federation_enabled=True) + tfs = list(reversed(factories['music.TrackFile'].create_batch(size=5))) + page_conf = { + 'actor': library.actor, + 'id': library.url, + 'page': Paginator(tfs, 3).page(1), + 'item_serializer': serializers.AudioSerializer, + } + page = serializers.CollectionPageSerializer(page_conf) + r_mock.get(page.data['id'], json=page.data) + + tasks.scan_library_page( + library_id=library.pk, + page_url=page.data['id'], + until=tfs[1].creation_date) + + lts = list(library.tracks.all().order_by('-published_date')) + assert len(lts) == 2 + for i, tf in enumerate(tfs[:1]): + assert tf.creation_date == lts[i].published_date + + +def test_clean_federation_music_cache_if_no_listen(preferences, factories): + preferences['federation__music_cache_duration'] = 60 + lt1 = factories['federation.LibraryTrack'](with_audio_file=True) + lt2 = factories['federation.LibraryTrack'](with_audio_file=True) + lt3 = factories['federation.LibraryTrack'](with_audio_file=True) + tf1 = factories['music.TrackFile'](library_track=lt1) + tf2 = factories['music.TrackFile'](library_track=lt2) + tf3 = factories['music.TrackFile'](library_track=lt3) + + # we listen to the first one, and the second one (but weeks ago) + listening1 = factories['history.Listening']( + track=tf1.track, + creation_date=timezone.now()) + listening2 = factories['history.Listening']( + track=tf2.track, + creation_date=timezone.now() - datetime.timedelta(minutes=61)) + + tasks.clean_music_cache() + + lt1.refresh_from_db() + lt2.refresh_from_db() + lt3.refresh_from_db() + + assert bool(lt1.audio_file) is True + assert bool(lt2.audio_file) is False + assert bool(lt3.audio_file) is False diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index 0d2ac882f..8c5235b8b 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -1,13 +1,17 @@ +from django.core.paginator import Paginator from django.urls import reverse +from django.utils import timezone import pytest from funkwhale_api.federation import actors +from funkwhale_api.federation import activity +from funkwhale_api.federation import models from funkwhale_api.federation import serializers +from funkwhale_api.federation import utils from funkwhale_api.federation import webfinger - @pytest.mark.parametrize('system_actor', actors.SYSTEM_ACTORS.keys()) def test_instance_actors(system_actor, db, settings, api_client): actor = actors.SYSTEM_ACTORS[system_actor].get_actor_instance() @@ -17,6 +21,8 @@ def test_instance_actors(system_actor, db, settings, api_client): response = api_client.get(url) serializer = serializers.ActorSerializer(actor) + if system_actor == 'library': + response.data.pop('url') assert response.status_code == 200 assert response.data == serializer.data @@ -62,3 +68,315 @@ def test_wellknown_webfinger_system( assert response.status_code == 200 assert response['Content-Type'] == 'application/jrd+json' assert response.data == serializer.data + + +def test_audio_file_list_requires_authenticated_actor( + db, settings, api_client): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = True + url = reverse('federation:music:files-list') + response = api_client.get(url) + + assert response.status_code == 403 + + +def test_audio_file_list_actor_no_page( + db, settings, api_client, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + settings.FEDERATION_COLLECTION_PAGE_SIZE = 2 + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + tfs = factories['music.TrackFile'].create_batch(size=5) + conf = { + 'id': utils.full_url(reverse('federation:music:files-list')), + 'page_size': 2, + 'items': list(reversed(tfs)), # we order by -creation_date + 'item_serializer': serializers.AudioSerializer, + 'actor': library + } + expected = serializers.PaginatedCollectionSerializer(conf).data + url = reverse('federation:music:files-list') + response = api_client.get(url) + + assert response.status_code == 200 + assert response.data == expected + + +def test_audio_file_list_actor_page( + db, settings, api_client, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + settings.FEDERATION_COLLECTION_PAGE_SIZE = 2 + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + tfs = factories['music.TrackFile'].create_batch(size=5) + conf = { + 'id': utils.full_url(reverse('federation:music:files-list')), + 'page': Paginator(list(reversed(tfs)), 2).page(2), + 'item_serializer': serializers.AudioSerializer, + 'actor': library + } + expected = serializers.CollectionPageSerializer(conf).data + url = reverse('federation:music:files-list') + response = api_client.get(url, data={'page': 2}) + + assert response.status_code == 200 + assert response.data == expected + + +def test_audio_file_list_actor_page_exclude_federated_files( + db, settings, api_client, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + library = actors.SYSTEM_ACTORS['library'].get_actor_instance() + tfs = factories['music.TrackFile'].create_batch(size=5, federation=True) + + url = reverse('federation:music:files-list') + response = api_client.get(url) + + assert response.status_code == 200 + assert response.data['totalItems'] == 0 + + +def test_audio_file_list_actor_page_error( + db, settings, api_client, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + url = reverse('federation:music:files-list') + response = api_client.get(url, data={'page': 'nope'}) + + assert response.status_code == 400 + + +def test_audio_file_list_actor_page_error_too_far( + db, settings, api_client, factories): + settings.FEDERATION_MUSIC_NEEDS_APPROVAL = False + url = reverse('federation:music:files-list') + response = api_client.get(url, data={'page': 5000}) + + assert response.status_code == 404 + + +def test_library_actor_includes_library_link(db, settings, api_client): + actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + url = reverse( + 'federation:instance-actors-detail', + kwargs={'actor': 'library'}) + response = api_client.get(url) + expected_links = [ + { + 'type': 'Link', + 'name': 'library', + 'mediaType': 'application/activity+json', + 'href': utils.full_url(reverse('federation:music:files-list')) + } + ] + assert response.status_code == 200 + assert response.data['url'] == expected_links + + +def test_can_fetch_library(superuser_api_client, mocker): + result = {'test': 'test'} + scan = mocker.patch( + 'funkwhale_api.federation.library.scan_from_account_name', + return_value=result) + + url = reverse('api:v1:federation:libraries-fetch') + response = superuser_api_client.get( + url, data={'account': 'test@test.library'}) + + assert response.status_code == 200 + assert response.data == result + scan.assert_called_once_with('test@test.library') + + +def test_follow_library(superuser_api_client, mocker, factories, r_mock): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + actor = factories['federation.Actor']() + follow = {'test': 'follow'} + on_commit = mocker.patch( + 'funkwhale_api.common.utils.on_commit') + actor_data = serializers.ActorSerializer(actor).data + actor_data['url'] = [{ + 'href': 'https://test.library', + 'name': 'library', + 'type': 'Link', + }] + library_conf = { + 'id': 'https://test.library', + 'items': range(10), + 'actor': actor, + 'page_size': 5, + } + library_data = serializers.PaginatedCollectionSerializer(library_conf).data + r_mock.get(actor.url, json=actor_data) + r_mock.get('https://test.library', json=library_data) + data = { + 'actor': actor.url, + 'autoimport': False, + 'federation_enabled': True, + 'download_files': False, + } + + url = reverse('api:v1:federation:libraries-list') + response = superuser_api_client.post( + url, data) + + assert response.status_code == 201 + + follow = models.Follow.objects.get( + actor=library_actor, + target=actor, + approved=None, + ) + library = follow.library + + assert response.data == serializers.APILibraryCreateSerializer( + library).data + + on_commit.assert_called_once_with( + activity.deliver, + serializers.FollowSerializer(follow).data, + on_behalf_of=library_actor, + to=[actor.url] + ) + + +def test_can_list_system_actor_following(factories, superuser_api_client): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow1 = factories['federation.Follow'](actor=library_actor) + follow2 = factories['federation.Follow']() + + url = reverse('api:v1:federation:libraries-following') + response = superuser_api_client.get(url) + + assert response.status_code == 200 + assert response.data['results'] == [ + serializers.APIFollowSerializer(follow1).data + ] + + +def test_can_list_system_actor_followers(factories, superuser_api_client): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow1 = factories['federation.Follow'](actor=library_actor) + follow2 = factories['federation.Follow'](target=library_actor) + + url = reverse('api:v1:federation:libraries-followers') + response = superuser_api_client.get(url) + + assert response.status_code == 200 + assert response.data['results'] == [ + serializers.APIFollowSerializer(follow2).data + ] + + +def test_can_list_libraries(factories, superuser_api_client): + library1 = factories['federation.Library']() + library2 = factories['federation.Library']() + + url = reverse('api:v1:federation:libraries-list') + response = superuser_api_client.get(url) + + assert response.status_code == 200 + assert response.data['results'] == [ + serializers.APILibrarySerializer(library1).data, + serializers.APILibrarySerializer(library2).data, + ] + + +def test_can_detail_library(factories, superuser_api_client): + library = factories['federation.Library']() + + url = reverse( + 'api:v1:federation:libraries-detail', + kwargs={'uuid': str(library.uuid)}) + response = superuser_api_client.get(url) + + assert response.status_code == 200 + assert response.data == serializers.APILibrarySerializer(library).data + + +def test_can_patch_library(factories, superuser_api_client): + library = factories['federation.Library']() + data = { + 'federation_enabled': not library.federation_enabled, + 'download_files': not library.download_files, + 'autoimport': not library.autoimport, + } + url = reverse( + 'api:v1:federation:libraries-detail', + kwargs={'uuid': str(library.uuid)}) + response = superuser_api_client.patch(url, data) + + assert response.status_code == 200 + library.refresh_from_db() + + for k, v in data.items(): + assert getattr(library, k) == v + + +def test_scan_library(factories, mocker, superuser_api_client): + scan = mocker.patch( + 'funkwhale_api.federation.tasks.scan_library.delay', + return_value=mocker.Mock(id='id')) + library = factories['federation.Library']() + now = timezone.now() + data = { + 'until': now, + } + url = reverse( + 'api:v1:federation:libraries-scan', + kwargs={'uuid': str(library.uuid)}) + response = superuser_api_client.post(url, data) + + assert response.status_code == 200 + assert response.data == {'task': 'id'} + scan.assert_called_once_with( + library_id=library.pk, + until=now + ) + + +def test_list_library_tracks(factories, superuser_api_client): + library = factories['federation.Library']() + lts = list(reversed(factories['federation.LibraryTrack'].create_batch( + size=5, library=library))) + factories['federation.LibraryTrack'].create_batch(size=5) + url = reverse('api:v1:federation:library-tracks-list') + response = superuser_api_client.get(url, {'library': library.uuid}) + + assert response.status_code == 200 + assert response.data == { + 'results': serializers.APILibraryTrackSerializer(lts, many=True).data, + 'count': 5, + 'previous': None, + 'next': None, + } + + +def test_can_update_follow_status(factories, superuser_api_client, mocker): + patched_accept = mocker.patch( + 'funkwhale_api.federation.activity.accept_follow' + ) + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow = factories['federation.Follow'](target=library_actor) + + payload = { + 'follow': follow.pk, + 'approved': True + } + url = reverse('api:v1:federation:libraries-followers') + response = superuser_api_client.patch(url, payload) + follow.refresh_from_db() + + assert response.status_code == 200 + assert follow.approved is True + patched_accept.assert_called_once_with(follow) + + +def test_can_filter_pending_follows(factories, superuser_api_client): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow = factories['federation.Follow']( + target=library_actor, + approved=True) + + params = {'pending': True} + url = reverse('api:v1:federation:libraries-followers') + response = superuser_api_client.get(url, params) + + assert response.status_code == 200 + assert len(response.data['results']) == 0 diff --git a/api/tests/federation/test_webfinger.py b/api/tests/federation/test_webfinger.py index 96258455a..4b8dca207 100644 --- a/api/tests/federation/test_webfinger.py +++ b/api/tests/federation/test_webfinger.py @@ -40,3 +40,29 @@ def test_webfinger_clean_acct_errors(resource, message, settings): webfinger.clean_resource(resource) assert message == str(excinfo) + + +def test_webfinger_get_resource(r_mock): + resource = 'acct:test@test.webfinger' + payload = { + 'subject': resource, + 'aliases': ['https://test.webfinger'], + 'links': [ + { + 'rel': 'self', + 'type': 'application/activity+json', + 'href': 'https://test.webfinger/user/test' + } + ] + } + r_mock.get( + 'https://test.webfinger/.well-known/webfinger?resource={}'.format( + resource + ), + json=payload + ) + + data = webfinger.get_resource('acct:test@test.webfinger') + + assert data['actor_url'] == 'https://test.webfinger/user/test' + assert data['subject'] == resource diff --git a/api/tests/music/test_import.py b/api/tests/music/test_import.py index 0f709e81f..2f22ed69a 100644 --- a/api/tests/music/test_import.py +++ b/api/tests/music/test_import.py @@ -1,7 +1,12 @@ import json +import pytest from django.urls import reverse +from funkwhale_api.federation import actors +from funkwhale_api.federation import serializers as federation_serializers +from funkwhale_api.music import tasks + def test_create_import_can_bind_to_request( artists, albums, mocker, factories, superuser_api_client): @@ -33,3 +38,196 @@ def test_create_import_can_bind_to_request( batch = request.import_batches.latest('id') assert batch.import_request == request + + +def test_import_job_from_federation_no_musicbrainz(factories): + lt = factories['federation.LibraryTrack']( + artist_name='Hello', + album_title='World', + title='Ping', + ) + job = factories['music.ImportJob']( + federation=True, + library_track=lt, + ) + + tasks.import_job_run(import_job_id=job.pk) + job.refresh_from_db() + + tf = job.track_file + assert tf.mimetype == lt.audio_mimetype + assert tf.library_track == job.library_track + assert tf.track.title == 'Ping' + assert tf.track.artist.name == 'Hello' + assert tf.track.album.title == 'World' + + +def test_import_job_from_federation_musicbrainz_recording(factories, mocker): + t = factories['music.Track']() + track_from_api = mocker.patch( + 'funkwhale_api.music.models.Track.get_or_create_from_api', + return_value=t) + lt = factories['federation.LibraryTrack']( + metadata__recording__musicbrainz=True, + artist_name='Hello', + album_title='World', + ) + job = factories['music.ImportJob']( + federation=True, + library_track=lt, + ) + + tasks.import_job_run(import_job_id=job.pk) + job.refresh_from_db() + + tf = job.track_file + assert tf.mimetype == lt.audio_mimetype + assert tf.library_track == job.library_track + assert tf.track == t + track_from_api.assert_called_once_with( + mbid=lt.metadata['recording']['musicbrainz_id']) + + +def test_import_job_from_federation_musicbrainz_release(factories, mocker): + a = factories['music.Album']() + album_from_api = mocker.patch( + 'funkwhale_api.music.models.Album.get_or_create_from_api', + return_value=a) + lt = factories['federation.LibraryTrack']( + metadata__release__musicbrainz=True, + artist_name='Hello', + title='Ping', + ) + job = factories['music.ImportJob']( + federation=True, + library_track=lt, + ) + + tasks.import_job_run(import_job_id=job.pk) + job.refresh_from_db() + + tf = job.track_file + assert tf.mimetype == lt.audio_mimetype + assert tf.library_track == job.library_track + assert tf.track.title == 'Ping' + assert tf.track.artist == a.artist + assert tf.track.album == a + + album_from_api.assert_called_once_with( + mbid=lt.metadata['release']['musicbrainz_id']) + + +def test_import_job_from_federation_musicbrainz_artist(factories, mocker): + a = factories['music.Artist']() + artist_from_api = mocker.patch( + 'funkwhale_api.music.models.Artist.get_or_create_from_api', + return_value=a) + lt = factories['federation.LibraryTrack']( + metadata__artist__musicbrainz=True, + album_title='World', + title='Ping', + ) + job = factories['music.ImportJob']( + federation=True, + library_track=lt, + ) + + tasks.import_job_run(import_job_id=job.pk) + job.refresh_from_db() + + tf = job.track_file + assert tf.mimetype == lt.audio_mimetype + assert tf.library_track == job.library_track + + assert tf.track.title == 'Ping' + assert tf.track.artist == a + assert tf.track.album.artist == a + assert tf.track.album.title == 'World' + + artist_from_api.assert_called_once_with( + mbid=lt.metadata['artist']['musicbrainz_id']) + + +def test_import_job_run_triggers_notifies_followers( + factories, mocker, tmpfile): + mocker.patch( + 'funkwhale_api.downloader.download', + return_value={'audio_file_path': tmpfile.name}) + mocked_notify = mocker.patch( + 'funkwhale_api.music.tasks.import_batch_notify_followers.delay') + batch = factories['music.ImportBatch']() + job = factories['music.ImportJob']( + finished=True, batch=batch) + track = factories['music.Track'](mbid=job.mbid) + + batch.update_status() + batch.refresh_from_db() + + assert batch.status == 'finished' + + mocked_notify.assert_called_once_with(import_batch_id=batch.pk) + + +def test_import_batch_notifies_followers_skip_on_disabled_federation( + settings, factories, mocker): + mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver') + batch = factories['music.ImportBatch'](finished=True) + settings.FEDERATION_ENABLED = False + tasks.import_batch_notify_followers(import_batch_id=batch.pk) + + mocked_deliver.assert_not_called() + + +def test_import_batch_notifies_followers_skip_on_federation_import( + factories, mocker): + mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver') + batch = factories['music.ImportBatch'](finished=True, federation=True) + tasks.import_batch_notify_followers(import_batch_id=batch.pk) + + mocked_deliver.assert_not_called() + + +def test_import_batch_notifies_followers( + factories, mocker): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + + f1 = factories['federation.Follow'](approved=True, target=library_actor) + f2 = factories['federation.Follow'](approved=False, target=library_actor) + f3 = factories['federation.Follow']() + + mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver') + batch = factories['music.ImportBatch']() + job1 = factories['music.ImportJob']( + finished=True, batch=batch) + job2 = factories['music.ImportJob']( + finished=True, federation=True, batch=batch) + job3 = factories['music.ImportJob']( + status='pending', batch=batch) + + batch.status = 'finished' + batch.save() + tasks.import_batch_notify_followers(import_batch_id=batch.pk) + + # only f1 match the requirements to be notified + # and only job1 is a non federated track with finished import + expected = { + '@context': federation_serializers.AP_CONTEXT, + 'actor': library_actor.url, + 'type': 'Create', + 'id': batch.get_federation_url(), + 'to': [f1.actor.url], + 'object': federation_serializers.CollectionSerializer( + { + 'id': batch.get_federation_url(), + 'items': [job1.track_file], + 'actor': library_actor, + 'item_serializer': federation_serializers.AudioSerializer + } + ).data + } + + mocked_deliver.assert_called_once_with( + expected, + on_behalf_of=library_actor, + to=[f1.actor.url] + ) diff --git a/api/tests/music/test_permissions.py b/api/tests/music/test_permissions.py new file mode 100644 index 000000000..d36f37886 --- /dev/null +++ b/api/tests/music/test_permissions.py @@ -0,0 +1,71 @@ +from rest_framework.views import APIView + +from funkwhale_api.federation import actors +from funkwhale_api.music import permissions + + +def test_list_permission_no_protect(anonymous_user, api_request, settings): + settings.PROTECT_AUDIO_FILES = False + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + assert permission.has_permission(request, view) is True + + +def test_list_permission_protect_anonymous( + anonymous_user, api_request, settings): + settings.PROTECT_AUDIO_FILES = True + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + assert permission.has_permission(request, view) is False + + +def test_list_permission_protect_authenticated( + factories, api_request, settings): + settings.PROTECT_AUDIO_FILES = True + user = factories['users.User']() + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + setattr(request, 'user', user) + assert permission.has_permission(request, view) is True + + +def test_list_permission_protect_not_following_actor( + factories, api_request, settings): + settings.PROTECT_AUDIO_FILES = True + actor = factories['federation.Actor']() + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + setattr(request, 'actor', actor) + assert permission.has_permission(request, view) is False + + +def test_list_permission_protect_following_actor( + factories, api_request, settings): + settings.PROTECT_AUDIO_FILES = True + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow = factories['federation.Follow']( + approved=True, target=library_actor) + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + setattr(request, 'actor', follow.actor) + + assert permission.has_permission(request, view) is True + + +def test_list_permission_protect_following_actor_not_approved( + factories, api_request, settings): + settings.PROTECT_AUDIO_FILES = True + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow = factories['federation.Follow']( + approved=False, target=library_actor) + view = APIView.as_view() + permission = permissions.Listen() + request = api_request.get('/') + setattr(request, 'actor', follow.actor) + + assert permission.has_permission(request, view) is False diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py index 295604616..81f34fbe9 100644 --- a/api/tests/music/test_views.py +++ b/api/tests/music/test_views.py @@ -1,6 +1,10 @@ +import io import pytest +from django.urls import reverse + from funkwhale_api.music import views +from funkwhale_api.federation import actors @pytest.mark.parametrize('param,expected', [ @@ -43,3 +47,65 @@ def test_album_view_filter_listenable( queryset = view.filter_queryset(view.get_queryset()) assert list(queryset) == expected + + +def test_can_serve_track_file_as_remote_library( + factories, authenticated_actor, settings, api_client): + settings.PROTECT_AUDIO_FILES = True + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + follow = factories['federation.Follow']( + approved=True, + actor=authenticated_actor, + target=library_actor) + + track_file = factories['music.TrackFile']() + response = api_client.get(track_file.path) + + assert response.status_code == 200 + assert response['X-Accel-Redirect'] == "{}{}".format( + settings.PROTECT_FILES_PATH, + track_file.audio_file.url) + + +def test_can_serve_track_file_as_remote_library_deny_not_following( + factories, authenticated_actor, settings, api_client): + settings.PROTECT_AUDIO_FILES = True + track_file = factories['music.TrackFile']() + response = api_client.get(track_file.path) + + assert response.status_code == 403 + + +def test_can_proxy_remote_track( + factories, settings, api_client, r_mock): + settings.PROTECT_AUDIO_FILES = False + track_file = factories['music.TrackFile'](federation=True) + + r_mock.get(track_file.library_track.audio_url, body=io.BytesIO(b'test')) + response = api_client.get(track_file.path) + + library_track = track_file.library_track + library_track.refresh_from_db() + assert response.status_code == 200 + assert response['X-Accel-Redirect'] == "{}{}".format( + settings.PROTECT_FILES_PATH, + library_track.audio_file.url) + assert library_track.audio_file.read() == b'test' + + +def test_can_create_import_from_federation_tracks( + factories, superuser_api_client, mocker): + lts = factories['federation.LibraryTrack'].create_batch(size=5) + mocker.patch('funkwhale_api.music.tasks.import_job_run') + + payload = { + 'library_tracks': [l.pk for l in lts] + } + url = reverse('api:v1:submit-federation') + response = superuser_api_client.post(url, payload) + + assert response.status_code == 201 + batch = superuser_api_client.user.imports.latest('id') + assert batch.jobs.count() == 5 + for i, job in enumerate(batch.jobs.all()): + assert job.library_track == lts[i] diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 69d5e1721..cc4f357ca 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -53,3 +53,19 @@ services: links: - postgres - redis + + # If you want to have the nginx proxy managed by docker for some reason + # (i.e. if you use apache as a proxy on your host), + # you can uncomment the following lines. + # nginx: + # image: nginx + # links: + # - api + # volumes: + # - ./nginx.conf:/etc/nginx/conf.d/funkwhale.conf:ro + # - ./funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro + # - ./data/media:/srv/funkwhale/data/media:ro + # - ./front/dist:/srv/funkwhale/front/dist:ro + # - ./data/static:/srv/funkwhale/data/static/:ro + # ports: + # - "127.0.0.1:5001:80" diff --git a/deploy/env.prod.sample b/deploy/env.prod.sample index a016b34c7..9e9938500 100644 --- a/deploy/env.prod.sample +++ b/deploy/env.prod.sample @@ -85,3 +85,12 @@ API_AUTHENTICATION_REQUIRED=True # This will help us detect and correct bugs RAVEN_ENABLED=false RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5 + +# This settings enable/disable federation on the instance level +FEDERATION_ENABLED=True +# This setting decide wether music library is shared automatically +# to followers or if it requires manual approval before. +# FEDERATION_MUSIC_NEEDS_APPROVAL=False +# means anyone can subscribe to your library and import your file, +# use with caution. +FEDERATION_MUSIC_NEEDS_APPROVAL=True diff --git a/deploy/funkwhale-beat.service b/deploy/funkwhale-beat.service new file mode 100644 index 000000000..209896dd4 --- /dev/null +++ b/deploy/funkwhale-beat.service @@ -0,0 +1,14 @@ +[Unit] +Description=Funkwhale celery beat process +After=redis.service postgresql.service +PartOf=funkwhale.target + +[Service] +User=funkwhale +# adapt this depending on the path of your funkwhale installation +WorkingDirectory=/srv/funkwhale/api +EnvironmentFile=/srv/funkwhale/config/.env +ExecStart=/srv/funkwhale/virtualenv/bin/celery -A funkwhale_api.taskapp beat -l INFO + +[Install] +WantedBy=multi-user.target diff --git a/deploy/funkwhale.target b/deploy/funkwhale.target index a920c7e34..a09e381c4 100644 --- a/deploy/funkwhale.target +++ b/deploy/funkwhale.target @@ -1,3 +1,3 @@ [Unit] Description=Funkwhale -Wants=funkwhale-server.service funkwhale-worker.service +Wants=funkwhale-server.service funkwhale-worker.service funkwhale-beat.service diff --git a/dev.yml b/dev.yml index c0470a2ab..2df7b44e6 100644 --- a/dev.yml +++ b/dev.yml @@ -10,21 +10,40 @@ services: - "HOST=0.0.0.0" - "WEBPACK_DEVSERVER_PORT=${WEBPACK_DEVSERVER_PORT-8080}" ports: - - "${WEBPACK_DEVSERVER_PORT-8080}:${WEBPACK_DEVSERVER_PORT-8080}" + - "${WEBPACK_DEVSERVER_PORT_BINDING-8080:}${WEBPACK_DEVSERVER_PORT-8080}" volumes: - './front:/app' + - '/app/node_modules' + - './po:/po' + networks: + - federation + - internal + labels: + traefik.backend: "${COMPOSE_PROJECT_NAME-node1}" + traefik.frontend.rule: "Host: ${COMPOSE_PROJECT_NAME-node1}.funkwhale.test" + traefik.enable: 'true' + traefik.federation.protocol: 'http' + traefik.federation.port: "${WEBPACK_DEVSERVER_PORT-8080}" postgres: env_file: - .env.dev - .env image: postgres + volumes: + - "./data/${COMPOSE_PROJECT_NAME-node1}/postgres:/var/lib/postgresql/data" + networks: + - internal redis: env_file: - .env.dev - .env image: redis:3.0 + volumes: + - "./data/${COMPOSE_PROJECT_NAME-node1}/redis:/data" + networks: + - internal celeryworker: env_file: @@ -38,11 +57,17 @@ services: - redis command: celery -A funkwhale_api.taskapp worker -l debug environment: + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}" + - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test" + - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}" + - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL-http}" - "DATABASE_URL=postgresql://postgres@postgres/postgres" - "CACHE_URL=redis://redis:6379/0" volumes: - ./api:/app - ./data/music:/music + networks: + - internal api: env_file: - .env.dev @@ -55,12 +80,17 @@ services: - ./api:/app - ./data/music:/music environment: + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}" + - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test" + - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}" + - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL-http}" - "DATABASE_URL=postgresql://postgres@postgres/postgres" - "CACHE_URL=redis://redis:6379/0" links: - postgres - redis - + networks: + - internal nginx: command: /entrypoint.sh env_file: @@ -69,6 +99,8 @@ services: image: nginx environment: - "WEBPACK_DEVSERVER_PORT=${WEBPACK_DEVSERVER_PORT-8080}" + - "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME- }" + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}" links: - api - front @@ -78,8 +110,9 @@ services: - ./deploy/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf.template:ro - ./api/funkwhale_api/media:/protected/media ports: - - "0.0.0.0:6001:6001" - + - "6001" + networks: + - internal docs: build: docs command: python serve.py @@ -88,3 +121,9 @@ services: ports: - '35730:35730' - '8001:8001' + +networks: + internal: + federation: + external: + name: federation diff --git a/docker/nginx/entrypoint.sh b/docker/nginx/entrypoint.sh index 1819acf1c..14e072a7e 100755 --- a/docker/nginx/entrypoint.sh +++ b/docker/nginx/entrypoint.sh @@ -1,10 +1,17 @@ #!/bin/bash -eux -FIRST_HOST=$(echo ${DJANGO_ALLOWED_HOSTS} | cut -d, -f1) + +FORWARDED_PORT="$WEBPACK_DEVSERVER_PORT" +COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME// /}" +if [ -n "$COMPOSE_PROJECT_NAME" ]; then + echo + FUNKWHALE_HOSTNAME="$COMPOSE_PROJECT_NAME.funkwhale.test" + FORWARDED_PORT="443" +fi echo "Copying template file..." cp /etc/nginx/funkwhale_proxy.conf{.template,} -sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host ${FIRST_HOST}:${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf -sed -i "s/proxy_set_header Host \$host/proxy_set_header Host ${FIRST_HOST}/" /etc/nginx/funkwhale_proxy.conf -sed -i "s/proxy_set_header X-Forwarded-Port \$server_port/proxy_set_header X-Forwarded-Port ${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host ${FUNKWHALE_HOSTNAME}:${FORWARDED_PORT}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/proxy_set_header Host \$host/proxy_set_header Host ${FUNKWHALE_HOSTNAME}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/proxy_set_header X-Forwarded-Port \$server_port/proxy_set_header X-Forwarded-Port ${FORWARDED_PORT}/" /etc/nginx/funkwhale_proxy.conf cat /etc/nginx/funkwhale_proxy.conf nginx -g "daemon off;" diff --git a/docker/ssl/test.crt b/docker/ssl/test.crt new file mode 100644 index 000000000..e4d3eefb8 --- /dev/null +++ b/docker/ssl/test.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDljCCAn6gAwIBAgIJAOA/w9NwL3aMMA0GCSqGSIb3DQEBCwUAMGAxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQxGTAXBgNVBAMMECouZnVua3doYWxlLnRlc3QwHhcNMTgw +NDA4MTMwNDAzWhcNMjgwNDA1MTMwNDAzWjBgMQswCQYDVQQGEwJBVTETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRkwFwYDVQQDDBAqLmZ1bmt3aGFsZS50ZXN0MIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEAyGqRLEMFs1mpRwauTicIRj2zwBUe6JMNRbIvOUkaj2KY6avA +7tiNti/ygBoTyJl2JK3mmLqxElqedpMhjVvYde/PyjXoZ+0Vq4FWv89LV6ZM/Scf +TCIYwWF1ppi6GYFmU3WCIMISkKiPBtMArB0oZxiUWLmkyd8jih2wnQOpkQ20FfG0 +CtlrKlQKyAe7X3zPuqGfaMUN7J4w9g3/SC66YulbAtI1/Z4tuG8J4m2RC6jH1hVy +364l3ifEC+m9Kax/ystfu/mkLdyQgRfOZTNf2JhS3BL8zpoWMXFK+4+7TYisrV1h +0pzIAsoQeBB+cFOOFEwRAv0FxSWnZ+/shjnwbwIDAQABo1MwUTAdBgNVHQ4EFgQU +sULmofttRyWUMM93IsD8jBvyCd4wHwYDVR0jBBgwFoAUsULmofttRyWUMM93IsD8 +jBvyCd4wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAUg/fiXut +hW6fDx9f0JdB4uLiLnv8tDP35ackLLapFJhXtflIXcqCzxStQ46nMs1wjaZPb+ws +pLULzvTKTxJbu+JYc2nvis4m2oSFczJ3S9tgug4Ppv8yS7N1pp7kfjOvBjgh6sYW +p+Ctb5r8qvgvT9yDTeCnsqktb/OkRHlHwhRYfnuxh+96s4mzifqFUP4uCCcFYPTc +RE0Ag3oI5sHOdDk/cdYE5PGQPjSP6gzn0lsrz1Q3x1C8+txSHzsJnvS3Ost+dwcy +JSjDBXauy9cZv93Voevcl16Ioo7trtkp4dwAoep52vOT/KMkJ4zm19msV3BP4wMa +BUqrV2F7twD5zw== +-----END CERTIFICATE----- diff --git a/docker/ssl/test.key b/docker/ssl/test.key new file mode 100644 index 000000000..669e5f700 --- /dev/null +++ b/docker/ssl/test.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDIapEsQwWzWalH +Bq5OJwhGPbPAFR7okw1Fsi85SRqPYpjpq8Du2I22L/KAGhPImXYkreaYurESWp52 +kyGNW9h178/KNehn7RWrgVa/z0tXpkz9Jx9MIhjBYXWmmLoZgWZTdYIgwhKQqI8G +0wCsHShnGJRYuaTJ3yOKHbCdA6mRDbQV8bQK2WsqVArIB7tffM+6oZ9oxQ3snjD2 +Df9ILrpi6VsC0jX9ni24bwnibZELqMfWFXLfriXeJ8QL6b0prH/Ky1+7+aQt3JCB +F85lM1/YmFLcEvzOmhYxcUr7j7tNiKytXWHSnMgCyhB4EH5wU44UTBEC/QXFJadn +7+yGOfBvAgMBAAECggEBAMVB3lEqRloYTbxSnwzc7g/0ew77usg+tDl8/23qvfGS +od6b5fEvw4sl9hCPmhk+skG3x9dbKR1fg8hBWCzB0XOC7YmhNXXUrBd53eA8L3O9 +gtlHwE424Ra0zg+DEug3rHdImSOU4KDwxpV46Jh+ul1+m8QYNFFdBqXSQxrHmAXj +MQ6++rjoJ+bhucmjBouzMYXHTGhdae3kjDFrFJ4cUsH6F03NcDwS+AmZxa/DWQ/H +SoBQBeLoE6I1aKhLgY91yO1e7CtSzS2GFCODReN4b3cylaR7jE7Mg87TZcga6Wfa +Xcd120VVlVq6HmZc/Xob7aUim3AuY2er8bcvmg1XOsECgYEA5EMM5UlpLdNWv1hp +5IMvkeCbXtLJ3IOHO0xLkFdx0CxaR9TyAAqIrSh1t9rFhYqLUNiOdMc2TqrvdgEU +B/QZrAevWRc5sjPvFXmYeWSCi/tjRgQh4jClWDX/TlfAlP55z2BFyMPMX6//WbBQ +5aL9xymTymzFFcaE8EytT5Jz8rUCgYEA4MVF3IkaQepl6H1gf2T6ev+MtGk9AGg9 +DSJpio7hfMcY5X3NrTJJFF9DJFXqfo3ILOMyUpIUHqkCGKXil0n9ypLp4vq7l+6c +m1gtKFXh7uKAV4XtSnR0nuK/N10JJp2HbbFYGlziRaa1iEPAFvLDQHu4jyf5sXyV +HvreuQgGWRMCgYEAlUaQKWaP5UsfoPUGE04DjwfvM9zv7EkL6CimBhhZswU+aVmG +haZd6bfa/EiTAhkvsMheqVoaVuoMvgRIgEcPfuRrtPyuW68A/O9PWpvzj+3v5zsO +maisiPqPI0HaDNY6/PZ9zKTXhABKIvJehT7JbjTvlOL7JJl2GNxcPvyM3T0CgYEA +tnVtUKi69+ce8qtUOhXufwoTXiBPtJTpelAE/MUfpfq46xJEc+PuDuuFxWk5AaJ2 +bHnBz+VlD76CRR/j4IvfySGZWvfOcHbyCeh6P9P3o8OaC3JcPaRrRs8qCfcsBny6 +AwGDU2MzCvdZRVQ6CmbmuOG13//DYaCQLKXZRrqM7KECgYEAxDsqtyHA/a38UhS8 +iQ8HqrZp8CuzJoJw/QILvzjojD1cvmwF73RrPEpRfEaLWVQGQ5F1IlHk/009C5zy +eUT4ZaPxLem6khBf7pn3xXaVBGZsYoltek5sUBsu/jA+4Sw6bcUmhBRBCs98JGpR +DVJtvOTk9aGW8M8UbgqwW+e/6ng= +-----END PRIVATE KEY----- diff --git a/docker/traefik.toml b/docker/traefik.toml new file mode 100644 index 000000000..85da2ea72 --- /dev/null +++ b/docker/traefik.toml @@ -0,0 +1,26 @@ +defaultEntryPoints = ["http", "https"] + +################################################################ +# Web configuration backend +################################################################ +[web] +address = ":8040" +################################################################ +# Docker configuration backend +################################################################ +[docker] +domain = "funkwhale.test" +watch = true +exposedbydefault = false + +[entryPoints] + [entryPoints.http] + address = ":80" + [entryPoints.http.redirect] + entryPoint = "https" + [entryPoints.https] + address = ":443" + [entryPoints.https.tls] + [[entryPoints.https.tls.certificates]] + certFile = "/ssl/traefik.crt" + keyFile = "/ssl/traefik.key" diff --git a/docker/traefik.yml b/docker/traefik.yml new file mode 100644 index 000000000..0b15b3290 --- /dev/null +++ b/docker/traefik.yml @@ -0,0 +1,22 @@ +version: '2.1' + +services: + traefik: + image: traefik:alpine + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./traefik.toml:/traefik.toml + - ./ssl/test.key:/ssl/traefik.key + - ./ssl/test.crt:/ssl/traefik.crt + ports: + - '80:80' + - '443:443' + - '8040:8040' + networks: + federation: + + +networks: + federation: + external: + name: federation diff --git a/docs/features.rst b/docs/features.rst index d72a826b6..e0f490e4b 100644 --- a/docs/features.rst +++ b/docs/features.rst @@ -42,3 +42,14 @@ The project itself is splitted in two parts: While the main interface to the server and API is the bundled front-end, the project itself is agnostic in the way you connect to it. Therefore, desktop clients or apps could be developped and implement the same (or even more) features as the bundled frontend. This modularity also makes it possible do deploy only a single component from the system. + +Federation +---------- + +Each Funkwhale instance is able to fetch music from other compatible servers, +and share its own library on the network. The federation is implemented +using the ActivityPub protocol, in order to leverage existing tools +and be compatible with other services such as Mastodon. + +As of today, federation only targets music acquisition, meaning user interaction +are not shared via ActivityPub. This will be implemented at a later point. diff --git a/docs/federation.rst b/docs/federation.rst new file mode 100644 index 000000000..5b030074c --- /dev/null +++ b/docs/federation.rst @@ -0,0 +1,57 @@ +Federation +========== + +Each Funkwale instance can federates its music library with other instances +of the network. This means that an instance A can acquire music from instance B +and share its own library with an instance C. + +We support various levels of controls for federation-related features. + +Managing federation +------------------- + +Federation management is only available to instance admins and users +who have the proper permissions. You can disable federation completely +at the instance level by setting the FEDERATION_ENABLED environment variable +to False. + +On the front end, assuming you have the proper permission, you will see +a "Federation" link in the sidebar. + + +Acquire music via federation +---------------------------- + +Instance libraries are protected by default. To access another instance +library, you have to follow it. Each funkwhale instance gets a dedicated +ActivityPub Actor you can follow via the username "library@yourinstance.domain". + +When submitted, a follow request will be sent to +the other instance which can accept or deny it. Once your follow request +is accepted, you can start browsing the other instance library +and import music from it. + +By default, we do not duplicate audio files from federated tracks, to reduce +disk usage on your instance. When someone listens to a federated track, +the audio file is requested on the fly from the remote instance, and +store in a local cache. It is automatically deleted after a configurable +amount of time if it was not listened again in the meantime. + +If you want to mirror a remote instance collection, including its audio files, +we offer an option for that. + +We also support an "autoimport" mode for each remote library. When enabled, +any new track published in the remote library will be directly imported +in your instance. + +Share music via federation +-------------------------- + +Federation is enabled by default, but requires manually approving +each other instance asking for access to library. This is by design, +to ensure your library is not shared publicly without your consent. + +However, if you're confident about federating publicly without manual approval, +you can set the FEDERATION_MUSIC_NEEDS_APPROVAL environment variable to false. +Follow requests will be accepted automatically and followers +given access to your library without manual intervention. diff --git a/docs/importing-music.rst b/docs/importing-music.rst index 92ff6bfb5..f09eea7b1 100644 --- a/docs/importing-music.rst +++ b/docs/importing-music.rst @@ -67,3 +67,9 @@ under creative commons (courtesy of Jamendo): ./download-tracks.sh music.txt This will download a bunch of zip archives (one per album) under the ``data/music`` directory and unzip their content. + +From other instances +-------------------- + +Funkwhale also supports importing music from other instances. Please refer +to :doc:`federation` for more details. diff --git a/docs/index.rst b/docs/index.rst index f5acecce5..82dcf8c88 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,6 +15,7 @@ Funkwhale is a self-hosted, modern free and open-source music server, heavily in installation/index configuration importing-music + federation upgrading changelog diff --git a/docs/installation/systemd.rst b/docs/installation/systemd.rst index 67af98432..27781c447 100644 --- a/docs/installation/systemd.rst +++ b/docs/installation/systemd.rst @@ -13,11 +13,13 @@ First, download the sample unitfiles: curl -L -o "/etc/systemd/system/funkwhale.target" "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale.target" curl -L -o "/etc/systemd/system/funkwhale-server.service" "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale-server.service" curl -L -o "/etc/systemd/system/funkwhale-worker.service" "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale-worker.service" + curl -L -o "/etc/systemd/system/funkwhale-beat.service" "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale-beat.service" This will download three unitfiles: - ``funkwhale-server.service`` to launch the funkwhale web server - ``funkwhale-worker.service`` to launch the funkwhale task worker +- ``funkwhale-beat.service`` to launch the funkwhale task beat (this is for recurring tasks) - ``funkwhale.target`` to easily stop and start all of the services at once You can of course review and edit them to suit your deployment scenario diff --git a/front/.dockerignore b/front/.dockerignore new file mode 100644 index 000000000..3c3629e64 --- /dev/null +++ b/front/.dockerignore @@ -0,0 +1 @@ +node_modules diff --git a/front/Dockerfile b/front/Dockerfile index 60b03c9b8..3d4c65e64 100644 --- a/front/Dockerfile +++ b/front/Dockerfile @@ -4,7 +4,7 @@ EXPOSE 8080 WORKDIR /app/ ADD package.json . RUN yarn install -VOLUME ["/app/node_modules"] + COPY . . CMD ["npm", "run", "dev"] diff --git a/front/build/dev-server.js b/front/build/dev-server.js index 634a6d41e..f9c389e72 100644 --- a/front/build/dev-server.js +++ b/front/build/dev-server.js @@ -14,6 +14,8 @@ var webpackConfig = process.env.NODE_ENV === 'testing' ? require('./webpack.prod.conf') : require('./webpack.dev.conf') +require('./i18n') + // default port where dev server listens for incoming traffic var port = process.env.PORT || config.dev.port var host = process.env.HOST || config.dev.host diff --git a/front/build/i18n.js b/front/build/i18n.js new file mode 100644 index 000000000..ef31070c7 --- /dev/null +++ b/front/build/i18n.js @@ -0,0 +1,49 @@ +const fs = require('fs'); +const path = require('path'); +const { gettextToI18next } = require('i18next-conv'); + +const poDir = path.join(__dirname, '..', '..', 'po') +const outDir = path.join(__dirname, '..', 'static', 'translations') +if (!fs.existsSync(outDir) || !fs.statSync(outDir).isDirectory()) { + fs.mkdirSync(outDir) +} + +// Convert .po files to i18next files +fs.readdir(poDir, (err, files) => { + if (err) { + return console.log(err) + } + + for (const file of files) { + if (file.endsWith('.po')) { + const lang = file.replace(/\.po$/, '') + const output = path.join(outDir, `${lang}.json`) + fs.readFile(path.join(poDir, file), (err, content) => { + if (err) { + return console.log(err) + } + + gettextToI18next(lang, content).then(res => { + fs.writeFile(output, res, err => { + if (err) { + console.log(err) + } else { + console.log(`Wrote translation file: ${output}`) + if (lang === 'en') { + // for english, we need to specify that json values are equal to the keys. + // otherwise we end up with empty strings on the front end for english + var contents = fs.readFileSync(output) + var jsonContent = JSON.parse(contents) + var finalContent = {} + Object.keys(jsonContent).forEach(function(key) { + finalContent[key] = key + }) + fs.writeFile(output, JSON.stringify(finalContent)) + } + } + }) + }) + }) + } + } +}) diff --git a/front/package.json b/front/package.json index d67375f7e..8844e8bee 100644 --- a/front/package.json +++ b/front/package.json @@ -8,6 +8,8 @@ "dev": "node build/dev-server.js", "start": "node build/dev-server.js", "build": "node build/build.js", + "i18n-extract": "find src/ -name '*.vue' | xargs vendor/vue-i18n-xgettext/index.js > ../po/en.po", + "i18n-compile": "node build/i18n.js", "unit": "cross-env BABEL_ENV=test karma start test/unit/karma.conf.js --single-run", "unit-watch": "cross-env BABEL_ENV=test karma start test/unit/karma.conf.js", "e2e": "node test/e2e/runner.js", @@ -15,9 +17,13 @@ "lint": "eslint --ext .js,.vue src test/unit/specs test/e2e/specs" }, "dependencies": { + "@panter/vue-i18next": "^0.9.1", "axios": "^0.17.1", "dateformat": "^2.0.0", "django-channels": "^1.1.6", + "i18next": "^11.1.1", + "i18next-conv": "^6.0.0", + "i18next-fetch-backend": "^0.1.0", "js-logger": "^1.3.0", "jwt-decode": "^2.2.0", "lodash": "^4.17.4", @@ -34,7 +40,7 @@ "vue-upload-component": "^2.7.4", "vuedraggable": "^2.14.1", "vuex": "^3.0.1", - "vuex-persistedstate": "^2.4.2", + "vuex-persistedstate": "^2.5.2", "vuex-router-sync": "^5.0.0" }, "devDependencies": { @@ -98,6 +104,7 @@ "sinon-chai": "^2.8.0", "sinon-stub-promise": "^4.0.0", "url-loader": "^0.5.8", + "vue-i18n-xgettext": "^0.0.4", "vue-loader": "^12.1.0", "vue-style-loader": "^3.0.1", "vue-template-compiler": "^2.3.3", diff --git a/front/src/App.vue b/front/src/App.vue index e8ab18694..e8cac7476 100644 --- a/front/src/App.vue +++ b/front/src/App.vue @@ -7,21 +7,25 @@
-

Links

+
- About this instance + - Official website - Documentation - Source code - Issue tracker + + + +
-

About funkwhale

-

Funkwhale is a free and open-source project run by volunteers. You can help us improve the platform by reporting bugs, suggesting features and share the project with your friends!

-

The funkwhale logo was kindly designed and provided by Francis Gading.

+ +

+ +

+

+ +

@@ -31,7 +35,6 @@ :dsn="$store.state.instance.settings.raven.front_dsn.value"> - diff --git a/front/src/components/Home.vue b/front/src/components/Home.vue index 0e24dcd59..ce1307ff0 100644 --- a/front/src/components/Home.vue +++ b/front/src/components/Home.vue @@ -3,7 +3,7 @@

- Welcome on Funkwhale + Welcome on Funkwhale

We think listening music should be simple.

diff --git a/front/src/components/Pagination.vue b/front/src/components/Pagination.vue index 47cf5183a..ca40cdd07 100644 --- a/front/src/components/Pagination.vue +++ b/front/src/components/Pagination.vue @@ -1,25 +1,25 @@