From d4028450a95ceb407d2378ae7c784bc18673f2d3 Mon Sep 17 00:00:00 2001 From: Agate Date: Fri, 3 Jul 2020 10:59:12 +0200 Subject: [PATCH] Plugins infrastructure --- api/config/api_urls.py | 1 + api/config/plugins.py | 291 ++++++++++++ api/config/settings/common.py | 65 +-- api/config/urls.py | 5 +- api/funkwhale_api/cli/main.py | 1 + api/funkwhale_api/cli/plugins.py | 35 ++ api/funkwhale_api/common/apps.py | 4 + api/funkwhale_api/common/factories.py | 9 + api/funkwhale_api/common/middleware.py | 20 + .../migrations/0008_auto_20200701_1317.py | 37 ++ api/funkwhale_api/common/models.py | 21 + api/funkwhale_api/common/views.py | 101 +++++ .../contrib/scrobbler/__init__.py | 0 .../contrib/scrobbler/funkwhale_ready.py | 39 ++ .../contrib/scrobbler/funkwhale_startup.py | 27 ++ .../contrib/scrobbler/scrobbler.py | 98 ++++ api/funkwhale_api/history/views.py | 7 + .../music/management/commands/import_files.py | 8 +- api/funkwhale_api/users/models.py | 4 +- api/funkwhale_api/users/oauth/scopes.py | 8 +- api/setup.cfg | 3 +- api/tests/manage/test_views.py | 16 +- api/tests/plugins/test_plugins.py | 424 ++++++++++++++++++ api/tests/users/oauth/test_scopes.py | 9 + docs/conf.py | 9 +- docs/developers/index.rst | 1 + docs/developers/plugins.rst | 165 +++++++ front/scripts/fix-fomantic-css.py | 5 +- front/src/components/auth/Plugin.vue | 113 +++++ front/src/components/auth/Settings.vue | 14 + front/src/router/index.js | 8 + front/src/views/auth/Plugins.vue | 59 +++ 32 files changed, 1560 insertions(+), 47 deletions(-) create mode 100644 api/config/plugins.py create mode 100644 api/funkwhale_api/cli/plugins.py create mode 100644 api/funkwhale_api/common/migrations/0008_auto_20200701_1317.py create mode 100644 api/funkwhale_api/contrib/scrobbler/__init__.py create mode 100644 api/funkwhale_api/contrib/scrobbler/funkwhale_ready.py create mode 100644 api/funkwhale_api/contrib/scrobbler/funkwhale_startup.py create mode 100644 api/funkwhale_api/contrib/scrobbler/scrobbler.py create mode 100644 api/tests/plugins/test_plugins.py create mode 100644 docs/developers/plugins.rst create mode 100644 front/src/components/auth/Plugin.vue create mode 100644 front/src/views/auth/Plugins.vue diff --git a/api/config/api_urls.py b/api/config/api_urls.py index 74c5e248d..e67e432f6 100644 --- a/api/config/api_urls.py +++ b/api/config/api_urls.py @@ -17,6 +17,7 @@ router = common_routers.OptionalSlashRouter() router.register(r"settings", GlobalPreferencesViewSet, basename="settings") router.register(r"activity", activity_views.ActivityViewSet, "activity") router.register(r"tags", tags_views.TagViewSet, "tags") +router.register(r"plugins", common_views.PluginViewSet, "plugins") router.register(r"tracks", views.TrackViewSet, "tracks") router.register(r"uploads", views.UploadViewSet, "uploads") router.register(r"libraries", views.LibraryViewSet, "libraries") diff --git a/api/config/plugins.py b/api/config/plugins.py new file mode 100644 index 000000000..f390c4366 --- /dev/null +++ b/api/config/plugins.py @@ -0,0 +1,291 @@ +import copy +import logging +import os +import subprocess +import sys + +import persisting_theory +from django.db.models import Q + +from rest_framework import serializers + +logger = logging.getLogger("plugins") + + +class Startup(persisting_theory.Registry): + look_into = "persisting_theory" + + +class Ready(persisting_theory.Registry): + look_into = "persisting_theory" + + +startup = Startup() +ready = Ready() + +_plugins = {} +_filters = {} +_hooks = {} + + +def get_plugin_config( + name, + user=False, + source=False, + registry=_plugins, + conf={}, + description=None, + version=None, + label=None, +): + conf = { + "name": name, + "label": label or name, + "logger": logger, + "conf": conf, + "user": True if source else user, + "source": source, + "description": description, + "version": version, + } + registry[name] = conf + return conf + + +def get_session(): + from funkwhale_api.common import session + + return session.get_session() + + +def register_filter(name, plugin_config, registry=_filters): + def decorator(func): + handlers = registry.setdefault(name, []) + + def inner(*args, **kwargs): + plugin_config["logger"].debug("Calling filter for %s", name) + rval = func(*args, **kwargs) + return rval + + handlers.append((plugin_config["name"], inner)) + return inner + + return decorator + + +def register_hook(name, plugin_config, registry=_hooks): + def decorator(func): + handlers = registry.setdefault(name, []) + + def inner(*args, **kwargs): + plugin_config["logger"].debug("Calling hook for %s", name) + func(*args, **kwargs) + + handlers.append((plugin_config["name"], inner)) + return inner + + return decorator + + +class Skip(Exception): + pass + + +def trigger_filter(name, value, enabled=False, **kwargs): + """ + Call filters registered for "name" with the given + args and kwargs. + + Return the value (that could be modified by handlers) + """ + logger.debug("Calling handlers for filter %s", name) + registry = kwargs.pop("registry", _filters) + confs = kwargs.pop("confs", {}) + for plugin_name, handler in registry.get(name, []): + if not enabled and confs.get(plugin_name, {}).get("enabled") is False: + continue + try: + value = handler(value, conf=confs.get(plugin_name, {}), **kwargs) + except Skip: + pass + except Exception as e: + logger.warn("Plugin %s errored during filter %s: %s", plugin_name, name, e) + return value + + +def trigger_hook(name, enabled=False, **kwargs): + """ + Call hooks registered for "name" with the given + args and kwargs. + + Returns nothing + """ + logger.debug("Calling handlers for hook %s", name) + registry = kwargs.pop("registry", _hooks) + confs = kwargs.pop("confs", {}) + for plugin_name, handler in registry.get(name, []): + if not enabled and confs.get(plugin_name, {}).get("enabled") is False: + continue + try: + handler(conf=confs.get(plugin_name, {}).get("conf"), **kwargs) + except Skip: + pass + except Exception as e: + logger.warn("Plugin %s errored during hook %s: %s", plugin_name, name, e) + + +def set_conf(name, conf, user=None, registry=_plugins): + from funkwhale_api.common import models + + if not registry[name]["conf"] and not registry[name]["source"]: + return + conf_serializer = get_serializer_from_conf_template( + registry[name]["conf"], user=user, source=registry[name]["source"], + )(data=conf) + conf_serializer.is_valid(raise_exception=True) + if "library" in conf_serializer.validated_data: + conf_serializer.validated_data["library"] = str( + conf_serializer.validated_data["library"] + ) + conf, _ = models.PluginConfiguration.objects.update_or_create( + user=user, code=name, defaults={"conf": conf_serializer.validated_data} + ) + + +def get_confs(user=None): + from funkwhale_api.common import models + + qs = models.PluginConfiguration.objects.filter(code__in=list(_plugins.keys())) + if user: + qs = qs.filter(Q(user=None) | Q(user=user)) + else: + qs = qs.filter(user=None) + confs = { + v["code"]: {"conf": v["conf"], "enabled": v["enabled"]} + for v in qs.values("code", "conf", "enabled") + } + for p, v in _plugins.items(): + if p not in confs: + confs[p] = {"conf": None, "enabled": False} + return confs + + +def get_conf(plugin, user=None): + return get_confs(user=user)[plugin] + + +def enable_conf(code, value, user): + from funkwhale_api.common import models + + models.PluginConfiguration.objects.update_or_create( + code=code, user=user, defaults={"enabled": value} + ) + + +class LibraryField(serializers.UUIDField): + def __init__(self, *args, **kwargs): + self.actor = kwargs.pop("actor") + super().__init__(*args, **kwargs) + + def to_internal_value(self, v): + v = super().to_internal_value(v) + if not self.actor.libraries.filter(uuid=v).first(): + raise serializers.ValidationError("Invalid library id") + return v + + +def get_serializer_from_conf_template(conf, source=False, user=None): + conf = copy.deepcopy(conf) + validators = {f["name"]: f.pop("validator") for f in conf if "validator" in f} + mapping = { + "url": serializers.URLField, + "boolean": serializers.BooleanField, + "text": serializers.CharField, + "long_text": serializers.CharField, + "password": serializers.CharField, + "number": serializers.IntegerField, + } + + for attr in ["label", "help"]: + for c in conf: + c.pop(attr, None) + + class Serializer(serializers.Serializer): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + for field_conf in conf: + field_kwargs = copy.copy(field_conf) + name = field_kwargs.pop("name") + self.fields[name] = mapping[field_kwargs.pop("type")](**field_kwargs) + if source: + self.fields["library"] = LibraryField(actor=user.actor) + + for vname, v in validators.items(): + setattr(Serializer, "validate_{}".format(vname), v) + return Serializer + + +def serialize_plugin(plugin_conf, confs): + return { + "name": plugin_conf["name"], + "label": plugin_conf["label"], + "description": plugin_conf.get("description") or None, + "user": plugin_conf.get("user", False), + "source": plugin_conf.get("source", False), + "conf": plugin_conf.get("conf", None), + "values": confs.get(plugin_conf["name"], {"conf"}).get("conf"), + "enabled": plugin_conf["name"] in confs + and confs[plugin_conf["name"]]["enabled"], + } + + +def install_dependencies(deps): + if not deps: + return + logger.info("Installing plugins dependencies %s", deps) + pip_path = os.path.join(os.path.dirname(sys.executable), "pip") + subprocess.check_call([pip_path, "install"] + deps) + + +def background_task(name): + from funkwhale_api.taskapp import celery + + def decorator(func): + return celery.app.task(func, name=name) + + return decorator + + +# HOOKS +LISTENING_CREATED = "listening_created" +""" +Called when a track is being listened +""" +SCAN = "scan" +""" + +""" +# FILTERS +PLUGINS_DEPENDENCIES = "plugins_dependencies" +""" +Called with an empty list, use this filter to append pip dependencies +to the list for installation. +""" +PLUGINS_APPS = "plugins_apps" +""" +Called with an empty list, use this filter to append apps to INSTALLED_APPS +""" +MIDDLEWARES_BEFORE = "middlewares_before" +""" +Called with an empty list, use this filter to prepend middlewares +to MIDDLEWARE +""" +MIDDLEWARES_AFTER = "middlewares_after" +""" +Called with an empty list, use this filter to append middlewares +to MIDDLEWARE +""" +URLS = "urls" +""" +Called with an empty list, use this filter to register new urls and views +""" diff --git a/api/config/settings/common.py b/api/config/settings/common.py index ad24d43db..8dc16537e 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -46,6 +46,12 @@ logging.config.dictConfig( # required to avoid double logging with root logger "propagate": False, }, + "plugins": { + "level": LOGLEVEL, + "handlers": ["console"], + # required to avoid double logging with root logger + "propagate": False, + }, "": {"level": "WARNING", "handlers": ["console"]}, }, } @@ -87,6 +93,20 @@ Path to a directory containing Funkwhale plugins. These will be imported at runt """ sys.path.append(FUNKWHALE_PLUGINS_PATH) +PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p] +""" +List of Funkwhale plugins to load. +""" +if PLUGINS: + logger.info("Running with the following plugins enabled: %s", ", ".join(PLUGINS)) +else: + logger.info("Running with no plugins") + +from .. import plugins # noqa + +plugins.startup.autodiscover([p + ".funkwhale_startup" for p in PLUGINS]) +DEPENDENCIES = plugins.trigger_filter(plugins.PLUGINS_DEPENDENCIES, [], enabled=True) +plugins.install_dependencies(DEPENDENCIES) FUNKWHALE_HOSTNAME = None FUNKWHALE_HOSTNAME_SUFFIX = env("FUNKWHALE_HOSTNAME_SUFFIX", default=None) FUNKWHALE_HOSTNAME_PREFIX = env("FUNKWHALE_HOSTNAME_PREFIX", default=None) @@ -247,16 +267,6 @@ LOCAL_APPS = ( # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps - -PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p] -""" -List of Funkwhale plugins to load. -""" -if PLUGINS: - logger.info("Running with the following plugins enabled: %s", ", ".join(PLUGINS)) -else: - logger.info("Running with no plugins") - ADDITIONAL_APPS = env.list("ADDITIONAL_APPS", default=[]) """ List of Django apps to load in addition to Funkwhale plugins and apps. @@ -265,27 +275,32 @@ INSTALLED_APPS = ( DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS - + tuple(["{}.apps.Plugin".format(p) for p in PLUGINS]) + tuple(ADDITIONAL_APPS) + + tuple(plugins.trigger_filter(plugins.PLUGINS_APPS, [], enabled=True)) ) # MIDDLEWARE CONFIGURATION # ------------------------------------------------------------------------------ ADDITIONAL_MIDDLEWARES_BEFORE = env.list("ADDITIONAL_MIDDLEWARES_BEFORE", default=[]) -MIDDLEWARE = tuple(ADDITIONAL_MIDDLEWARES_BEFORE) + ( - "django.middleware.security.SecurityMiddleware", - "django.middleware.clickjacking.XFrameOptionsMiddleware", - "corsheaders.middleware.CorsMiddleware", - # needs to be before SPA middleware - "django.contrib.sessions.middleware.SessionMiddleware", - "django.middleware.common.CommonMiddleware", - "django.middleware.csrf.CsrfViewMiddleware", - # /end - "funkwhale_api.common.middleware.SPAFallbackMiddleware", - "django.contrib.auth.middleware.AuthenticationMiddleware", - "django.contrib.messages.middleware.MessageMiddleware", - "funkwhale_api.users.middleware.RecordActivityMiddleware", - "funkwhale_api.common.middleware.ThrottleStatusMiddleware", +MIDDLEWARE = ( + tuple(plugins.trigger_filter(plugins.MIDDLEWARES_BEFORE, [], enabled=True)) + + tuple(ADDITIONAL_MIDDLEWARES_BEFORE) + + ( + "django.middleware.security.SecurityMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "corsheaders.middleware.CorsMiddleware", + # needs to be before SPA middleware + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + # /end + "funkwhale_api.common.middleware.SPAFallbackMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "funkwhale_api.users.middleware.RecordActivityMiddleware", + "funkwhale_api.common.middleware.ThrottleStatusMiddleware", + ) + + tuple(plugins.trigger_filter(plugins.MIDDLEWARES_AFTER, [], enabled=True)) ) # DEBUG diff --git a/api/config/urls.py b/api/config/urls.py index 2cd4f4662..3af9bd87d 100644 --- a/api/config/urls.py +++ b/api/config/urls.py @@ -8,7 +8,9 @@ from django.conf.urls.static import static from funkwhale_api.common import admin from django.views import defaults as default_views +from config import plugins +plugins_patterns = plugins.trigger_filter(plugins.URLS, [], enabled=True) urlpatterns = [ # Django Admin, use {% url 'admin:index' %} url(settings.ADMIN_URL, admin.site.urls), @@ -21,8 +23,7 @@ urlpatterns = [ ), url(r"^api/v1/auth/", include("funkwhale_api.users.rest_auth_urls")), url(r"^accounts/", include("allauth.urls")), - # Your stuff: custom urls includes go here -] +] + plugins_patterns if settings.DEBUG: # This allows the error pages to be debugged during development, just visit diff --git a/api/funkwhale_api/cli/main.py b/api/funkwhale_api/cli/main.py index 1453ca5d2..db7e87a22 100644 --- a/api/funkwhale_api/cli/main.py +++ b/api/funkwhale_api/cli/main.py @@ -4,6 +4,7 @@ import sys from . import base from . import library # noqa from . import media # noqa +from . import plugins # noqa from . import users # noqa from rest_framework.exceptions import ValidationError diff --git a/api/funkwhale_api/cli/plugins.py b/api/funkwhale_api/cli/plugins.py new file mode 100644 index 000000000..9ab24dccf --- /dev/null +++ b/api/funkwhale_api/cli/plugins.py @@ -0,0 +1,35 @@ +import os +import subprocess +import sys + +import click +from django.conf import settings + + +from . import base + + +@base.cli.group() +def plugins(): + """Manage plugins""" + pass + + +@plugins.command("install") +@click.argument("plugin", nargs=-1) +def install(plugin): + """ + Install a plugin from a given URL (zip, pip or git are supported) + """ + if not plugin: + return click.echo("No plugin provided") + + click.echo("Installing plugins…") + pip_install(list(plugin), settings.FUNKWHALE_PLUGINS_PATH) + + +def pip_install(deps, target): + if not deps: + return + pip_path = os.path.join(os.path.dirname(sys.executable), "pip") + subprocess.check_call([pip_path, "install", "-t", target] + deps) diff --git a/api/funkwhale_api/common/apps.py b/api/funkwhale_api/common/apps.py index 7d94695a1..afd834a5a 100644 --- a/api/funkwhale_api/common/apps.py +++ b/api/funkwhale_api/common/apps.py @@ -1,4 +1,7 @@ from django.apps import AppConfig, apps +from django.conf import settings + +from config import plugins from . import mutations from . import utils @@ -13,3 +16,4 @@ class CommonConfig(AppConfig): app_names = [app.name for app in apps.app_configs.values()] mutations.registry.autodiscover(app_names) utils.monkey_patch_request_build_absolute_uri() + plugins.startup.autodiscover([p + ".funkwhale_ready" for p in settings.PLUGINS]) diff --git a/api/funkwhale_api/common/factories.py b/api/funkwhale_api/common/factories.py index 9af602de7..f897f5532 100644 --- a/api/funkwhale_api/common/factories.py +++ b/api/funkwhale_api/common/factories.py @@ -35,3 +35,12 @@ class CommonFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): class Meta: model = "common.Content" + + +@registry.register +class PluginConfiguration(NoUpdateOnCreate, factory.django.DjangoModelFactory): + code = "test" + conf = {"foo": "bar"} + + class Meta: + model = "common.PluginConfiguration" diff --git a/api/funkwhale_api/common/middleware.py b/api/funkwhale_api/common/middleware.py index 64bb6f80b..b085a18fe 100644 --- a/api/funkwhale_api/common/middleware.py +++ b/api/funkwhale_api/common/middleware.py @@ -11,6 +11,7 @@ from django import http from django.conf import settings from django.core.cache import caches from django.middleware import csrf +from django.contrib import auth from django import urls from rest_framework import views @@ -282,6 +283,25 @@ def monkey_patch_rest_initialize_request(): monkey_patch_rest_initialize_request() +def monkey_patch_auth_get_user(): + """ + We need an actor on our users for many endpoints, so we monkey patch + auth.get_user to create it if it's missing + """ + original = auth.get_user + + def replacement(request): + r = original(request) + if not r.is_anonymous and not r.actor: + r.create_actor() + return r + + setattr(auth, "get_user", replacement) + + +monkey_patch_auth_get_user() + + class ThrottleStatusMiddleware: """ Include useful information regarding throttling in API responses to diff --git a/api/funkwhale_api/common/migrations/0008_auto_20200701_1317.py b/api/funkwhale_api/common/migrations/0008_auto_20200701_1317.py new file mode 100644 index 000000000..3abfff42d --- /dev/null +++ b/api/funkwhale_api/common/migrations/0008_auto_20200701_1317.py @@ -0,0 +1,37 @@ +# Generated by Django 3.0.8 on 2020-07-01 13:17 + +from django.conf import settings +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('common', '0007_auto_20200116_1610'), + ] + + operations = [ + migrations.AlterField( + model_name='attachment', + name='url', + field=models.URLField(blank=True, max_length=500, null=True), + ), + migrations.CreateModel( + name='PluginConfiguration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('code', models.CharField(max_length=100)), + ('conf', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)), + ('enabled', models.BooleanField(default=False)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='plugins', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'code')}, + }, + ), + ] diff --git a/api/funkwhale_api/common/models.py b/api/funkwhale_api/common/models.py index 1a31b2dcd..e0bd21619 100644 --- a/api/funkwhale_api/common/models.py +++ b/api/funkwhale_api/common/models.py @@ -363,3 +363,24 @@ def remove_attached_content(sender, instance, **kwargs): getattr(instance, field).delete() except Content.DoesNotExist: pass + + +class PluginConfiguration(models.Model): + """ + Store plugin configuration in DB + """ + + code = models.CharField(max_length=100) + user = models.ForeignKey( + "users.User", + related_name="plugins", + on_delete=models.CASCADE, + null=True, + blank=True, + ) + conf = JSONField(null=True, blank=True) + enabled = models.BooleanField(default=False) + creation_date = models.DateTimeField(default=timezone.now) + + class Meta: + unique_together = ("user", "code") diff --git a/api/funkwhale_api/common/views.py b/api/funkwhale_api/common/views.py index a6ee0c926..a4818acd9 100644 --- a/api/funkwhale_api/common/views.py +++ b/api/funkwhale_api/common/views.py @@ -12,6 +12,8 @@ from rest_framework import response from rest_framework import views from rest_framework import viewsets +from config import plugins + from funkwhale_api.users.oauth import permissions as oauth_permissions from . import filters @@ -210,3 +212,102 @@ class TextPreviewView(views.APIView): ) } return response.Response(data, status=200) + + +class PluginViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): + required_scope = "plugins" + serializer_class = serializers.serializers.Serializer + queryset = models.PluginConfiguration.objects.none() + + def list(self, request, *args, **kwargs): + user = request.user + user_plugins = [p for p in plugins._plugins.values() if p["user"] is True] + + return response.Response( + [ + plugins.serialize_plugin(p, confs=plugins.get_confs(user=user)) + for p in user_plugins + ] + ) + + def retrieve(self, request, *args, **kwargs): + user = request.user + user_plugin = [ + p + for p in plugins._plugins.values() + if p["user"] is True and p["name"] == kwargs["pk"] + ] + if not user_plugin: + return response.Response(status=404) + + return response.Response( + plugins.serialize_plugin(user_plugin[0], confs=plugins.get_confs(user=user)) + ) + + def post(self, request, *args, **kwargs): + return self.create(request, *args, **kwargs) + + def create(self, request, *args, **kwargs): + user = request.user + confs = plugins.get_confs(user=user) + + user_plugin = [ + p + for p in plugins._plugins.values() + if p["user"] is True and p["name"] == kwargs["pk"] + ] + if kwargs["pk"] not in confs: + return response.Response(status=404) + plugins.set_conf(kwargs["pk"], request.data, user) + return response.Response( + plugins.serialize_plugin(user_plugin[0], confs=plugins.get_confs(user=user)) + ) + + def delete(self, request, *args, **kwargs): + user = request.user + confs = plugins.get_confs(user=user) + if kwargs["pk"] not in confs: + return response.Response(status=404) + + user.plugins.filter(code=kwargs["pk"]).delete() + return response.Response(status=204) + + @action(detail=True, methods=["post"]) + def enable(self, request, *args, **kwargs): + user = request.user + if kwargs["pk"] not in plugins._plugins: + return response.Response(status=404) + plugins.enable_conf(kwargs["pk"], True, user) + return response.Response({}, status=200) + + @action(detail=True, methods=["post"]) + def disable(self, request, *args, **kwargs): + user = request.user + if kwargs["pk"] not in plugins._plugins: + return response.Response(status=404) + plugins.enable_conf(kwargs["pk"], False, user) + return response.Response({}, status=200) + + @action(detail=True, methods=["post"]) + def scan(self, request, *args, **kwargs): + user = request.user + if kwargs["pk"] not in plugins._plugins: + return response.Response(status=404) + conf = plugins.get_conf(kwargs["pk"], user=user) + + if not conf["enabled"]: + return response.Response(status=405) + + library = request.user.actor.libraries.get(uuid=conf["conf"]["library"]) + hook = [ + hook + for p, hook in plugins._hooks.get(plugins.SCAN, []) + if p == kwargs["pk"] + ] + + if not hook: + return response.Response(status=405) + + hook[0](library=library, conf=conf["conf"]) + + return response.Response({}, status=200) diff --git a/api/funkwhale_api/contrib/scrobbler/__init__.py b/api/funkwhale_api/contrib/scrobbler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/funkwhale_api/contrib/scrobbler/funkwhale_ready.py b/api/funkwhale_api/contrib/scrobbler/funkwhale_ready.py new file mode 100644 index 000000000..b7a278f83 --- /dev/null +++ b/api/funkwhale_api/contrib/scrobbler/funkwhale_ready.py @@ -0,0 +1,39 @@ +from config import plugins + +from .funkwhale_startup import PLUGIN + +from . import scrobbler + +# https://listenbrainz.org/lastfm-proxy +DEFAULT_SCROBBLER_URL = "http://post.audioscrobbler.com" + + +@plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN) +def forward_to_scrobblers(listening, conf, **kwargs): + if not conf: + raise plugins.Skip() + + username = conf.get("username") + password = conf.get("password") + url = conf.get("url", DEFAULT_SCROBBLER_URL) or DEFAULT_SCROBBLER_URL + if username and password: + PLUGIN["logger"].info("Forwarding scrobbler to %s", url) + session = plugins.get_session() + session_key, now_playing_url, scrobble_url = scrobbler.handshake_v1( + session=session, url=url, username=username, password=password + ) + scrobbler.submit_now_playing_v1( + session=session, + track=listening.track, + session_key=session_key, + now_playing_url=now_playing_url, + ) + scrobbler.submit_scrobble_v1( + session=session, + track=listening.track, + scrobble_time=listening.creation_date, + session_key=session_key, + scrobble_url=scrobble_url, + ) + else: + PLUGIN["logger"].debug("No scrobbler configuration for user, skipping") diff --git a/api/funkwhale_api/contrib/scrobbler/funkwhale_startup.py b/api/funkwhale_api/contrib/scrobbler/funkwhale_startup.py new file mode 100644 index 000000000..517a1eadb --- /dev/null +++ b/api/funkwhale_api/contrib/scrobbler/funkwhale_startup.py @@ -0,0 +1,27 @@ +from config import plugins + +PLUGIN = plugins.get_plugin_config( + name="scrobbler", + label="Scrobbler", + description="A plugin that enables scrobbling to ListenBrainz and Last.fm", + version="0.1", + user=True, + conf=[ + { + "name": "url", + "type": "url", + "allow_null": True, + "allow_blank": True, + "required": False, + "label": "URL of the scrobbler service", + "help": ( + "Suggested choices:\n\n" + "- LastFM (default if left empty): http://post.audioscrobbler.com\n" + "- ListenBrainz: http://proxy.listenbrainz.org/\n" + "- Libre.fm: http://turtle.libre.fm/" + ), + }, + {"name": "username", "type": "text", "label": "Your scrobbler username"}, + {"name": "password", "type": "password", "label": "Your scrobbler password"}, + ], +) diff --git a/api/funkwhale_api/contrib/scrobbler/scrobbler.py b/api/funkwhale_api/contrib/scrobbler/scrobbler.py new file mode 100644 index 000000000..3cf82be26 --- /dev/null +++ b/api/funkwhale_api/contrib/scrobbler/scrobbler.py @@ -0,0 +1,98 @@ +import hashlib +import time + + +# https://github.com/jlieth/legacy-scrobbler +from .funkwhale_startup import PLUGIN + + +class ScrobblerException(Exception): + pass + + +def handshake_v1(session, url, username, password): + timestamp = str(int(time.time())).encode("utf-8") + password_hash = hashlib.md5(password.encode("utf-8")).hexdigest() + auth = hashlib.md5(password_hash.encode("utf-8") + timestamp).hexdigest() + params = { + "hs": "true", + "p": "1.2", + "c": PLUGIN["name"], + "v": PLUGIN["version"], + "u": username, + "t": timestamp, + "a": auth, + } + + PLUGIN["logger"].debug( + "Performing scrobbler handshake for username %s at %s", username, url + ) + handshake_response = session.get(url, params=params) + # process response + result = handshake_response.text.split("\n") + if len(result) >= 4 and result[0] == "OK": + session_key = result[1] + nowplaying_url = result[2] + scrobble_url = result[3] + elif result[0] == "BANNED": + raise ScrobblerException("BANNED") + elif result[0] == "BADAUTH": + raise ScrobblerException("BADAUTH") + elif result[0] == "BADTIME": + raise ScrobblerException("BADTIME") + else: + raise ScrobblerException(handshake_response.text) + + PLUGIN["logger"].debug("Handshake successful, scrobble url: %s", scrobble_url) + return session_key, nowplaying_url, scrobble_url + + +def submit_scrobble_v1(session, scrobble_time, track, session_key, scrobble_url): + payload = get_scrobble_payload(track, scrobble_time) + PLUGIN["logger"].debug("Sending scrobble with payload %s", payload) + payload["s"] = session_key + response = session.post(scrobble_url, payload) + response.raise_for_status() + if response.text.startswith("OK"): + return + elif response.text.startswith("BADSESSION"): + raise ScrobblerException("Remote server says the session is invalid") + else: + raise ScrobblerException(response.text) + + PLUGIN["logger"].debug("Scrobble successfull!") + + +def submit_now_playing_v1(session, track, session_key, now_playing_url): + payload = get_scrobble_payload(track, date=None, suffix="") + PLUGIN["logger"].debug("Sending now playing with payload %s", payload) + payload["s"] = session_key + response = session.post(now_playing_url, payload) + response.raise_for_status() + if response.text.startswith("OK"): + return + elif response.text.startswith("BADSESSION"): + raise ScrobblerException("Remote server says the session is invalid") + else: + raise ScrobblerException(response.text) + + PLUGIN["logger"].debug("Now playing successfull!") + + +def get_scrobble_payload(track, date, suffix="[0]"): + """ + Documentation available at https://web.archive.org/web/20190531021725/https://www.last.fm/api/submissions + """ + upload = track.uploads.filter(duration__gte=0).first() + data = { + "a{}".format(suffix): track.artist.name, + "t{}".format(suffix): track.title, + "l{}".format(suffix): upload.duration if upload else 0, + "b{}".format(suffix): (track.album.title if track.album else "") or "", + "n{}".format(suffix): track.position or "", + "m{}".format(suffix): str(track.mbid) or "", + "o{}".format(suffix): "P", # Source: P = chosen by user + } + if date: + data["i{}".format(suffix)] = int(date.timestamp()) + return data diff --git a/api/funkwhale_api/history/views.py b/api/funkwhale_api/history/views.py index 56afadf40..a14917fc1 100644 --- a/api/funkwhale_api/history/views.py +++ b/api/funkwhale_api/history/views.py @@ -2,6 +2,8 @@ from rest_framework import mixins, viewsets from django.db.models import Prefetch +from config import plugins + from funkwhale_api.activity import record from funkwhale_api.common import fields, permissions from funkwhale_api.music.models import Track @@ -39,6 +41,11 @@ class ListeningViewSet( def perform_create(self, serializer): r = super().perform_create(serializer) + plugins.trigger_hook( + plugins.LISTENING_CREATED, + listening=serializer.instance, + confs=plugins.get_confs(self.request.user), + ) record.send(serializer.instance) return r diff --git a/api/funkwhale_api/music/management/commands/import_files.py b/api/funkwhale_api/music/management/commands/import_files.py index 6643d04c6..0d44af49c 100644 --- a/api/funkwhale_api/music/management/commands/import_files.py +++ b/api/funkwhale_api/music/management/commands/import_files.py @@ -655,9 +655,7 @@ def handle_modified(event, stdout, library, in_place, **kwargs): and to_update.track.attributed_to != library.actor ): stdout.write( - " Cannot update track metadata, track belongs to someone else".format( - to_update.pk - ) + " Cannot update track metadata, track belongs to someone else" ) return else: @@ -777,9 +775,7 @@ def check_upload(stdout, upload): ) if upload.library.actor_id != upload.track.attributed_to_id: stdout.write( - " Cannot update track metadata, track belongs to someone else".format( - upload.pk - ) + " Cannot update track metadata, track belongs to someone else" ) else: track = models.Track.objects.select_related("artist", "album__artist").get( diff --git a/api/funkwhale_api/users/models.py b/api/funkwhale_api/users/models.py index 8942321b6..c6da1a115 100644 --- a/api/funkwhale_api/users/models.py +++ b/api/funkwhale_api/users/models.py @@ -103,7 +103,9 @@ class UserQuerySet(models.QuerySet): user=models.OuterRef("id"), primary=True ).values("verified")[:1] subquery = models.Subquery(verified_emails) - return qs.annotate(has_verified_primary_email=subquery) + return qs.annotate(has_verified_primary_email=subquery).prefetch_related( + "plugins" + ) class UserManager(BaseUserManager): diff --git a/api/funkwhale_api/users/oauth/scopes.py b/api/funkwhale_api/users/oauth/scopes.py index 23958ccad..f5390eb04 100644 --- a/api/funkwhale_api/users/oauth/scopes.py +++ b/api/funkwhale_api/users/oauth/scopes.py @@ -23,6 +23,7 @@ BASE_SCOPES = [ Scope("notifications", "Access personal notifications"), Scope("security", "Access security settings"), Scope("reports", "Access reports"), + Scope("plugins", "Access plugins"), # Privileged scopes that require specific user permissions Scope("instance:settings", "Access instance settings"), Scope("instance:users", "Access local user accounts"), @@ -81,7 +82,12 @@ COMMON_SCOPES = ANONYMOUS_SCOPES | { "write:listenings", } -LOGGED_IN_SCOPES = COMMON_SCOPES | {"read:security", "write:security"} +LOGGED_IN_SCOPES = COMMON_SCOPES | { + "read:security", + "write:security", + "read:plugins", + "write:plugins", +} # We don't allow admin access for oauth apps yet OAUTH_APP_SCOPES = COMMON_SCOPES diff --git a/api/setup.cfg b/api/setup.cfg index 44718f388..724d1c87f 100644 --- a/api/setup.cfg +++ b/api/setup.cfg @@ -1,7 +1,7 @@ [flake8] max-line-length = 120 exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules,tests/data,tests/music/conftest.py -ignore = F405,W503,E203 +ignore = F405,W503,E203,E741 [isort] skip_glob = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules @@ -35,3 +35,4 @@ env = EXTERNAL_MEDIA_PROXY_ENABLED=true DISABLE_PASSWORD_VALIDATORS=false DISABLE_PASSWORD_VALIDATORS=false + FUNKWHALE_PLUGINS= diff --git a/api/tests/manage/test_views.py b/api/tests/manage/test_views.py index bf4f62bb9..92f9b9ceb 100644 --- a/api/tests/manage/test_views.py +++ b/api/tests/manage/test_views.py @@ -67,11 +67,11 @@ def test_domain_create(superuser_api_client, mocker): "funkwhale_api.federation.tasks.update_domain_nodeinfo" ) url = reverse("api:v1:manage:federation:domains-list") - response = superuser_api_client.post(url, {"name": "test.federation"}) + response = superuser_api_client.post(url, {"name": "test.domain"}) assert response.status_code == 201 - assert federation_models.Domain.objects.filter(pk="test.federation").exists() - update_domain_nodeinfo.assert_called_once_with(domain_name="test.federation") + assert federation_models.Domain.objects.filter(pk="test.domain").exists() + update_domain_nodeinfo.assert_called_once_with(domain_name="test.domain") def test_domain_update_allowed(superuser_api_client, factories): @@ -85,6 +85,8 @@ def test_domain_update_allowed(superuser_api_client, factories): def test_domain_update_cannot_change_name(superuser_api_client, factories): + superuser_api_client.user.create_actor() + domain = factories["federation.Domain"]() old_name = domain.name url = reverse("api:v1:manage:federation:domains-detail", kwargs={"pk": old_name}) @@ -96,7 +98,9 @@ def test_domain_update_cannot_change_name(superuser_api_client, factories): assert domain.name == old_name # changing the pk of a model and saving results in a new DB entry in django, # so we check that no other entry was created - assert domain.__class__.objects.count() == 1 + assert ( + domain.__class__.objects.count() == 2 + ) # one for pod domain, and the other one def test_domain_nodeinfo(factories, superuser_api_client, mocker): @@ -131,8 +135,8 @@ def test_actor_list(factories, superuser_api_client, settings): assert response.status_code == 200 - assert response.data["count"] == 1 - assert response.data["results"][0]["id"] == actor.id + assert response.data["count"] == 2 + assert response.data["results"][1]["id"] == actor.id def test_actor_detail(factories, superuser_api_client): diff --git a/api/tests/plugins/test_plugins.py b/api/tests/plugins/test_plugins.py new file mode 100644 index 000000000..ab06fe1ba --- /dev/null +++ b/api/tests/plugins/test_plugins.py @@ -0,0 +1,424 @@ +import os +import sys + +import pytest + +from django.urls import reverse + +from rest_framework import serializers + +from funkwhale_api.common import models +from config import plugins + + +@pytest.fixture(autouse=True) +def _plugins(): + plugins._filters.clear() + plugins._hooks.clear() + plugins._plugins.clear() + yield + plugins._filters.clear() + plugins._hooks.clear() + plugins._plugins.clear() + + +def test_register_filter(): + filters = {} + plugin_config = plugins.get_plugin_config("test", {}) + + def handler(value, conf): + return value + 1 + + plugins.register_filter("test_filter", plugin_config, filters)(handler) + plugins.register_filter("test_filter", plugin_config, filters)(handler) + + assert len(filters["test_filter"]) == 2 + assert plugins.trigger_filter("test_filter", 1, confs={}, registry=filters) == 3 + + +def test_register_hook(mocker): + hooks = {} + plugin_config = plugins.get_plugin_config("test", {}) + mock = mocker.Mock() + + def handler(conf): + mock() + + plugins.register_hook("test_hook", plugin_config, hooks)(handler) + plugins.register_hook("test_hook", plugin_config, hooks)(handler) + plugins.trigger_hook("test_hook", confs={}, registry=hooks) + assert mock.call_count == 2 + assert len(hooks["test_hook"]) == 2 + + +def test_get_plugin_conf(): + _plugins = {} + plugin_config = plugins.get_plugin_config( + "test", description="Hello", registry=_plugins + ) + assert plugin_config["name"] == "test" + assert plugin_config["description"] == "Hello" + assert plugin_config["user"] is False + assert _plugins == { + "test": plugin_config, + } + + +def test_set_plugin_conf_validates(): + _plugins = {} + plugins.get_plugin_config( + "test", registry=_plugins, conf=[{"name": "foo", "type": "boolean"}] + ) + + with pytest.raises(serializers.ValidationError): + plugins.set_conf("test", {"foo": "noop"}, registry=_plugins) + + +def test_set_plugin_conf_valid(): + _plugins = {} + plugins.get_plugin_config( + "test", registry=_plugins, conf=[{"name": "foo", "type": "boolean"}] + ) + plugins.set_conf("test", {"foo": True}, registry=_plugins) + + conf = models.PluginConfiguration.objects.latest("id") + assert conf.code == "test" + assert conf.conf == {"foo": True} + assert conf.user is None + + +def test_set_plugin_conf_valid_user(factories): + user = factories["users.User"]() + _plugins = {} + plugins.get_plugin_config( + "test", registry=_plugins, conf=[{"name": "foo", "type": "boolean"}] + ) + + plugins.set_conf("test", {"foo": True}, user=user, registry=_plugins) + + conf = models.PluginConfiguration.objects.latest("id") + assert conf.code == "test" + assert conf.conf == {"foo": True} + assert conf.user == user + + +def test_get_confs(factories): + plugins.get_plugin_config("test1") + plugins.get_plugin_config("test2") + factories["common.PluginConfiguration"](code="test1", conf={"hello": "world"}) + factories["common.PluginConfiguration"](code="test2", conf={"foo": "bar"}) + + assert plugins.get_confs() == { + "test1": {"conf": {"hello": "world"}, "enabled": False}, + "test2": {"conf": {"foo": "bar"}, "enabled": False}, + } + + +def test_get_confs_user(factories): + plugins.get_plugin_config("test1") + plugins.get_plugin_config("test2") + plugins.get_plugin_config("test3") + user1 = factories["users.User"]() + user2 = factories["users.User"]() + factories["common.PluginConfiguration"](code="test1", conf={"hello": "world"}) + factories["common.PluginConfiguration"](code="test2", conf={"foo": "bar"}) + factories["common.PluginConfiguration"]( + code="test3", conf={"user": True}, user=user1 + ) + factories["common.PluginConfiguration"]( + code="test4", conf={"user": False}, user=user2 + ) + + assert plugins.get_confs(user=user1) == { + "test1": {"conf": {"hello": "world"}, "enabled": False}, + "test2": {"conf": {"foo": "bar"}, "enabled": False}, + "test3": {"conf": {"user": True}, "enabled": False}, + } + + +def test_filter_is_called_with_plugin_conf(mocker, factories): + plugins.get_plugin_config("test1",) + plugins.get_plugin_config("test2",) + factories["common.PluginConfiguration"](code="test1", enabled=True) + factories["common.PluginConfiguration"]( + code="test2", conf={"foo": "baz"}, enabled=True + ) + confs = plugins.get_confs() + filters = {} + plugin_config1 = plugins.get_plugin_config("test1", {}) + plugin_config2 = plugins.get_plugin_config("test2", {}) + + handler1 = mocker.Mock() + handler2 = mocker.Mock() + + plugins.register_filter("test_filter", plugin_config1, filters)(handler1) + plugins.register_filter("test_filter", plugin_config2, filters)(handler2) + + plugins.trigger_filter("test_filter", 1, confs=confs, registry=filters) + + handler1.assert_called_once_with(1, conf=confs["test1"]) + handler2.assert_called_once_with(handler1.return_value, conf=confs["test2"]) + + +def test_get_serializer_from_conf_template(): + template = [ + { + "name": "enabled", + "type": "boolean", + "default": True, + "label": "Enable plugin", + }, + { + "name": "api_url", + "type": "url", + "label": "URL of the scrobbler API", + "validator": lambda self, v: v + "/test", + }, + ] + + serializer_class = plugins.get_serializer_from_conf_template(template) + + data = { + "enabled": True, + "api_url": "http://hello.world", + } + + serializer = serializer_class(data=data) + assert serializer.is_valid(raise_exception=True) is True + assert serializer.validated_data == { + "enabled": True, + "api_url": "http://hello.world/test", + } + + +def test_serialize_plugin(): + plugin = plugins.get_plugin_config( + name="test_plugin", + description="Hello world", + conf=[{"name": "foo", "type": "boolean"}], + ) + + expected = { + "name": "test_plugin", + "enabled": False, + "description": "Hello world", + "conf": [{"name": "foo", "type": "boolean"}], + "user": False, + "source": False, + "label": "test_plugin", + "values": None, + } + + assert plugins.serialize_plugin(plugin, plugins.get_confs()) == expected + + +def test_serialize_plugin_user(factories): + user = factories["users.User"]() + plugin = plugins.get_plugin_config( + name="test_plugin", + description="Hello world", + conf=[{"name": "foo", "type": "boolean"}], + user=True, + ) + + expected = { + "name": "test_plugin", + "enabled": False, + "description": "Hello world", + "conf": [{"name": "foo", "type": "boolean"}], + "user": True, + "source": False, + "label": "test_plugin", + "values": None, + } + + assert plugins.serialize_plugin(plugin, plugins.get_confs(user)) == expected + + +def test_serialize_plugin_user_enabled(factories): + user = factories["users.User"]() + plugin = plugins.get_plugin_config( + name="test_plugin", + description="Hello world", + conf=[{"name": "foo", "type": "boolean"}], + user=True, + ) + + factories["common.PluginConfiguration"]( + code="test_plugin", user=user, enabled=True, conf={"foo": "bar"} + ) + expected = { + "name": "test_plugin", + "enabled": True, + "description": "Hello world", + "conf": [{"name": "foo", "type": "boolean"}], + "user": True, + "source": False, + "label": "test_plugin", + "values": {"foo": "bar"}, + } + + assert plugins.serialize_plugin(plugin, plugins.get_confs(user)) == expected + + +def test_can_list_user_plugins(logged_in_api_client): + plugin = plugins.get_plugin_config( + name="test_plugin", + description="Hello world", + conf=[{"name": "foo", "type": "boolean"}], + user=True, + ) + plugins.get_plugin_config(name="test_plugin2", user=False) + url = reverse("api:v1:plugins-list") + response = logged_in_api_client.get(url) + + assert response.status_code == 200 + assert response.data == [ + plugins.serialize_plugin(plugin, plugins.get_confs(logged_in_api_client.user)) + ] + + +def test_can_retrieve_user_plugin(logged_in_api_client): + plugin = plugins.get_plugin_config( + name="test_plugin", + description="Hello world", + conf=[{"name": "foo", "type": "boolean"}], + user=True, + ) + plugins.get_plugin_config(name="test_plugin2", user=False) + url = reverse("api:v1:plugins-detail", kwargs={"pk": "test_plugin"}) + response = logged_in_api_client.get(url) + + assert response.status_code == 200 + assert response.data == plugins.serialize_plugin( + plugin, plugins.get_confs(logged_in_api_client.user) + ) + + +def test_can_update_user_plugin(logged_in_api_client): + plugin = plugins.get_plugin_config( + name="test_plugin", + description="Hello world", + conf=[{"name": "foo", "type": "boolean"}], + user=True, + ) + plugins.get_plugin_config(name="test_plugin2", user=False) + url = reverse("api:v1:plugins-detail", kwargs={"pk": "test_plugin"}) + response = logged_in_api_client.post(url, {"foo": True}) + assert response.status_code == 200 + assert logged_in_api_client.user.plugins.latest("id").conf == {"foo": True} + assert response.data == plugins.serialize_plugin( + plugin, plugins.get_confs(logged_in_api_client.user) + ) + + +def test_can_destroy_user_plugin(logged_in_api_client): + plugins.get_plugin_config( + name="test_plugin", + description="Hello world", + conf=[{"name": "foo", "type": "boolean"}], + user=True, + ) + plugins.set_conf("test_plugin", {"foo": True}, user=logged_in_api_client.user) + plugins.get_plugin_config(name="test_plugin2", user=False) + url = reverse("api:v1:plugins-detail", kwargs={"pk": "test_plugin"}) + response = logged_in_api_client.delete(url, {"enabled": True}) + assert response.status_code == 204 + + with pytest.raises(models.PluginConfiguration.DoesNotExist): + assert logged_in_api_client.user.plugins.latest("id") + + +def test_can_enable_user_plugin(logged_in_api_client): + plugins.get_plugin_config( + name="test_plugin", + description="Hello world", + conf=[{"name": "foo", "type": "boolean"}], + user=True, + ) + plugins.set_conf("test_plugin", {"foo": True}, user=logged_in_api_client.user) + url = reverse("api:v1:plugins-enable", kwargs={"pk": "test_plugin"}) + response = logged_in_api_client.post(url) + assert response.status_code == 200 + + assert logged_in_api_client.user.plugins.latest("id").enabled is True + + +def test_can_disable_user_plugin(logged_in_api_client): + plugins.get_plugin_config( + name="test_plugin", + description="Hello world", + conf=[{"name": "foo", "type": "boolean"}], + user=True, + ) + plugins.set_conf("test_plugin", {"foo": True}, user=logged_in_api_client.user) + url = reverse("api:v1:plugins-disable", kwargs={"pk": "test_plugin"}) + response = logged_in_api_client.post(url) + assert response.status_code == 200 + + assert logged_in_api_client.user.plugins.latest("id").enabled is False + + +def test_can_install_dependencies(mocker): + dependencies = ["depa==12", "depb"] + check_call = mocker.patch("subprocess.check_call") + expected = [ + os.path.join(os.path.dirname(sys.executable), "pip"), + "install", + ] + dependencies + plugins.install_dependencies(dependencies) + check_call.assert_called_once_with(expected) + + +def test_set_plugin_source_conf_invalid(factories): + user = factories["users.User"]() + _plugins = {} + plugins.get_plugin_config( + "test", + source=True, + registry=_plugins, + conf=[{"name": "foo", "type": "boolean"}], + ) + with pytest.raises(serializers.ValidationError): + plugins.set_conf("test", {"foo": True}, user=user, registry=_plugins) + + +def test_set_plugin_source_conf_valid(factories): + library = factories["music.Library"](actor__local=True) + _plugins = {} + plugins.get_plugin_config( + "test", + source=True, + registry=_plugins, + conf=[{"name": "foo", "type": "boolean"}], + ) + plugins.set_conf( + "test", + {"foo": True, "library": library.uuid}, + user=library.actor.user, + registry=_plugins, + ) + conf = models.PluginConfiguration.objects.latest("id") + assert conf.code == "test" + assert conf.conf == {"foo": True, "library": str(library.uuid)} + assert conf.user == library.actor.user + + +def test_can_trigger_scan(logged_in_api_client, mocker, factories): + library = factories["music.Library"](actor=logged_in_api_client.user.create_actor()) + plugin = plugins.get_plugin_config( + name="test_plugin", description="Hello world", conf=[], source=True, + ) + handler = mocker.Mock() + plugins.register_hook(plugins.SCAN, plugin)(handler) + plugins.set_conf( + "test_plugin", {"library": library.uuid}, user=logged_in_api_client.user + ) + url = reverse("api:v1:plugins-scan", kwargs={"pk": "test_plugin"}) + plugins.enable_conf("test_plugin", True, logged_in_api_client.user) + response = logged_in_api_client.post(url) + assert response.status_code == 200 + + handler.assert_called_once_with( + library=library, conf={"library": str(library.uuid)} + ) diff --git a/api/tests/users/oauth/test_scopes.py b/api/tests/users/oauth/test_scopes.py index 7261ac6b1..e55b78319 100644 --- a/api/tests/users/oauth/test_scopes.py +++ b/api/tests/users/oauth/test_scopes.py @@ -34,6 +34,9 @@ from funkwhale_api.users.oauth import scopes "write:listenings", "read:security", "write:security", + "write:listenings", + "read:plugins", + "write:plugins", "read:instance:policies", "write:instance:policies", "read:instance:accounts", @@ -85,6 +88,8 @@ from funkwhale_api.users.oauth import scopes "write:listenings", "read:security", "write:security", + "read:plugins", + "write:plugins", "read:instance:policies", "write:instance:policies", "read:instance:accounts", @@ -132,6 +137,8 @@ from funkwhale_api.users.oauth import scopes "write:listenings", "read:security", "write:security", + "read:plugins", + "write:plugins", "read:instance:policies", "write:instance:policies", "read:instance:accounts", @@ -173,6 +180,8 @@ from funkwhale_api.users.oauth import scopes "write:listenings", "read:security", "write:security", + "read:plugins", + "write:plugins", }, ), ], diff --git a/docs/conf.py b/docs/conf.py index ae278c5e2..5ac3127be 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,6 +23,7 @@ import datetime from shutil import copyfile sys.path.insert(0, os.path.abspath("../api")) +sys.path.insert(0, os.path.abspath("../api/config")) import funkwhale_api # NOQA @@ -30,9 +31,9 @@ FUNKWHALE_CONFIG = { "FUNKWHALE_URL": "mypod.funkwhale", "FUNKWHAL_PROTOCOL": "https", "DATABASE_URL": "postgres://localhost:5432/db", - "AWS_ACCESS_KEY_ID": 'my_access_key', - "AWS_SECRET_ACCESS_KEY": 'my_secret_key', - "AWS_STORAGE_BUCKET_NAME": 'my_bucket', + "AWS_ACCESS_KEY_ID": "my_access_key", + "AWS_SECRET_ACCESS_KEY": "my_secret_key", + "AWS_STORAGE_BUCKET_NAME": "my_bucket", } for key, value in FUNKWHALE_CONFIG.items(): os.environ[key] = value @@ -46,7 +47,7 @@ for key, value in FUNKWHALE_CONFIG.items(): # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ["sphinx.ext.graphviz", "sphinx.ext.autodoc"] -autodoc_mock_imports = ["celery", "django_auth_ldap", "ldap"] +autodoc_mock_imports = ["celery", "django_auth_ldap", "ldap", "persisting_theory", "rest_framework"] add_module_names = False # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/developers/index.rst b/docs/developers/index.rst index 966cac3af..f214c7819 100644 --- a/docs/developers/index.rst +++ b/docs/developers/index.rst @@ -13,5 +13,6 @@ Reference architecture ../api ./authentication + ./plugins ../federation/index subsonic diff --git a/docs/developers/plugins.rst b/docs/developers/plugins.rst new file mode 100644 index 000000000..f9f537479 --- /dev/null +++ b/docs/developers/plugins.rst @@ -0,0 +1,165 @@ +Funkwhale plugins +================= + +Starting with Funkwhale 1.0, it is now possible to implement new features +via plugins. + +Some plugins are maintained by the Funkwhale team (e.g. this is the case of the ``scrobbler`` plugin), +or by third-parties. + +Installing a plugin +------------------- + +To install a plugin, ensure its directory is present in the ``FUNKWHALE_PLUGINS_PATH`` directory. + +Then, add its name to the ``FUNKWHALE_PLUGINS`` environment variable, like this:: + + FUNKWHALE_PLUGINS=myplugin,anotherplugin + +We provide a command to make it easy to install third-party plugins:: + + python manage.py fw plugins install https://pluginurl.zip + +.. note:: + + If you use the command, you will still need to append the plugin name to ``FUNKWHALE_PLUGINS`` + + +Types of plugins +---------------- + +There are two types of plugins: + +1. Plugins that are accessible to end-users, a.k.a. user-level plugins. This is the case of our Scrobbler plugin +2. Pod-level plugins that are configured by pod admins and are not tied to a particular user + +Additionally, user-level plugins can be regular plugins or source plugins. A source plugin provides +a way to import files from a third-party service, e.g via webdav, FTP or something similar. + +Hooks and filters +----------------- + +Funkwhale includes two kind of entrypoints for plugins to use: hooks and filters. B + +Hooks should be used when you want to react to some change. For instance, the ``LISTENING_CREATED`` hook +notify each registered callback that a listening was created. Our ``scrobbler`` plugin has a callback +registered to this hook, so that it can notify Last.fm properly: + +.. code-block:: python + + from config import plugins + from .funkwhale_startup import PLUGIN + + @plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN) + def notify_lastfm(listening, conf, **kwargs): + # do something + +Filters work slightly differently, and expect callbacks to return a value that will be used by Funkwhale. + +For instance, the ``PLUGINS_DEPENDENCIES`` filter can be used as a way to install additional dependencies needed by your plugin: + + +.. code-block:: python + + # funkwhale_startup.py + # ... + from config import plugins + + @plugins.register_filter(plugins.PLUGINS_DEPENDENCIES, PLUGIN) + def dependencies(dependencies, **kwargs): + return dependencies + ["django_prometheus"] + +To sum it up, hooks are used when you need to react to something, and filters when you need to alter something. + +Writing a plugin +---------------- + +Regardless of the type of plugin you want to write, lots of concepts are similar. + +First, a plugin need three files: + +- a ``__init__.py`` file, since it's a Python package +- a ``funkwhale_startup.py`` file, that is loaded during Funkwhale initialization +- a ``funkwhale_ready.py`` file, that is loaded when Funkwhale is configured and ready + +So your plugin directory should look like this:: + + myplugin + ├── funkwhale_ready.py + ├── funkwhale_startup.py + └── __init__.py + +Now, let's write our plugin! + +``funkwhale_startup.py`` is where you declare your plugin and it's configuration options: + +.. code-block:: python + + # funkwhale_startup.py + from config import plugins + + PLUGIN = plugins.get_plugin_config( + name="myplugin", + label="My Plugin", + description="An example plugin that greets you", + version="0.1", + # here, we write a user-level plugin + user=True, + conf=[ + # this configuration options are editable by each user + {"name": "greeting", "type": "text", "label": "Greeting", "default": "Hello"}, + ], + ) + +Now that our plugin is declared and configured, let's implement actual functionality in ``funkwhale_ready.py``: + +.. code-block:: python + + # funkwhale_ready.py + from django.urls import path + from rest_framework import response + from rest_framework import views + + from config import plugins + + from .funkwhale_startup import PLUGIN + + # Our greeting view, where the magic happens + class GreetingView(views.APIView): + permission_classes = [] + def get(self, request, *args, **kwargs): + # retrieve plugin configuration for the current user + conf = plugins.get_conf(PLUGIN["name"], request.user) + if not conf["enabled"]: + # plugin is disabled for this user + return response.Response(status=405) + greeting = conf["conf"]["greeting"] + data = { + "greeting": "{} {}!".format(greeting, request.user.username) + } + return response.Response(data) + + # Ensure our view is known by Django and available at /greeting + @plugins.register_filter(plugins.URLS, PLUGIN) + def register_view(urls, **kwargs): + return urls + [ + path('greeting', GreetingView.as_view()) + ] + +And that's pretty much it. Now, login, visit https://yourpod.domain/settings/plugins, set a value in the ``greeting`` field and enable the plugin. + +After that, you should be greeted properly if you go to https://yourpod.domain/greeting. + +Hooks reference +--------------- + +.. autodata:: config.plugins.LISTENING_CREATED + +Filters reference +----------------- + +.. autodata:: config.plugins.PLUGINS_DEPENDENCIES +.. autodata:: config.plugins.PLUGINS_APPS +.. autodata:: config.plugins.PLUGINSMIDDLEWARES_BEFORE_DEPENDENCIES +.. autodata:: config.plugins.MIDDLEWARES_AFTER +.. autodata:: config.plugins.URLS diff --git a/front/scripts/fix-fomantic-css.py b/front/scripts/fix-fomantic-css.py index 0e9f51771..692383f59 100755 --- a/front/scripts/fix-fomantic-css.py +++ b/front/scripts/fix-fomantic-css.py @@ -79,6 +79,7 @@ GLOBAL_REPLACES = [ ("#ff4335", "var(--danger-focus-color)"), ] + def discard_unused_icons(rule): """ Add an icon to this list if you want to use it in the app. @@ -890,7 +891,9 @@ def replace_vars(source, dest): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Replace hardcoded values by CSS vars and strip unused rules") + parser = argparse.ArgumentParser( + description="Replace hardcoded values by CSS vars and strip unused rules" + ) parser.add_argument( "source", help="Source path of the fomantic-ui-less distribution to fix" ) diff --git a/front/src/components/auth/Plugin.vue b/front/src/components/auth/Plugin.vue new file mode 100644 index 000000000..315af0449 --- /dev/null +++ b/front/src/components/auth/Plugin.vue @@ -0,0 +1,113 @@ +