Initial plugin API

This commit is contained in:
Eliot Berriot 2019-09-23 17:32:32 +02:00
parent 6f5716a128
commit 8a3a42e485
No known key found for this signature in database
GPG Key ID: 6B501DFD73514E14
12 changed files with 684 additions and 3 deletions

View File

@ -81,11 +81,15 @@ else:
logger.info("Loaded env file at %s/.env", path)
break
FUNKWHALE_PLUGINS_PATH = env(
"FUNKWHALE_PLUGINS_PATH", default="/srv/funkwhale/plugins/"
FUNKWHALE_PLUGINS_PATH = env.list(
"FUNKWHALE_PLUGINS_PATH",
default=["/srv/funkwhale/plugins/", str(ROOT_DIR.path("plugins"))],
)
sys.path.append(FUNKWHALE_PLUGINS_PATH)
for path in FUNKWHALE_PLUGINS_PATH:
sys.path.append(path)
print("HELLO", sys.path)
FUNKWHALE_HOSTNAME = None
FUNKWHALE_HOSTNAME_SUFFIX = env("FUNKWHALE_HOSTNAME_SUFFIX", default=None)
FUNKWHALE_HOSTNAME_PREFIX = env("FUNKWHALE_HOSTNAME_PREFIX", default=None)
@ -186,6 +190,7 @@ if RAVEN_ENABLED:
# Apps specific for this project go here.
LOCAL_APPS = (
"funkwhale_api.common.apps.CommonConfig",
"funkwhale_api.plugins",
"funkwhale_api.activity.apps.ActivityConfig",
"funkwhale_api.users", # custom users app
"funkwhale_api.users.oauth",

View File

@ -0,0 +1,181 @@
import persisting_theory
import django.dispatch
from django import apps
import logging
from . import config
logger = logging.getLogger(__name__)
class Plugin(apps.AppConfig):
_is_funkwhale_plugin = True
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.hooks = HookRegistry()
self.settings = SettingRegistry()
self.user_settings = SettingRegistry()
def ready(self):
super().ready()
logging.info("Loading plugin %s", self.label)
self.load()
logging.info("Plugin %s loaded", self.label)
def load(self):
pass
class FuncRegistry(persisting_theory.Registry):
def connect(self, hook_name):
def inner(handler):
self[hook_name] = handler
return handler
return inner
class HookRegistry(FuncRegistry):
pass
class SettingRegistry(persisting_theory.Registry):
def prepare_name(self, data, name):
return data().identifier()
class PluginException(Exception):
pass
class PluginNotFound(PluginException):
pass
class Skip(PluginException):
pass
class PluginSignal(object):
def __init__(self, name, providing_args=[]):
self.name = name
self.providing_args = providing_args
class Hook(PluginSignal):
pass
class SignalsRegistry(persisting_theory.Registry):
def prepare_name(self, data, name):
return data.name
def dispatch(self, hook_name, plugins_conf, **kwargs):
"""
Call all handlers connected to hook_name in turn.
"""
if hook_name not in self:
raise LookupError(hook_name)
logger.debug("[Plugin:hook:%s] Dispatching hook", hook_name)
matching_hooks = []
for row in plugins_conf:
try:
matching_hooks.append((row, row["obj"].hooks[hook_name]))
except KeyError:
continue
if matching_hooks:
logger.debug(
"[Plugin:hook:%s] %s handlers found", hook_name, len(matching_hooks)
)
else:
logger.debug("[Plugin:hook:%s] No handler founds", hook_name)
return
for row, handler in matching_hooks:
logger.debug(
"[Plugin:hook:%s] Calling handler %s from plugin %s",
hook_name,
handler,
row["obj"].name,
)
try:
handler(plugin_conf=row, **kwargs)
except Skip:
logger.debug("[Plugin:hook:%s] handler skipped", hook_name)
except Exception:
logger.exception(
"[Plugin:hook:%s] unknown exception with handler %s",
hook_name,
handler,
)
else:
logger.debug("[Plugin:hook:%s] handler %s called successfully", handler)
logger.debug("[Plugin:hook:%s] Done", hook_name)
hooks = SignalsRegistry()
def get_plugin(name):
try:
plugin = apps.apps.get_app_config(name)
except LookupError:
raise PluginNotFound(name)
if not getattr(plugin, "_is_funkwhale_plugin", False):
raise PluginNotFound(name)
return plugin
def get_all_plugins():
return [
app
for app in apps.apps.get_app_configs()
if getattr(app, "_is_funkwhale_plugin", False)
]
def generate_plugin_conf(plugins, user=None):
from . import models
plugin_conf = []
qs = models.Plugin.objects.filter(is_enabled=True).values("name", "config")
by_plugin_name = {obj["name"]: obj["config"] for obj in qs}
for plugin in plugins:
if plugin.name not in by_plugin_name:
continue
conf = {
"obj": plugin,
"user": None,
"settings": by_plugin_name[plugin.name] or {},
}
plugin_conf.append(conf)
if plugin_conf and user and user.is_authenticated:
qs = models.UserPlugin.objects.filter(
user=user, plugin__is_enabled=True, is_enabled=True
).values("plugin__name", "config")
by_plugin_name = {obj["plugin__name"]: obj["config"] for obj in qs}
for row in plugin_conf:
if row["obj"].name in by_plugin_name:
row["user"] = {
"id": user.pk,
"settings": by_plugin_name[row["obj"].name],
}
return plugin_conf
def attach_plugin_conf(obj, user):
from funkwhale_api.common import preferences
plugins_enabled = preferences.get("plugins__enabled")
if plugins_enabled:
conf = generate_plugin_conf(plugins=get_all_plugins(), user=user)
else:
conf = None
setattr(obj, "plugin_conf", conf)

View File

@ -0,0 +1,18 @@
from funkwhale_api.common import admin
from . import models
@admin.register(models.Plugin)
class PluginAdmin(admin.ModelAdmin):
list_display = ["name", "creation_date", "is_enabled"]
list_filter = ["is_enabled"]
list_select_related = True
@admin.register(models.UserPlugin)
class UserPluginAdmin(admin.ModelAdmin):
list_display = ["plugin", "user", "creation_date", "is_enabled"]
search_fields = ["user__username", "plugin__name"]
list_filter = ["plugin__name", "is_enabled"]
list_select_related = True

View File

@ -0,0 +1,49 @@
from django import forms
from dynamic_preferences import types
SettingSection = types.Section
StringSetting = types.StringPreference
class PasswordSetting(types.StringPreference):
widget = forms.PasswordInput
class BooleanSetting(types.BooleanPreference):
# Boolean are supported in JSON, so no need to serialized to a string
serializer = None
class IntSetting(types.IntegerPreference):
# Integers are supported in JSON, so no need to serialized to a string
serializer = None
def validate_config(payload, settings):
"""
Dynamic preferences stores settings in a separate database table. However
it is a bit too much for our use cases, and we simply want to store
these in a JSONField on the corresponding model row.
This validates the payload using the dynamic preferences serializers
and return a config that is ready to be persisted as JSON
"""
final = {}
for klass in settings:
setting = klass()
setting_id = setting.identifier()
try:
value = payload[setting_id]
except KeyError:
continue
setting.validate(value)
if setting.serializer:
value = setting.serializer.serialize(value)
final[setting_id] = value
return final

View File

@ -0,0 +1,41 @@
import factory
from funkwhale_api.factories import registry, NoUpdateOnCreate
from funkwhale_api.users.factories import UserFactory
@registry.register
class PluginFactory(factory.django.DjangoModelFactory):
is_enabled = True
config = factory.Faker("pydict", nb_elements=3)
class Meta:
model = "plugins.Plugin"
@factory.post_generation
def refresh(self, created, *args, **kwargs):
"""
Needed to ensure we have JSON serialized value in the config field
"""
if created:
self.refresh_from_db()
@registry.register
class UserPluginFactory(factory.django.DjangoModelFactory):
is_enabled = True
user = factory.SubFactory(UserFactory)
plugin = factory.SubFactory(PluginFactory)
config = factory.Faker("pydict", nb_elements=3)
class Meta:
model = "plugins.UserPlugin"
@factory.post_generation
def refresh(self, created, *args, **kwargs):
"""
Needed to ensure we have JSON serialized value in the config field
"""
if created:
self.refresh_from_db()

View File

@ -0,0 +1,44 @@
# Generated by Django 2.2.4 on 2019-09-23 15:17
from django.conf import settings
import django.contrib.postgres.fields.jsonb
import django.core.serializers.json
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='Plugin',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=70, unique=True)),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('is_enabled', models.BooleanField()),
('config', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000, null=True)),
],
),
migrations.CreateModel(
name='UserPlugin',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('is_enabled', models.BooleanField()),
('config', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000, null=True)),
('plugin', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='user_plugins', to='plugins.Plugin')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='user_plugins', to=settings.AUTH_USER_MODEL)),
],
options={
'unique_together': {('user', 'plugin')},
},
),
]

View File

@ -0,0 +1,35 @@
from django.core.serializers.json import DjangoJSONEncoder
from django.contrib.postgres.fields import JSONField
from django.db import models
from django.utils import timezone
class Plugin(models.Model):
name = models.CharField(unique=True, max_length=70)
creation_date = models.DateTimeField(default=timezone.now)
is_enabled = models.BooleanField()
config = JSONField(
default=None, max_length=50000, encoder=DjangoJSONEncoder, blank=True, null=True
)
def __str__(self):
return self.name
class UserPlugin(models.Model):
plugin = models.ForeignKey(
Plugin, related_name="user_plugins", on_delete=models.CASCADE
)
user = models.ForeignKey(
"users.User", related_name="user_plugins", on_delete=models.CASCADE
)
creation_date = models.DateTimeField(default=timezone.now)
is_enabled = models.BooleanField()
config = JSONField(
default=None, max_length=50000, encoder=DjangoJSONEncoder, blank=True, null=True
)
class Meta:
unique_together = ("user", "plugin")

View File

@ -24,6 +24,7 @@ from aioresponses import aioresponses
from dynamic_preferences.registries import global_preferences_registry
from rest_framework.test import APIClient, APIRequestFactory
from funkwhale_api import plugins
from funkwhale_api.activity import record
from funkwhale_api.federation import actors
from funkwhale_api.moderation import mrf
@ -422,3 +423,17 @@ def clear_license_cache(db):
licenses._cache = None
yield
licenses._cache = None
@pytest.fixture
def plugin_class():
class DummyPlugin(plugins.Plugin):
path = "noop"
return DummyPlugin
@pytest.fixture
def plugin(plugin_class):
return plugin_class("test", "test")

View File

@ -0,0 +1,181 @@
import pytest
from funkwhale_api import plugins
from funkwhale_api.plugins import models
def test_plugin_ready_calls_load(mocker, plugin):
load = mocker.spy(plugin, "load")
plugin.ready()
load.assert_called_once_with()
def test_get_plugin_not_found(mocker):
get_app_config = mocker.patch(
"django.apps.apps.get_app_config", side_effect=LookupError
)
with pytest.raises(plugins.PluginNotFound):
plugins.get_plugin("noop")
get_app_config.assert_called_once_with("noop")
def test_get_plugin_not_plugin(mocker):
get_app_config = mocker.spy(plugins.apps.apps, "get_app_config")
with pytest.raises(plugins.PluginNotFound):
plugins.get_plugin("music")
get_app_config.assert_called_once_with("music")
def test_get_plugin_valid(mocker):
get_app_config = mocker.patch("django.apps.apps.get_app_config")
get_app_config.return_value = mocker.Mock(_is_funkwhale_plugin=True)
assert plugins.get_plugin("test") is get_app_config.return_value
get_app_config.assert_called_once_with("test")
def test_plugin_attributes(plugin):
assert isinstance(plugin.hooks, plugins.HookRegistry)
assert isinstance(plugin.settings, plugins.SettingRegistry)
assert isinstance(plugin.user_settings, plugins.SettingRegistry)
def test_plugin_hook_connect(plugin):
@plugin.hooks.connect("hook_name")
def handler(**kwargs):
pass
assert plugin.hooks["hook_name"] == handler
def test_plugin_user_settings_register(plugin):
@plugin.user_settings.register
class TestSetting(plugins.config.StringSetting):
section = plugins.config.SettingSection("test")
name = "test_setting"
default = ""
assert plugin.user_settings["test__test_setting"] == TestSetting
def test_plugin_settings_register(plugin):
@plugin.settings.register
class TestSetting(plugins.config.StringSetting):
section = plugins.config.SettingSection("test")
name = "test_setting"
default = ""
assert plugin.settings["test__test_setting"] == TestSetting
def test_get_all_plugins(mocker):
pl1 = mocker.Mock(_is_funkwhale_plugin=True)
pl2 = mocker.Mock(_is_funkwhale_plugin=True)
app = mocker.Mock(_is_funkwhale_plugin=False)
mocker.patch("django.apps.apps.get_app_configs", return_value=[pl1, pl2, app])
all_plugins = plugins.get_all_plugins()
assert all_plugins == [pl1, pl2]
def test_generate_plugin_conf(factories, plugin_class):
plugin1 = plugin_class("test1", "test1")
plugin2 = plugin_class("test2", "test2")
plugin3 = plugin_class("test3", "test3")
plugin4 = plugin_class("test4", "test4")
user = factories["users.User"]()
# this one is enabled
plugin1_db_conf = factories["plugins.Plugin"](name=plugin1.name)
# this one is enabled, with additional user-level configuration (see below)
plugin2_db_conf = factories["plugins.Plugin"](name=plugin2.name)
# this one is disabled at the plugin level, so it shouldn't appear in the final conf
plugin3_db_conf = factories["plugins.Plugin"](name=plugin3.name, is_enabled=False)
# this one is enabled, but disabled at user level (see below)
plugin4_db_conf = factories["plugins.Plugin"](name=plugin4.name)
# this one doesn't match any registered app
factories["plugins.Plugin"](name="noop")
# a user-level configuration but with a different user, so irrelevant
factories["plugins.UserPlugin"](plugin=plugin1_db_conf)
# a user-level configuration but the plugin is disabled
factories["plugins.UserPlugin"](user=user, plugin=plugin3_db_conf)
# a user-level configuration, plugin is enabled, should be reflected in the final conf
plugin2_user_db_conf = factories["plugins.UserPlugin"](
user=user, plugin=plugin2_db_conf
)
# a user-level configuration, plugin is enabled but disabled by user, should be reflected in the final conf
factories["plugins.UserPlugin"](user=user, plugin=plugin4_db_conf, is_enabled=False)
expected = [
{"obj": plugin1, "settings": plugin1_db_conf.config, "user": None},
{
"obj": plugin2,
"settings": plugin2_db_conf.config,
"user": {"id": user.pk, "settings": plugin2_user_db_conf.config},
},
{"obj": plugin4, "settings": plugin4_db_conf.config, "user": None},
]
conf = plugins.generate_plugin_conf([plugin1, plugin2, plugin3, plugin4], user=user)
assert conf == expected
def test_generate_plugin_conf_anonymous_user(factories, plugin_class):
plugin1 = plugin_class("test1", "test1")
plugin2 = plugin_class("test2", "test2")
plugin3 = plugin_class("test3", "test3")
plugin4 = plugin_class("test4", "test4")
user = factories["users.User"]()
# this one is enabled
plugin1_db_conf = factories["plugins.Plugin"](name=plugin1.name)
# this one is enabled, with additional user-level configuration (see below)
plugin2_db_conf = factories["plugins.Plugin"](name=plugin2.name)
# this one is disabled at the plugin level, so it shouldn't appear in the final conf
plugin3_db_conf = factories["plugins.Plugin"](name=plugin3.name, is_enabled=False)
# this one is enabled, but disabled at user level (see below)
plugin4_db_conf = factories["plugins.Plugin"](name=plugin4.name)
# this one doesn't match any registered app
factories["plugins.Plugin"](name="noop")
# a user-level configuration but with a different user, so irrelevant
factories["plugins.UserPlugin"](plugin=plugin1_db_conf)
# a user-level configuration but the plugin is disabled
factories["plugins.UserPlugin"](user=user, plugin=plugin3_db_conf)
expected = [
{"obj": plugin1, "settings": plugin1_db_conf.config, "user": None},
{"obj": plugin2, "settings": plugin2_db_conf.config, "user": None},
{"obj": plugin4, "settings": plugin4_db_conf.config, "user": None},
]
conf = plugins.generate_plugin_conf([plugin1, plugin2, plugin3, plugin4], user=None)
assert conf == expected
def test_attach_plugin_conf(mocker):
request = mocker.Mock()
generate_plugin_conf = mocker.patch.object(plugins, "generate_plugin_conf")
get_all_plugins = mocker.patch.object(plugins, "get_all_plugins")
user = mocker.Mock()
plugins.attach_plugin_conf(request, user=user)
generate_plugin_conf.assert_called_once_with(
plugins=get_all_plugins.return_value, user=user
)
assert request.plugin_conf == generate_plugin_conf.return_value
def test_attach_plugin_noop_if_plugins_disabled(mocker, preferences):
preferences["plugins__enabled"] = False
request = mocker.Mock()
plugins.attach_plugin_conf(request, user=None)
assert request.plugin_conf is None

View File

@ -0,0 +1,61 @@
import pytest
from funkwhale_api.plugins import config
@pytest.mark.parametrize(
"payload, expected",
[
({"test__test_value1": "hello"}, {"test__test_value1": "hello"}),
(
{"noop": "noop", "test__test_value1": "hello"},
{"test__test_value1": "hello"},
),
(
{"test__test_value1": "hello", "test__test_value2": "world"},
{"test__test_value1": "hello", "test__test_value2": "world"},
),
],
)
def test_validate_config(payload, expected):
test_section = config.SettingSection("test")
class TestSetting1(config.StringSetting):
name = "test_value1"
section = test_section
default = ""
class TestSetting2(config.StringSetting):
name = "test_value2"
section = test_section
default = ""
final = config.validate_config(payload, settings=[TestSetting1, TestSetting2])
assert final == expected
def test_validate_config_boolean():
test_section = config.SettingSection("test")
class TestSetting(config.BooleanSetting):
name = "test_value"
section = test_section
default = False
final = config.validate_config({"test__test_value": True}, settings=[TestSetting])
assert final == {"test__test_value": True}
def test_validate_config_number():
test_section = config.SettingSection("test")
class TestSetting(config.IntSetting):
name = "test_value"
section = test_section
default = 12
final = config.validate_config({"test__test_value": 12}, settings=[TestSetting])
assert final == {"test__test_value": 12}

View File

@ -0,0 +1,51 @@
from funkwhale_api import plugins
def test_hooks_register():
hook = plugins.Hook("history.listenings.created", providing_args=["listening"])
plugins.hooks.register(hook)
assert plugins.hooks["history.listenings.created"] == hook
def test_hooks_dispatch(mocker, plugin_class):
plugin1 = plugin_class("test1", "test1")
plugin2 = plugin_class("test2", "test2")
hook = plugins.Hook("history.listenings.created", providing_args=["listening"])
plugins.hooks.register(hook)
handler1 = mocker.stub()
handler2 = mocker.stub()
plugin1.hooks.connect("history.listenings.created")(handler1)
plugin2.hooks.connect("history.listenings.created")(handler2)
plugins_conf = [
{"obj": plugin1, "user": {"hello": "world"}, "settings": {"foo": "bar"}},
{"obj": plugin2},
]
plugins.hooks.dispatch(
"history.listenings.created", listening="test", plugins_conf=plugins_conf
)
handler1.assert_called_once_with(listening="test", plugin_conf=plugins_conf[0])
handler2.assert_called_once_with(listening="test", plugin_conf=plugins_conf[1])
def test_hooks_dispatch_exception(mocker, plugin_class):
plugin1 = plugin_class("test1", "test1")
plugin2 = plugin_class("test2", "test2")
hook = plugins.Hook("history.listenings.created", providing_args=["listening"])
plugins.hooks.register(hook)
handler1 = mocker.stub()
handler2 = mocker.Mock(side_effect=Exception("hello"))
plugin1.hooks.connect("history.listenings.created")(handler1)
plugin2.hooks.connect("history.listenings.created")(handler2)
plugins_conf = [
{"obj": plugin1, "user": {"hello": "world"}, "settings": {"foo": "bar"}},
{"obj": plugin2},
]
plugins.hooks.dispatch(
"history.listenings.created", listening="test", plugins_conf=plugins_conf
)
handler1.assert_called_once_with(listening="test", plugin_conf=plugins_conf[0])
handler2.assert_called_once_with(listening="test", plugin_conf=plugins_conf[1])