Fixed various issue with plugins, shaping prometheus plugin

api/config/api_urls.py

@@ -13,6 +13,8 @@ from funkwhale_api.subsonic.views import SubsonicViewSet
 from funkwhale_api.tags import views as tags_views
 from funkwhale_api.users import jwt_views
 
+from config import plugins
+
 router = common_routers.OptionalSlashRouter()
 router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
@@ -98,3 +100,11 @@ v1_patterns += [
 urlpatterns = [
     url(r"^v1/", include((v1_patterns, "v1"), namespace="v1"))
 ] + format_suffix_patterns(subsonic_router.urls, allowed=["view"])
+
+plugin_urls = []
+for group in plugins.trigger_hook("urls"):
+    for u in group:
+        plugin_urls.append(u)
+urlpatterns += [
+    url("^plugins/", include((plugin_urls, "plugins"), namespace="plugins")),
+]

api/config/plugins.py

@@ -1,52 +1,27 @@
-from django import urls
+from django.apps import AppConfig
+
 from pluggy import PluginManager, HookimplMarker, HookspecMarker
 
 plugins_manager = PluginManager("funkwhale")
-hook = HookimplMarker("funkwhale")
-hookspec = HookspecMarker("funkwhale")
-
-
-class PluginViewMiddleware:
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        from django.conf import settings
-
-        response = self.get_response(request)
-        if response.status_code == 404 and request.path.startswith("/plugins/"):
-            match = urls.resolve(request.path, urlconf=settings.PLUGINS_URLCONF)
-            response = match.func(request, *match.args, **match.kwargs)
-        return response
+plugin_hook = HookimplMarker("funkwhale")
+plugin_spec = HookspecMarker("funkwhale")
 
 
 class ConfigError(ValueError):
     pass
 
 
-class Plugin:
+class Plugin(AppConfig):
     conf = {}
+    path = "noop"
 
     def get_conf(self):
         return {"enabled": self.plugin_settings.enabled}
 
-    def register_api_view(self, path, name=None):
-        def register(view):
-            return urls.path(
-                "plugins/{}/{}".format(self.name.replace("_", "-"), path),
-                view,
-                name="plugins-{}-{}".format(self.name, name),
-            )
-
-        return register
-
     def plugin_settings(self):
         """
         Return plugin specific settings from django.conf.settings
         """
-        import ipdb
-
-        ipdb.set_trace()
         from django.conf import settings
 
         d = {}
@@ -80,47 +55,34 @@ def clean(d, conf, plugin_name):
     return cleaned
 
 
-def reverse(name, **kwargs):
-    from django.conf import settings
-
-    return urls.reverse(name, settings.PLUGINS_URLCONF, **kwargs)
-
-
-def resolve(name, **kwargs):
-    from django.conf import settings
-
-    return urls.resolve(name, settings.PLUGINS_URLCONF, **kwargs)
-
-
-# def install_plugin(name_or_path):
-
-#     subprocess.check_call([sys.executable, "-m", "pip", "install", package])
-#     sub
-
-
 class HookSpec:
-    @hookspec
+    @plugin_spec
+    def database_engine(self):
+        """
+        Customize the database engine with a new class
+        """
+
+    @plugin_spec
     def register_apps(self):
         """
         Register additional apps in INSTALLED_APPS.
 
         :rvalue: list"""
 
-    @hookspec
+    @plugin_spec
     def middlewares_before(self):
         """
         Register additional middlewares at the outer level.
 
         :rvalue: list"""
 
-    @hookspec
+    @plugin_spec
     def middlewares_after(self):
         """
         Register additional middlewares at the inner level.
 
         :rvalue: list"""
 
-    @hookspec
     def urls(self):
         """
         Register additional urls.
@@ -129,3 +91,19 @@ class HookSpec:
 
 
 plugins_manager.add_hookspecs(HookSpec())
+
+
+def register(plugin_class):
+    return plugins_manager.register(plugin_class("noop", "noop"))
+
+
+def trigger_hook(name, *args, **kwargs):
+    handler = getattr(plugins_manager.hook, name)
+    return handler(*args, **kwargs)
+
+
+@register
+class DefaultPlugin(Plugin):
+    @plugin_hook
+    def database_engine(self):
+        return "django.db.backends.postgresql"

api/config/settings/common.py

@@ -6,6 +6,8 @@ import logging.config
 import os
 import sys
 
+import persisting_theory
+
 from urllib.parse import urlsplit
 from celery.schedules import crontab
 from funkwhale_api import __version__
@@ -17,9 +19,21 @@ sys.path.append(os.path.join(APPS_DIR, "plugins"))
 logger = logging.getLogger("funkwhale_api.config")
 env = environ.Env()
 
-from .. import plugins  # noqa
 
-total = plugins.plugins_manager.load_setuptools_entrypoints("funkwhale")
+class Plugins(persisting_theory.Registry):
+    look_into = "entrypoint"
+
+
+PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p]
+"""
+List of Funkwhale plugins to load.
+"""
+from config import plugins  # noqa
+
+plugins_registry = Plugins()
+plugins_registry.autodiscover(PLUGINS)
+
+# plugins.plugins_manager.register(Plugin("noop", "noop"))
 
 LOGLEVEL = env("LOGLEVEL", default="info").upper()
 """
@@ -250,18 +264,14 @@ LOCAL_APPS = (
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 
 
-PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p]
-"""
-List of Funkwhale plugins to load.
-"""
-
 ADDITIONAL_APPS = env.list("ADDITIONAL_APPS", default=[])
 """
 List of Django apps to load in addition to Funkwhale plugins and apps.
 """
 
 PLUGINS_APPS = tuple()
-for p in plugins.plugins_manager.hook.register_apps():
+
+for p in plugins.trigger_hook("register_apps"):
     PLUGINS_APPS += (p,)
 
 INSTALLED_APPS = (
@@ -281,12 +291,12 @@ else:
 # MIDDLEWARE CONFIGURATION
 # ------------------------------------------------------------------------------
 ADDITIONAL_MIDDLEWARES_START = env.list("ADDITIONAL_MIDDLEWARES_START", default=[])
-for group in plugins.plugins_manager.hook.middlewares_before():
+for group in plugins.trigger_hook("middlewares_before"):
     for m in group:
         ADDITIONAL_MIDDLEWARES_START.append(m)
 
 ADDITIONAL_MIDDLEWARES_END = env.list("ADDITIONAL_MIDDLEWARES_END", default=[])
-for group in plugins.plugins_manager.hook.middlewares_after():
+for group in plugins.trigger_hook("middlewares_after"):
     for m in group:
         ADDITIONAL_MIDDLEWARES_END.append(m)
 
@@ -306,7 +316,6 @@ MIDDLEWARE = (
         "django.contrib.messages.middleware.MessageMiddleware",
         "funkwhale_api.users.middleware.RecordActivityMiddleware",
         "funkwhale_api.common.middleware.ThrottleStatusMiddleware",
-        "funkwhale_api.common.plugins.PluginViewMiddleware",
     )
     + tuple(ADDITIONAL_MIDDLEWARES_END)
 )
@@ -394,6 +403,9 @@ DATABASES["default"]["ATOMIC_REQUESTS"] = True
 DB_CONN_MAX_AGE = DATABASES["default"]["CONN_MAX_AGE"] = env(
     "DB_CONN_MAX_AGE", default=60 * 5
 )
+
+engine = plugins.trigger_hook("database_engine")[-1]
+DATABASES["default"]["ENGINE"] = engine
 """
 Max time, in seconds, before database connections are closed.
 """

api/config/urls.py

@@ -8,8 +8,6 @@ from django.conf.urls.static import static
 from funkwhale_api.common import admin
 from django.views import defaults as default_views
 
-from config import plugins
-
 
 urlpatterns = [
     # Django Admin, use {% url 'admin:index' %}
@@ -26,9 +24,6 @@ urlpatterns = [
     # Your stuff: custom urls includes go here
 ]
 
-for group in plugins.plugins_manager.hook.urls():
-    urlpatterns += group
-
 if settings.DEBUG:
     # This allows the error pages to be debugged during development, just visit
     # these url in browser to see how these error pages look like.

api/contrib/prometheus/main.py

@@ -1,15 +0,0 @@
-import json
-
-from django import http
-
-from .plugin import PLUGIN
-
-
-@PLUGIN.register_api_view(path="prometheus")
-def prometheus(request):
-    stats = get_stats()
-    return http.HttpResponse(json.dumps(stats))
-
-
-def get_stats():
-    return {"foo": "bar"}

api/contrib/prometheus/plugin.py

@@ -1,5 +0,0 @@
-from funkwhale_api.common import plugins
-
-
-class Plugin(plugins.Plugin):
-    name = "prometheus"

api/funkwhale_api/cli/main.py

@@ -4,6 +4,7 @@ import sys
 from . import base
 from . import library  # noqa
 from . import media  # noqa
+from . import plugins  # noqa
 from . import users  # noqa
 
 from rest_framework.exceptions import ValidationError

api/funkwhale_api/cli/plugins.py

@@ -0,0 +1,37 @@
+import os
+import subprocess
+
+import click
+
+from django.conf import settings
+
+from . import base
+
+
+@base.cli.group()
+def plugins():
+    """Install, configure and remove plugins"""
+    pass
+
+
+@plugins.command("install")
+@click.argument("name_or_url", nargs=-1)
+@click.option("--builtins", is_flag=True)
+@click.option("--pip-args")
+def install(name_or_url, builtins, pip_args):
+    """
+    Installed the specified plug using their name.
+
+    If --builtins is provided, it will also install
+    plugins present at FUNKWHALE_PLUGINS_PATH
+    """
+    pip_args = pip_args or ""
+    target_path = settings.FUNKWHALE_PLUGINS_PATH
+    builtins_path = os.path.join(settings.APPS_DIR, "plugins")
+    builtins_plugins = [f.path for f in os.scandir(builtins_path) if f.is_dir()]
+    command = "pip install {} --target={} {}".format(
+        pip_args, target_path, " ".join(builtins_plugins)
+    )
+    subprocess.run(
+        command, shell=True, check=True,
+    )

api/funkwhale_api/common/plugins.py

@@ -1,45 +0,0 @@
-from django.apps import AppConfig
-from django import urls
-from django.conf import settings
-
-
-urlpatterns = []
-
-
-class PluginViewMiddleware:
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        response = self.get_response(request)
-        if response.status_code == 404 and request.path.startswith("/plugins/"):
-            match = urls.resolve(request.path, urlconf=settings.PLUGINS_URLCONF)
-            response = match.func(request, *match.args, **match.kwargs)
-        return response
-
-
-class Plugin(AppConfig):
-    def ready(self):
-        from . import main  # noqa
-
-        return super().ready()
-
-    def register_api_view(self, path, name=None):
-        def register(view):
-            urlpatterns.append(
-                urls.path(
-                    "plugins/{}/{}".format(self.name.replace("_", "-"), path),
-                    view,
-                    name="plugins-{}-{}".format(self.name, name),
-                )
-            ),
-
-        return register
-
-
-def reverse(name, **kwargs):
-    return urls.reverse(name, settings.PLUGINS_URLCONF, **kwargs)
-
-
-def resolve(name, **kwargs):
-    return urls.resolve(name, settings.PLUGINS_URLCONF, **kwargs)

api/funkwhale_api/plugins/prometheus_exporter/README.md

@@ -0,0 +1,19 @@
+Prometheus exporter for Funkwhale
+=================================
+
+Use the following prometheus config:
+
+.. code-block: yaml
+
+  global:
+    scrape_interval: 15s
+
+  scrape_configs:
+    - job_name: funkwhale
+      static_configs:
+        - targets: ['yourpod']
+      metrics_path: /api/plugins/prometheus/metrics?token=test
+
+    - job_name: prometheus
+      static_configs:
+        - targets: ['localhost:9090']

api/funkwhale_api/plugins/prometheus_exporter/entrypoint.py

@@ -1,37 +1,32 @@
-import json
-from django import http
-from django import urls
+from django.conf.urls import url, include
 
 from config import plugins
 
 
+@plugins.register
 class Plugin(plugins.Plugin):
     name = "prometheus_exporter"
 
-    @plugins.hook
+    @plugins.plugin_hook
+    def database_engine(self):
+        return "django_prometheus.db.backends.postgresql"
+
+    @plugins.plugin_hook
     def register_apps(self):
         return "django_prometheus"
 
-    @plugins.hook
+    @plugins.plugin_hook
     def middlewares_before(self):
         return [
             "django_prometheus.middleware.PrometheusBeforeMiddleware",
         ]
 
-    @plugins.hook
+    @plugins.plugin_hook
     def middlewares_after(self):
         return [
             "django_prometheus.middleware.PrometheusAfterMiddleware",
         ]
 
-    @plugins.hook
+    @plugins.plugin_hook
     def urls(self):
-        return [urls.url(r"^plugins/prometheus/exporter/?$", prometheus)]
-
-
-plugins.plugins_manager.register(Plugin())
-
-
-def prometheus(request):
-    stats = {"foo": "bar"}
-    return http.HttpResponse(json.dumps(stats))
+        return [url(r"^prometheus/", include("django_prometheus.urls"))]

api/funkwhale_api/plugins/prometheus_exporter/setup.cfg

@@ -21,8 +21,8 @@ install_requires =
     django_prometheus
 
 [options.entry_points]
-funkwhale-plugin =
-	prometheus = prometheus_exporter.main
+funkwhale =
+	prometheus = prometheus_exporter.entrypoint
 
 
 [options.packages.find]

api/tests/common/test_plugins.py

@@ -1,57 +0,0 @@
-import os
-
-import pytest
-
-from django.urls import resolvers
-
-from funkwhale_api.common import plugins
-
-
-class P(plugins.Plugin):
-    name = "test_plugin"
-    path = os.path.abspath(__file__)
-
-
-@pytest.fixture
-def plugin(settings):
-    yield P(app_name="test_plugin", app_module="tests.common.test_plugins.main.P")
-
-
-@pytest.fixture(autouse=True)
-def clear_patterns():
-    plugins.urlpatterns.clear()
-    resolvers._get_cached_resolver.cache_clear()
-    yield
-    resolvers._get_cached_resolver.cache_clear()
-
-
-def test_can_register_view(plugin, mocker, settings):
-    view = mocker.Mock()
-    plugin.register_api_view("hello", name="hello")(view)
-    expected = "/plugins/test-plugin/hello"
-    assert plugins.reverse("plugins-test_plugin-hello") == expected
-    assert plugins.resolve(expected).func == view
-
-
-def test_plugin_view_middleware_not_matching(api_client, plugin, mocker, settings):
-    view = mocker.Mock()
-    get_response = mocker.Mock()
-    middleware = plugins.PluginViewMiddleware(get_response)
-    plugin.register_api_view("hello", name="hello")(view)
-    request = mocker.Mock(path=plugins.reverse("plugins-test_plugin-hello"))
-    response = middleware(request)
-    assert response == get_response.return_value
-    view.assert_not_called()
-
-
-def test_plugin_view_middleware_matching(api_client, plugin, mocker, settings):
-    view = mocker.Mock()
-    get_response = mocker.Mock(return_value=mocker.Mock(status_code=404))
-    middleware = plugins.PluginViewMiddleware(get_response)
-    plugin.register_api_view("hello/<slug:slug>", name="hello")(view)
-    request = mocker.Mock(
-        path=plugins.reverse("plugins-test_plugin-hello", kwargs={"slug": "world"})
-    )
-    response = middleware(request)
-    assert response == view.return_value
-    view.assert_called_once_with(request, slug="world")