From 1cea82dc31913457c44d9b9cfd7e1d99199e3fc2 Mon Sep 17 00:00:00 2001 From: Agate Date: Tue, 16 Jun 2020 22:29:16 +0200 Subject: [PATCH] Addiional hooks, still not working wih webserver though --- api/config/plugins.py | 131 ++++++++++++++++++ api/config/settings/common.py | 76 ++++++---- api/config/settings/local.py | 8 -- api/config/settings/production.py | 2 - api/config/urls.py | 5 + api/funkwhale_api/plugins/__init__.py | 0 .../plugins/prometheus_exporter/README.md | 0 .../plugins/prometheus_exporter/__init__.py | 0 .../plugins/prometheus_exporter/entrypoint.py | 37 +++++ .../plugins/prometheus_exporter/setup.cfg | 36 +++++ .../plugins/prometheus_exporter/setup.py | 5 + 11 files changed, 264 insertions(+), 36 deletions(-) create mode 100644 api/config/plugins.py create mode 100644 api/funkwhale_api/plugins/__init__.py create mode 100644 api/funkwhale_api/plugins/prometheus_exporter/README.md create mode 100644 api/funkwhale_api/plugins/prometheus_exporter/__init__.py create mode 100644 api/funkwhale_api/plugins/prometheus_exporter/entrypoint.py create mode 100644 api/funkwhale_api/plugins/prometheus_exporter/setup.cfg create mode 100644 api/funkwhale_api/plugins/prometheus_exporter/setup.py diff --git a/api/config/plugins.py b/api/config/plugins.py new file mode 100644 index 000000000..5e3cfcdf6 --- /dev/null +++ b/api/config/plugins.py @@ -0,0 +1,131 @@ +from django import urls +from pluggy import PluginManager, HookimplMarker, HookspecMarker + +plugins_manager = PluginManager("funkwhale") +hook = HookimplMarker("funkwhale") +hookspec = HookspecMarker("funkwhale") + + +class PluginViewMiddleware: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + from django.conf import settings + + response = self.get_response(request) + if response.status_code == 404 and request.path.startswith("/plugins/"): + match = urls.resolve(request.path, urlconf=settings.PLUGINS_URLCONF) + response = match.func(request, *match.args, **match.kwargs) + return response + + +class ConfigError(ValueError): + pass + + +class Plugin: + conf = {} + + def get_conf(self): + return {"enabled": self.plugin_settings.enabled} + + def register_api_view(self, path, name=None): + def register(view): + return urls.path( + "plugins/{}/{}".format(self.name.replace("_", "-"), path), + view, + name="plugins-{}-{}".format(self.name, name), + ) + + return register + + def plugin_settings(self): + """ + Return plugin specific settings from django.conf.settings + """ + import ipdb + + ipdb.set_trace() + from django.conf import settings + + d = {} + for key in dir(settings): + k = key.lower() + if not k.startswith("plugin_{}_".format(self.name.lower())): + continue + + value = getattr(settings, key) + s_key = k.replace("plugin_{}_".format(self.name.lower()), "") + d[s_key] = value + return clean(d, self.conf, self.name) + + +def clean(d, conf, plugin_name): + cleaned = {} + for key, c in conf.items(): + if key in d: + try: + cleaned[key] = c["validator"](d[key]) + except (ValueError, TypeError, AttributeError): + raise ConfigError( + "Invalid value {} for setting {} in plugin {}".format( + d[key], key, plugin_name + ) + ) + + else: + cleaned[key] = c["default"] + + return cleaned + + +def reverse(name, **kwargs): + from django.conf import settings + + return urls.reverse(name, settings.PLUGINS_URLCONF, **kwargs) + + +def resolve(name, **kwargs): + from django.conf import settings + + return urls.resolve(name, settings.PLUGINS_URLCONF, **kwargs) + + +# def install_plugin(name_or_path): + +# subprocess.check_call([sys.executable, "-m", "pip", "install", package]) +# sub + + +class HookSpec: + @hookspec + def register_apps(self): + """ + Register additional apps in INSTALLED_APPS. + + :rvalue: list""" + + @hookspec + def middlewares_before(self): + """ + Register additional middlewares at the outer level. + + :rvalue: list""" + + @hookspec + def middlewares_after(self): + """ + Register additional middlewares at the inner level. + + :rvalue: list""" + + @hookspec + def urls(self): + """ + Register additional urls. + + :rvalue: list""" + + +plugins_manager.add_hookspecs(HookSpec()) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 0150ee09b..165759c75 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -7,18 +7,20 @@ import os import sys from urllib.parse import urlsplit - -import environ from celery.schedules import crontab - from funkwhale_api import __version__ +import environ -logger = logging.getLogger("funkwhale_api.config") ROOT_DIR = environ.Path(__file__) - 3 # (/a/b/myfile.py - 3 = /) APPS_DIR = ROOT_DIR.path("funkwhale_api") - +sys.path.append(os.path.join(APPS_DIR, "plugins")) +logger = logging.getLogger("funkwhale_api.config") env = environ.Env() +from .. import plugins # noqa + +total = plugins.plugins_manager.load_setuptools_entrypoints("funkwhale") + LOGLEVEL = env("LOGLEVEL", default="info").upper() """ Default logging level for the Funkwhale processes""" # pylint: disable=W0105 @@ -252,43 +254,64 @@ PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p] """ List of Funkwhale plugins to load. """ -if PLUGINS: - logger.info("Running with the following plugins enabled: %s", ", ".join(PLUGINS)) -else: - logger.info("Running with no plugins") ADDITIONAL_APPS = env.list("ADDITIONAL_APPS", default=[]) """ List of Django apps to load in addition to Funkwhale plugins and apps. """ + +PLUGINS_APPS = tuple() +for p in plugins.plugins_manager.hook.register_apps(): + PLUGINS_APPS += (p,) + INSTALLED_APPS = ( DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS - + tuple(["{}.apps.Plugin".format(p) for p in PLUGINS]) + tuple(ADDITIONAL_APPS) + + tuple(PLUGINS) + + tuple(PLUGINS_APPS) ) +if PLUGINS: + logger.info("Running with the following plugins enabled: %s", ", ".join(PLUGINS)) +else: + logger.info("Running with no plugins") + # MIDDLEWARE CONFIGURATION # ------------------------------------------------------------------------------ -ADDITIONAL_MIDDLEWARES_BEFORE = env.list("ADDITIONAL_MIDDLEWARES_BEFORE", default=[]) -MIDDLEWARE = tuple(ADDITIONAL_MIDDLEWARES_BEFORE) + ( - "django.middleware.security.SecurityMiddleware", - "django.middleware.clickjacking.XFrameOptionsMiddleware", - "corsheaders.middleware.CorsMiddleware", - # needs to be before SPA middleware - "django.contrib.sessions.middleware.SessionMiddleware", - "django.middleware.common.CommonMiddleware", - "django.middleware.csrf.CsrfViewMiddleware", - # /end - "funkwhale_api.common.middleware.SPAFallbackMiddleware", - "django.contrib.auth.middleware.AuthenticationMiddleware", - "django.contrib.messages.middleware.MessageMiddleware", - "funkwhale_api.users.middleware.RecordActivityMiddleware", - "funkwhale_api.common.middleware.ThrottleStatusMiddleware", - "funkwhale_api.common.plugins.PluginViewMiddleware", +ADDITIONAL_MIDDLEWARES_START = env.list("ADDITIONAL_MIDDLEWARES_START", default=[]) +for group in plugins.plugins_manager.hook.middlewares_before(): + for m in group: + ADDITIONAL_MIDDLEWARES_START.append(m) + +ADDITIONAL_MIDDLEWARES_END = env.list("ADDITIONAL_MIDDLEWARES_END", default=[]) +for group in plugins.plugins_manager.hook.middlewares_after(): + for m in group: + ADDITIONAL_MIDDLEWARES_END.append(m) + +MIDDLEWARE = ( + tuple(ADDITIONAL_MIDDLEWARES_START) + + ( + "django.middleware.security.SecurityMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "corsheaders.middleware.CorsMiddleware", + # needs to be before SPA middleware + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + # /end + "funkwhale_api.common.middleware.SPAFallbackMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "funkwhale_api.users.middleware.RecordActivityMiddleware", + "funkwhale_api.common.middleware.ThrottleStatusMiddleware", + "funkwhale_api.common.plugins.PluginViewMiddleware", + ) + + tuple(ADDITIONAL_MIDDLEWARES_END) ) + # DEBUG # ------------------------------------------------------------------------------ # See: https://docs.djangoproject.com/en/dev/ref/settings/#debug @@ -480,6 +503,7 @@ AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID", default=None) """ Access-key ID for your S3 storage. """ +SECRET_KEY = env("DJANGO_SECRET_KEY") if AWS_ACCESS_KEY_ID: AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID diff --git a/api/config/settings/local.py b/api/config/settings/local.py index b51ec273e..314430217 100644 --- a/api/config/settings/local.py +++ b/api/config/settings/local.py @@ -17,14 +17,6 @@ DEBUG = env.bool("DJANGO_DEBUG", default=True) FORCE_HTTPS_URLS = env.bool("FORCE_HTTPS_URLS", default=False) TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG -# SECRET CONFIGURATION -# ------------------------------------------------------------------------------ -# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key -# Note: This key only used for development and testing. -SECRET_KEY = env( - "DJANGO_SECRET_KEY", default="mc$&b=5j#6^bv7tld1gyjp2&+^-qrdy=0sw@r5sua*1zp4fmxc" -) - # Mail settings # ------------------------------------------------------------------------------ EMAIL_HOST = "localhost" diff --git a/api/config/settings/production.py b/api/config/settings/production.py index 1ee9b8f7e..3fd8d900a 100644 --- a/api/config/settings/production.py +++ b/api/config/settings/production.py @@ -16,8 +16,6 @@ from .common import * # noqa # SECRET CONFIGURATION # ------------------------------------------------------------------------------ # See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key -# Raises ImproperlyConfigured exception if DJANGO_SECRET_KEY not in os.environ -SECRET_KEY = env("DJANGO_SECRET_KEY") # django-secure # ------------------------------------------------------------------------------ diff --git a/api/config/urls.py b/api/config/urls.py index 2cd4f4662..776ec8302 100644 --- a/api/config/urls.py +++ b/api/config/urls.py @@ -8,6 +8,8 @@ from django.conf.urls.static import static from funkwhale_api.common import admin from django.views import defaults as default_views +from config import plugins + urlpatterns = [ # Django Admin, use {% url 'admin:index' %} @@ -24,6 +26,9 @@ urlpatterns = [ # Your stuff: custom urls includes go here ] +for group in plugins.plugins_manager.hook.urls(): + urlpatterns += group + if settings.DEBUG: # This allows the error pages to be debugged during development, just visit # these url in browser to see how these error pages look like. diff --git a/api/funkwhale_api/plugins/__init__.py b/api/funkwhale_api/plugins/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/funkwhale_api/plugins/prometheus_exporter/README.md b/api/funkwhale_api/plugins/prometheus_exporter/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/api/funkwhale_api/plugins/prometheus_exporter/__init__.py b/api/funkwhale_api/plugins/prometheus_exporter/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/funkwhale_api/plugins/prometheus_exporter/entrypoint.py b/api/funkwhale_api/plugins/prometheus_exporter/entrypoint.py new file mode 100644 index 000000000..a52e7db63 --- /dev/null +++ b/api/funkwhale_api/plugins/prometheus_exporter/entrypoint.py @@ -0,0 +1,37 @@ +import json +from django import http +from django import urls + +from config import plugins + + +class Plugin(plugins.Plugin): + name = "prometheus_exporter" + + @plugins.hook + def register_apps(self): + return "django_prometheus" + + @plugins.hook + def middlewares_before(self): + return [ + "django_prometheus.middleware.PrometheusBeforeMiddleware", + ] + + @plugins.hook + def middlewares_after(self): + return [ + "django_prometheus.middleware.PrometheusAfterMiddleware", + ] + + @plugins.hook + def urls(self): + return [urls.url(r"^plugins/prometheus/exporter/?$", prometheus)] + + +plugins.plugins_manager.register(Plugin()) + + +def prometheus(request): + stats = {"foo": "bar"} + return http.HttpResponse(json.dumps(stats)) diff --git a/api/funkwhale_api/plugins/prometheus_exporter/setup.cfg b/api/funkwhale_api/plugins/prometheus_exporter/setup.cfg new file mode 100644 index 000000000..4d0c34d57 --- /dev/null +++ b/api/funkwhale_api/plugins/prometheus_exporter/setup.cfg @@ -0,0 +1,36 @@ +[metadata] +name = funkwhale-prometheus +description = "A prometheus metric exporter for your Funkwhale pod" +version = 0.1.dev0 +author = Agate Blue +author_email = me@agate.blue +url = https://dev.funkwhale.audio/funkwhale/funkwhale +long_description = file: README.md +license = AGPL3 +classifiers = + Development Status :: 3 - Alpha + License :: OSI Approved :: AGPL + Natural Language :: English + Programming Language :: Python :: 3.6 + +[options] +zip_safe = True +include_package_data = True +packages = find: +install_requires = + django_prometheus + +[options.entry_points] +funkwhale-plugin = + prometheus = prometheus_exporter.main + + +[options.packages.find] +exclude = + tests + +[bdist_wheel] +universal = 1 + +[tool:pytest] +testpaths = tests diff --git a/api/funkwhale_api/plugins/prometheus_exporter/setup.py b/api/funkwhale_api/plugins/prometheus_exporter/setup.py new file mode 100644 index 000000000..43deb57b9 --- /dev/null +++ b/api/funkwhale_api/plugins/prometheus_exporter/setup.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from setuptools import setup + +setup()