Merge branch 'develop'

.gitlab-ci.yml

@@ -1,8 +1,10 @@
 variables:
   IMAGE_NAME: funkwhale/funkwhale
   IMAGE: $IMAGE_NAME:$CI_COMMIT_REF_NAME
+  IMAGE_LATEST: $IMAGE_NAME:latest
   ALL_IN_ONE_IMAGE_NAME: funkwhale/all-in-one
   ALL_IN_ONE_IMAGE: $ALL_IN_ONE_IMAGE_NAME:$CI_COMMIT_REF_NAME
+  ALL_IN_ONE_IMAGE_LATEST: $ALL_IN_ONE_IMAGE_NAME:latest
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
   PYTHONDONTWRITEBYTECODE: "true"
   REVIEW_DOMAIN: preview.funkwhale.audio
@@ -16,6 +18,7 @@ stages:
   - deploy
 
 review_front:
+  interruptible: true
   stage: review
   image: node:11
   when: manual
@@ -54,8 +57,8 @@ review_front:
     name: review/front/$CI_COMMIT_REF_NAME
     url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/index.html
 
-
 review_docs:
+  interruptible: true
   stage: review
   when: manual
   allow_failure: true
@@ -68,7 +71,7 @@ review_docs:
     - cd docs
     - apt-get update
     - apt-get install -y graphviz
-    - pip install sphinx
+    - pip install sphinx sphinx_rtd_theme
   script:
     - ./build_docs.sh
   cache:
@@ -87,8 +90,8 @@ review_docs:
     name: review/docs/$CI_COMMIT_REF_NAME
     url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/docs-review/index.html
 
-
 black:
+  interruptible: true
   image: python:3.6
   stage: lint
   variables:
@@ -99,6 +102,7 @@ black:
     - black --check --diff api/
 
 flake8:
+  interruptible: true
   image: python:3.6
   stage: lint
   variables:
@@ -113,6 +117,7 @@ flake8:
       - "$PIP_CACHE_DIR"
 
 test_api:
+  interruptible: true
   services:
     - postgres:11
     - redis:3
@@ -129,6 +134,7 @@ test_api:
   only:
     - branches
   before_script:
+    - apk add make
     - cd api
     - sed -i '/Pillow/d' requirements/base.txt
     - pip3 install -r requirements/base.txt
@@ -140,6 +146,7 @@ test_api:
     - docker
 
 test_front:
+  interruptible: true
   stage: test
   image: node:11
   before_script:
@@ -196,7 +203,7 @@ pages:
     - cd docs
     - apt-get update
     - apt-get install -y graphviz
-    - pip install sphinx
+    - pip install sphinx sphinx_rtd_theme
   script:
     - ./build_docs.sh
   cache:
@@ -218,10 +225,12 @@ docker_release:
     - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
     - cp -r front/dist api/frontend
     - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
-    - cd api
   script:
-    - docker build -t $IMAGE .
+    - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
+    - cd api
+    - docker build -t $IMAGE $DOCKER_LATEST_TAG .
     - docker push $IMAGE
+    - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $IMAGE_LATEST; fi
   only:
     - develop@funkwhale/funkwhale
     - tags@funkwhale/funkwhale
@@ -239,6 +248,7 @@ docker_all_in_one_release:
     - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
     - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
   script:
+    - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
     - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
     - unzip -o all_in_one.zip -d tmpdir
     - mv tmpdir/docker-funkwhale-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
@@ -246,8 +256,9 @@ docker_all_in_one_release:
     - cp -r front $BUILD_PATH/src/front
     - cd $BUILD_PATH
     - ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
-    - docker build -t $ALL_IN_ONE_IMAGE .
+    - docker build -t $ALL_IN_ONE_IMAGE $DOCKER_LATEST_TAG .
     - docker push $ALL_IN_ONE_IMAGE
+    - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi
   only:
     - develop@funkwhale/funkwhale
     - tags@funkwhale/funkwhale

CONTRIBUTING.rst

@@ -185,9 +185,9 @@ If you prefer to launch them in the background instead, use the ``-d`` flag, and
 
 Once everything is up, you can access the various funkwhale's components:
 
-- The Vue webapp, on http://localhost:8080
-- The API, on http://localhost:8080/api/v1/
-- The django admin, on http://localhost:8080/api/admin/
+- The Vue webapp, on http://localhost:8000
+- The API, on http://localhost:8000/api/v1/
+- The django admin, on http://localhost:800/api/admin/
 
 Stopping everything
 ^^^^^^^^^^^^^^^^^^^

api/Dockerfile

@@ -17,6 +17,7 @@ RUN \
     libpq              \
     libmagic           \
     libffi-dev         \
+    make         \
     zlib-dev           \
     openldap-dev && \
     \
@@ -44,7 +45,7 @@ RUN \
     if [ "$install_dev_deps" = "1" ] ; then echo "Installing dev dependencies" && pip3 install --no-cache-dir -r /requirements/local.txt -r /requirements/test.txt ; else echo "Skipping dev deps installation" ; fi
 
 ENTRYPOINT ["./compose/django/entrypoint.sh"]
-CMD ["./compose/django/daphne.sh"]
+CMD ["./compose/django/server.sh"]
 
 COPY . /app
 WORKDIR /app

api/compose/django/daphne.sh

@@ -1,3 +0,0 @@
-#!/bin/bash -eux
-python /app/manage.py collectstatic --noinput
-daphne -b 0.0.0.0 -p 5000 config.asgi:application --proxy-headers

api/compose/django/server.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -eux
+python /app/manage.py collectstatic --noinput
+gunicorn config.asgi:application -w ${FUNKWHALE_WEB_WORKERS-1} -k uvicorn.workers.UvicornWorker -b 0.0.0.0:5000

api/config/api_urls.py

@@ -2,18 +2,20 @@ from django.conf.urls import include, url
 from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet
 from rest_framework import routers
 from rest_framework.urlpatterns import format_suffix_patterns
-from rest_framework_jwt import views as jwt_views
 
 from funkwhale_api.activity import views as activity_views
 from funkwhale_api.common import views as common_views
+from funkwhale_api.common import routers as common_routers
 from funkwhale_api.music import views
 from funkwhale_api.playlists import views as playlists_views
 from funkwhale_api.subsonic.views import SubsonicViewSet
+from funkwhale_api.tags import views as tags_views
+from funkwhale_api.users import jwt_views
 
-router = routers.SimpleRouter()
+router = common_routers.OptionalSlashRouter()
 router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
-router.register(r"tags", views.TagViewSet, "tags")
+router.register(r"tags", tags_views.TagViewSet, "tags")
 router.register(r"tracks", views.TrackViewSet, "tracks")
 router.register(r"uploads", views.UploadViewSet, "uploads")
 router.register(r"libraries", views.LibraryViewSet, "libraries")
@@ -79,8 +81,9 @@ v1_patterns += [
         r"^oauth/",
         include(("funkwhale_api.users.oauth.urls", "oauth"), namespace="oauth"),
     ),
-    url(r"^token/$", jwt_views.obtain_jwt_token, name="token"),
-    url(r"^token/refresh/$", jwt_views.refresh_jwt_token, name="token_refresh"),
+    url(r"^token/?$", jwt_views.obtain_jwt_token, name="token"),
+    url(r"^token/refresh/?$", jwt_views.refresh_jwt_token, name="token_refresh"),
+    url(r"^rate-limit/?$", common_views.RateLimitView.as_view(), name="rate-limit"),
 ]
 
 urlpatterns = [

api/config/settings/common.py

@@ -11,7 +11,8 @@ https://docs.djangoproject.com/en/dev/ref/settings/
 from __future__ import absolute_import, unicode_literals
 
 import datetime
-import logging
+import logging.config
+import sys
 
 from urllib.parse import urlsplit
 
@@ -20,13 +21,44 @@ from celery.schedules import crontab
 
 from funkwhale_api import __version__
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("funkwhale_api.config")
 ROOT_DIR = environ.Path(__file__) - 3  # (/a/b/myfile.py - 3 = /)
 APPS_DIR = ROOT_DIR.path("funkwhale_api")
 
 env = environ.Env()
+
+LOGLEVEL = env("LOGLEVEL", default="info").upper()
+LOGGING_CONFIG = None
+logging.config.dictConfig(
+    {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "console": {"format": "%(asctime)s %(name)-12s %(levelname)-8s %(message)s"}
+        },
+        "handlers": {
+            "console": {"class": "logging.StreamHandler", "formatter": "console"},
+            # # Add Handler for Sentry for `warning` and above
+            # 'sentry': {
+            #     'level': 'WARNING',
+            #     'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
+            # },
+        },
+        "loggers": {
+            "funkwhale_api": {
+                "level": LOGLEVEL,
+                "handlers": ["console"],
+                # required to avoid double logging with root logger
+                "propagate": False,
+            },
+            "": {"level": "WARNING", "handlers": ["console"]},
+        },
+    }
+)
+
 env_file = env("ENV_FILE", default=None)
 if env_file:
+    logger.info("Loading specified env file at %s", env_file)
     # we have an explicitely specified env file
     # so we try to load and it fail loudly if it does not exist
     env.read_env(env_file)
@@ -49,6 +81,11 @@ else:
         logger.info("Loaded env file at %s/.env", path)
         break
 
+FUNKWHALE_PLUGINS_PATH = env(
+    "FUNKWHALE_PLUGINS_PATH", default="/srv/funkwhale/plugins/"
+)
+sys.path.append(FUNKWHALE_PLUGINS_PATH)
+
 FUNKWHALE_HOSTNAME = None
 FUNKWHALE_HOSTNAME_SUFFIX = env("FUNKWHALE_HOSTNAME_SUFFIX", default=None)
 FUNKWHALE_HOSTNAME_PREFIX = env("FUNKWHALE_HOSTNAME_PREFIX", default=None)
@@ -124,7 +161,6 @@ THIRD_PARTY_APPS = (
     "oauth2_provider",
     "rest_framework",
     "rest_framework.authtoken",
-    "taggit",
     "rest_auth",
     "rest_auth.registration",
     "dynamic_preferences",
@@ -147,7 +183,6 @@ if RAVEN_ENABLED:
     }
     THIRD_PARTY_APPS += ("raven.contrib.django.raven_compat",)
 
-
 # Apps specific for this project go here.
 LOCAL_APPS = (
     "funkwhale_api.common.apps.CommonConfig",
@@ -160,29 +195,44 @@ LOCAL_APPS = (
     "funkwhale_api.requests",
     "funkwhale_api.favorites",
     "funkwhale_api.federation",
-    "funkwhale_api.moderation",
+    "funkwhale_api.moderation.apps.ModerationConfig",
     "funkwhale_api.radios",
     "funkwhale_api.history",
     "funkwhale_api.playlists",
     "funkwhale_api.subsonic",
+    "funkwhale_api.tags",
 )
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 
-INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
+
+PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p]
+if PLUGINS:
+    logger.info("Running with the following plugins enabled: %s", ", ".join(PLUGINS))
+else:
+    logger.info("Running with no plugins")
+
+INSTALLED_APPS = (
+    DJANGO_APPS
+    + THIRD_PARTY_APPS
+    + LOCAL_APPS
+    + tuple(["{}.apps.Plugin".format(p) for p in PLUGINS])
+)
 
 # MIDDLEWARE CONFIGURATION
 # ------------------------------------------------------------------------------
 MIDDLEWARE = (
+    "django.middleware.security.SecurityMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "corsheaders.middleware.CorsMiddleware",
     "funkwhale_api.common.middleware.SPAFallbackMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
-    "corsheaders.middleware.CorsMiddleware",
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
-    "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "funkwhale_api.users.middleware.RecordActivityMiddleware",
+    "funkwhale_api.common.middleware.ThrottleStatusMiddleware",
 )
 
 # DEBUG
@@ -350,6 +400,8 @@ ASGI_APPLICATION = "config.routing.application"
 
 # This ensures that Django will be able to detect a secure connection
 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+SECURE_BROWSER_XSS_FILTER = True
+SECURE_CONTENT_TYPE_NOSNIFF = True
 
 # AUTHENTICATION CONFIGURATION
 # ------------------------------------------------------------------------------
@@ -516,14 +568,32 @@ CELERY_BEAT_SCHEDULE = {
 
 NODEINFO_REFRESH_DELAY = env.int("NODEINFO_REFRESH_DELAY", default=3600 * 24)
 
+
+def get_user_secret_key(user):
+    from django.conf import settings
+
+    return settings.SECRET_KEY + str(user.secret_key)
+
+
 JWT_AUTH = {
     "JWT_ALLOW_REFRESH": True,
     "JWT_EXPIRATION_DELTA": datetime.timedelta(days=7),
     "JWT_REFRESH_EXPIRATION_DELTA": datetime.timedelta(days=30),
     "JWT_AUTH_HEADER_PREFIX": "JWT",
-    "JWT_GET_USER_SECRET_KEY": lambda user: user.secret_key,
+    "JWT_GET_USER_SECRET_KEY": get_user_secret_key,
 }
 OLD_PASSWORD_FIELD_ENABLED = True
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+        "OPTIONS": {"min_length": env.int("PASSWORD_MIN_LENGTH", default=8)},
+    },
+    {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
+    {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
+]
 ACCOUNT_ADAPTER = "funkwhale_api.users.adapters.FunkwhaleAccountAdapter"
 CORS_ORIGIN_ALLOW_ALL = True
 # CORS_ORIGIN_WHITELIST = (
@@ -557,7 +627,150 @@ REST_FRAMEWORK = {
         "django_filters.rest_framework.DjangoFilterBackend",
     ),
     "DEFAULT_RENDERER_CLASSES": ("rest_framework.renderers.JSONRenderer",),
+    "NUM_PROXIES": env.int("NUM_PROXIES", default=1),
 }
+THROTTLING_ENABLED = env.bool("THROTTLING_ENABLED", default=True)
+if THROTTLING_ENABLED:
+    REST_FRAMEWORK["DEFAULT_THROTTLE_CLASSES"] = env.list(
+        "THROTTLE_CLASSES",
+        default=["funkwhale_api.common.throttling.FunkwhaleThrottle"],
+    )
+
+THROTTLING_SCOPES = {
+    "*": {"anonymous": "anonymous-wildcard", "authenticated": "authenticated-wildcard"},
+    "create": {
+        "authenticated": "authenticated-create",
+        "anonymous": "anonymous-create",
+    },
+    "list": {"authenticated": "authenticated-list", "anonymous": "anonymous-list"},
+    "retrieve": {
+        "authenticated": "authenticated-retrieve",
+        "anonymous": "anonymous-retrieve",
+    },
+    "destroy": {
+        "authenticated": "authenticated-destroy",
+        "anonymous": "anonymous-destroy",
+    },
+    "update": {
+        "authenticated": "authenticated-update",
+        "anonymous": "anonymous-update",
+    },
+    "partial_update": {
+        "authenticated": "authenticated-update",
+        "anonymous": "anonymous-update",
+    },
+}
+
+THROTTLING_USER_RATES = env.dict("THROTTLING_RATES", default={})
+
+THROTTLING_RATES = {
+    "anonymous-wildcard": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-wildcard", "1000/h"),
+        "description": "Anonymous requests not covered by other limits",
+    },
+    "authenticated-wildcard": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-wildcard", "2000/h"),
+        "description": "Authenticated requests not covered by other limits",
+    },
+    "authenticated-create": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-create", "1000/hour"),
+        "description": "Authenticated POST requests",
+    },
+    "anonymous-create": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-create", "1000/day"),
+        "description": "Anonymous POST requests",
+    },
+    "authenticated-list": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-list", "10000/hour"),
+        "description": "Authenticated GET requests on resource lists",
+    },
+    "anonymous-list": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-list", "10000/day"),
+        "description": "Anonymous GET requests on resource lists",
+    },
+    "authenticated-retrieve": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-retrieve", "10000/hour"),
+        "description": "Authenticated GET requests on resource detail",
+    },
+    "anonymous-retrieve": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-retrieve", "10000/day"),
+        "description": "Anonymous GET requests on resource detail",
+    },
+    "authenticated-destroy": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-destroy", "500/hour"),
+        "description": "Authenticated DELETE requests on resource detail",
+    },
+    "anonymous-destroy": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-destroy", "1000/day"),
+        "description": "Anonymous DELETE requests on resource detail",
+    },
+    "authenticated-update": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-update", "1000/hour"),
+        "description": "Authenticated PATCH and PUT requests on resource detail",
+    },
+    "anonymous-update": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-update", "1000/day"),
+        "description": "Anonymous PATCH and PUT requests on resource detail",
+    },
+    # potentially spammy / dangerous endpoints
+    "authenticated-reports": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-reports", "100/day"),
+        "description": "Authenticated report submission",
+    },
+    "anonymous-reports": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-reports", "10/day"),
+        "description": "Anonymous report submission",
+    },
+    "authenticated-oauth-app": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-oauth-app", "10/hour"),
+        "description": "Authenticated OAuth app creation",
+    },
+    "anonymous-oauth-app": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-oauth-app", "10/day"),
+        "description": "Anonymous OAuth app creation",
+    },
+    "oauth-authorize": {
+        "rate": THROTTLING_USER_RATES.get("oauth-authorize", "100/hour"),
+        "description": "OAuth app authorization",
+    },
+    "oauth-token": {
+        "rate": THROTTLING_USER_RATES.get("oauth-token", "100/hour"),
+        "description": "OAuth token creation",
+    },
+    "oauth-revoke-token": {
+        "rate": THROTTLING_USER_RATES.get("oauth-revoke-token", "100/hour"),
+        "description": "OAuth token deletion",
+    },
+    "jwt-login": {
+        "rate": THROTTLING_USER_RATES.get("jwt-login", "30/hour"),
+        "description": "JWT token creation",
+    },
+    "jwt-refresh": {
+        "rate": THROTTLING_USER_RATES.get("jwt-refresh", "30/hour"),
+        "description": "JWT token refresh",
+    },
+    "signup": {
+        "rate": THROTTLING_USER_RATES.get("signup", "10/day"),
+        "description": "Account creation",
+    },
+    "verify-email": {
+        "rate": THROTTLING_USER_RATES.get("verify-email", "20/h"),
+        "description": "Email address confirmation",
+    },
+    "password-change": {
+        "rate": THROTTLING_USER_RATES.get("password-change", "20/h"),
+        "description": "Password change (when authenticated)",
+    },
+    "password-reset": {
+        "rate": THROTTLING_USER_RATES.get("password-reset", "20/h"),
+        "description": "Password reset request",
+    },
+    "password-reset-confirm": {
+        "rate": THROTTLING_USER_RATES.get("password-reset-confirm", "20/h"),
+        "description": "Password reset confirmation",
+    },
+}
+
 
 BROWSABLE_API_ENABLED = env.bool("BROWSABLE_API_ENABLED", default=False)
 if BROWSABLE_API_ENABLED:
@@ -661,3 +874,17 @@ ACTOR_KEY_ROTATION_DELAY = env.int("ACTOR_KEY_ROTATION_DELAY", default=3600 * 48
 SUBSONIC_DEFAULT_TRANSCODING_FORMAT = (
     env("SUBSONIC_DEFAULT_TRANSCODING_FORMAT", default="mp3") or None
 )
+
+# extra tags will be ignored
+TAGS_MAX_BY_OBJ = env.int("TAGS_MAX_BY_OBJ", default=30)
+FEDERATION_OBJECT_FETCH_DELAY = env.int(
+    "FEDERATION_OBJECT_FETCH_DELAY", default=60 * 24 * 3
+)
+
+MODERATION_EMAIL_NOTIFICATIONS_ENABLED = env.bool(
+    "MODERATION_EMAIL_NOTIFICATIONS_ENABLED", default=True
+)
+
+# Delay in days after signup before we show the "support us" messages
+INSTANCE_SUPPORT_MESSAGE_DELAY = env.int("INSTANCE_SUPPORT_MESSAGE_DELAY", default=15)
+FUNKWHALE_SUPPORT_MESSAGE_DELAY = env.int("FUNKWHALE_SUPPORT_MESSAGE_DELAY", default=15)

api/config/settings/production.py

@@ -73,50 +73,4 @@ TEMPLATES[0]["OPTIONS"]["loaders"] = [
 # ------------------------------------------------------------------------------
 # Heroku URL does not pass the DB number, so we parse it in
 
-
-# LOGGING CONFIGURATION
-# ------------------------------------------------------------------------------
-# See: https://docs.djangoproject.com/en/dev/ref/settings/#logging
-# A sample logging configuration. The only tangible logging
-# performed by this configuration is to send an email to
-# the site admins on every HTTP 500 error when DEBUG=False.
-# See http://docs.djangoproject.com/en/dev/topics/logging for
-# more details on how to customize your logging configuration.
-LOGGING = {
-    "version": 1,
-    "disable_existing_loggers": False,
-    "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}},
-    "formatters": {
-        "verbose": {
-            "format": "%(levelname)s %(asctime)s %(module)s "
-            "%(process)d %(thread)d %(message)s"
-        }
-    },
-    "handlers": {
-        "mail_admins": {
-            "level": "ERROR",
-            "filters": ["require_debug_false"],
-            "class": "django.utils.log.AdminEmailHandler",
-        },
-        "console": {
-            "level": "DEBUG",
-            "class": "logging.StreamHandler",
-            "formatter": "verbose",
-        },
-    },
-    "loggers": {
-        "django.request": {
-            "handlers": ["mail_admins"],
-            "level": "ERROR",
-            "propagate": True,
-        },
-        "django.security.DisallowedHost": {
-            "level": "ERROR",
-            "handlers": ["console", "mail_admins"],
-            "propagate": True,
-        },
-    },
-}
-
-
 # Your production stuff: Below this line define 3rd party library settings

api/config/spa_urls.py

@@ -15,4 +15,9 @@ urlpatterns = [
         spa_views.library_artist,
         name="library_artist",
     ),
+    urls.re_path(
+        r"^library/playlists/(?P<pk>\d+)/?$",
+        spa_views.library_playlist,
+        name="library_playlist",
+    ),
 ]

api/config/urls.py

@@ -19,8 +19,7 @@ urlpatterns = [
             ("funkwhale_api.federation.urls", "federation"), namespace="federation"
         ),
     ),
-    url(r"^api/v1/auth/", include("rest_auth.urls")),
-    url(r"^api/v1/auth/registration/", include("funkwhale_api.users.rest_auth_urls")),
+    url(r"^api/v1/auth/", include("funkwhale_api.users.rest_auth_urls")),
     url(r"^accounts/", include("allauth.urls")),
     # Your stuff: custom urls includes go here
 ]

api/funkwhale_api/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = "0.19.1"
+__version__ = "0.20.0-rc1"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

api/funkwhale_api/common/fields.py

@@ -1,7 +1,10 @@
 import django_filters
 from django import forms
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 
+from rest_framework import serializers
+
 from . import search
 
 PRIVACY_LEVEL_CHOICES = [
@@ -52,3 +55,118 @@ class SmartSearchFilter(django_filters.CharFilter):
         except (forms.ValidationError):
             return qs.none()
         return search.apply(qs, cleaned)
+
+
+def get_generic_filter_query(value, relation_name, choices):
+    parts = value.split(":", 1)
+    type = parts[0]
+    try:
+        conf = choices[type]
+    except KeyError:
+        raise forms.ValidationError("Invalid type")
+    related_queryset = conf["queryset"]
+    related_model = related_queryset.model
+    filter_query = models.Q(
+        **{
+            "{}_content_type__app_label".format(
+                relation_name
+            ): related_model._meta.app_label,
+            "{}_content_type__model".format(
+                relation_name
+            ): related_model._meta.model_name,
+        }
+    )
+    if len(parts) > 1:
+        id_attr = conf.get("id_attr", "id")
+        id_field = conf.get("id_field", serializers.IntegerField(min_value=1))
+        try:
+            id_value = parts[1]
+            id_value = id_field.to_internal_value(id_value)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise forms.ValidationError("Invalid id")
+        query_getter = conf.get(
+            "get_query", lambda attr, value: models.Q(**{attr: value})
+        )
+        obj_query = query_getter(id_attr, id_value)
+        try:
+            obj = related_queryset.get(obj_query)
+        except related_queryset.model.DoesNotExist:
+            raise forms.ValidationError("Invalid object")
+        filter_query &= models.Q(**{"{}_id".format(relation_name): obj.id})
+
+    return filter_query
+
+
+class GenericRelationFilter(django_filters.CharFilter):
+    def __init__(self, relation_name, choices, *args, **kwargs):
+        self.relation_name = relation_name
+        self.choices = choices
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if not value:
+            return qs
+        try:
+            filter_query = get_generic_filter_query(
+                value, relation_name=self.relation_name, choices=self.choices
+            )
+        except forms.ValidationError:
+            return qs.none()
+        return qs.filter(filter_query)
+
+
+class GenericRelation(serializers.JSONField):
+    def __init__(self, choices, *args, **kwargs):
+        self.choices = choices
+        self.encoder = kwargs.setdefault("encoder", DjangoJSONEncoder)
+        super().__init__(*args, **kwargs)
+
+    def to_representation(self, value):
+        if not value:
+            return
+        type = None
+        id = None
+        id_attr = None
+        for key, choice in self.choices.items():
+            if isinstance(value, choice["queryset"].model):
+                type = key
+                id_attr = choice.get("id_attr", "id")
+                id = getattr(value, id_attr)
+                break
+
+        if type:
+            return {"type": type, id_attr: id}
+
+    def to_internal_value(self, v):
+        v = super().to_internal_value(v)
+
+        if not v or not isinstance(v, dict):
+            raise serializers.ValidationError("Invalid data")
+
+        try:
+            type = v["type"]
+            field = serializers.ChoiceField(choices=list(self.choices.keys()))
+            type = field.to_internal_value(type)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise serializers.ValidationError("Invalid type")
+
+        conf = self.choices[type]
+        id_attr = conf.get("id_attr", "id")
+        id_field = conf.get("id_field", serializers.IntegerField(min_value=1))
+        queryset = conf["queryset"]
+        try:
+            id_value = v[id_attr]
+            id_value = id_field.to_internal_value(id_value)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise serializers.ValidationError("Invalid {}".format(id_attr))
+
+        query_getter = conf.get(
+            "get_query", lambda attr, value: models.Q(**{attr: value})
+        )
+        query = query_getter(id_attr, id_value)
+        try:
+            obj = queryset.get(query)
+        except queryset.model.DoesNotExist:
+            raise serializers.ValidationError("Object not found")
+
+        return obj

api/funkwhale_api/common/filters.py

@@ -15,7 +15,7 @@ class NoneObject(object):
 
 
 NONE = NoneObject()
-NULL_BOOLEAN_CHOICES = [
+BOOLEAN_CHOICES = [
     (True, True),
     ("true", True),
     ("True", True),
@@ -26,6 +26,8 @@ NULL_BOOLEAN_CHOICES = [
     ("False", False),
     ("0", False),
     ("no", False),
+]
+NULL_BOOLEAN_CHOICES = BOOLEAN_CHOICES + [
     ("None", NONE),
     ("none", NONE),
     ("Null", NONE),
@@ -76,10 +78,26 @@ def clean_null_boolean_filter(v):
     return v
 
 
+def clean_boolean_filter(v):
+    return CoerceChoiceField(choices=BOOLEAN_CHOICES).clean(v)
+
+
 def get_null_boolean_filter(name):
     return {"handler": lambda v: Q(**{name: clean_null_boolean_filter(v)})}
 
 
+def get_boolean_filter(name):
+    return {"handler": lambda v: Q(**{name: clean_boolean_filter(v)})}
+
+
+def get_generic_relation_filter(relation_name, choices):
+    return {
+        "handler": lambda v: fields.get_generic_filter_query(
+            v, relation_name=relation_name, choices=choices
+        )
+    }
+
+
 class DummyTypedMultipleChoiceField(forms.TypedMultipleChoiceField):
     def valid_value(self, value):
         return True
@@ -142,7 +160,7 @@ class MutationFilter(filters.FilterSet):
                 "domain": {"to": "created_by__domain__name__iexact"},
                 "is_approved": get_null_boolean_filter("is_approved"),
                 "target": {"handler": filter_target},
-                "is_applied": {"to": "is_applied"},
+                "is_applied": get_boolean_filter("is_applied"),
             },
         )
     )

api/funkwhale_api/common/management/commands/load_test_data.py

@@ -0,0 +1,325 @@
+import math
+import random
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+
+from funkwhale_api.federation import keys
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.music import models as music_models
+from funkwhale_api.tags import models as tags_models
+from funkwhale_api.users import models as users_models
+
+
+BATCH_SIZE = 500
+
+
+def create_local_accounts(factories, count, dependencies):
+    password = factories["users.User"].build().password
+    users = factories["users.User"].build_batch(size=count)
+    for user in users:
+        # we set the hashed password by hand, because computing one for each user
+        # is CPU intensive
+        user.password = password
+    users = users_models.User.objects.bulk_create(users, batch_size=BATCH_SIZE)
+    actors = []
+    domain = federation_models.Domain.objects.get_or_create(
+        name=settings.FEDERATION_HOSTNAME
+    )[0]
+    users = [u for u in users if u.pk]
+    private, public = keys.get_key_pair()
+    for user in users:
+        if not user.pk:
+            continue
+        actor = federation_models.Actor(
+            private_key=private.decode("utf-8"),
+            public_key=public.decode("utf-8"),
+            **users_models.get_actor_data(user.username, domain=domain)
+        )
+        actors.append(actor)
+    actors = federation_models.Actor.objects.bulk_create(actors, batch_size=BATCH_SIZE)
+    for user, actor in zip(users, actors):
+        user.actor = actor
+    users_models.User.objects.bulk_update(users, ["actor"])
+    return actors
+
+
+def create_taggable_items(dependency):
+    def inner(factories, count, dependencies):
+
+        objs = []
+        tagged_objects = dependencies.get(
+            dependency, list(CONFIG_BY_ID[dependency]["model"].objects.all().only("pk"))
+        )
+        tags = dependencies.get("tags", list(tags_models.Tag.objects.all().only("pk")))
+        for i in range(count):
+            tag = random.choice(tags)
+            tagged_object = random.choice(tagged_objects)
+            objs.append(
+                factories["tags.TaggedItem"].build(
+                    content_object=tagged_object, tag=tag
+                )
+            )
+
+        return tags_models.TaggedItem.objects.bulk_create(
+            objs, batch_size=BATCH_SIZE, ignore_conflicts=True
+        )
+
+    return inner
+
+
+CONFIG = [
+    {
+        "id": "tracks",
+        "model": music_models.Track,
+        "factory": "music.Track",
+        "factory_kwargs": {"artist": None, "album": None},
+        "depends_on": [
+            {"field": "album", "id": "albums", "default_factor": 0.1},
+            {"field": "artist", "id": "artists", "default_factor": 0.05},
+        ],
+    },
+    {
+        "id": "albums",
+        "model": music_models.Album,
+        "factory": "music.Album",
+        "factory_kwargs": {"artist": None},
+        "depends_on": [{"field": "artist", "id": "artists", "default_factor": 0.3}],
+    },
+    {"id": "artists", "model": music_models.Artist, "factory": "music.Artist"},
+    {
+        "id": "local_accounts",
+        "model": federation_models.Actor,
+        "handler": create_local_accounts,
+    },
+    {
+        "id": "local_libraries",
+        "model": music_models.Library,
+        "factory": "music.Library",
+        "factory_kwargs": {"actor": None},
+        "depends_on": [{"field": "actor", "id": "local_accounts", "default_factor": 1}],
+    },
+    {
+        "id": "local_uploads",
+        "model": music_models.Upload,
+        "factory": "music.Upload",
+        "factory_kwargs": {"import_status": "finished", "library": None, "track": None},
+        "depends_on": [
+            {
+                "field": "library",
+                "id": "local_libraries",
+                "default_factor": 0.05,
+                "queryset": music_models.Library.objects.all().select_related(
+                    "actor__user"
+                ),
+            },
+            {"field": "track", "id": "tracks", "default_factor": 1},
+        ],
+    },
+    {"id": "tags", "model": tags_models.Tag, "factory": "tags.Tag"},
+    {
+        "id": "track_tags",
+        "model": tags_models.TaggedItem,
+        "queryset": tags_models.TaggedItem.objects.filter(
+            content_type__app_label="music", content_type__model="track"
+        ),
+        "handler": create_taggable_items("tracks"),
+        "depends_on": [
+            {
+                "field": "tag",
+                "id": "tags",
+                "default_factor": 0.1,
+                "queryset": tags_models.Tag.objects.all(),
+                "set": False,
+            },
+            {
+                "field": "content_object",
+                "id": "tracks",
+                "default_factor": 1,
+                "set": False,
+            },
+        ],
+    },
+    {
+        "id": "album_tags",
+        "model": tags_models.TaggedItem,
+        "queryset": tags_models.TaggedItem.objects.filter(
+            content_type__app_label="music", content_type__model="album"
+        ),
+        "handler": create_taggable_items("albums"),
+        "depends_on": [
+            {
+                "field": "tag",
+                "id": "tags",
+                "default_factor": 0.1,
+                "queryset": tags_models.Tag.objects.all(),
+                "set": False,
+            },
+            {
+                "field": "content_object",
+                "id": "albums",
+                "default_factor": 1,
+                "set": False,
+            },
+        ],
+    },
+    {
+        "id": "artist_tags",
+        "model": tags_models.TaggedItem,
+        "queryset": tags_models.TaggedItem.objects.filter(
+            content_type__app_label="music", content_type__model="artist"
+        ),
+        "handler": create_taggable_items("artists"),
+        "depends_on": [
+            {
+                "field": "tag",
+                "id": "tags",
+                "default_factor": 0.1,
+                "queryset": tags_models.Tag.objects.all(),
+                "set": False,
+            },
+            {
+                "field": "content_object",
+                "id": "artists",
+                "default_factor": 1,
+                "set": False,
+            },
+        ],
+    },
+]
+
+CONFIG_BY_ID = {c["id"]: c for c in CONFIG}
+
+
+class Rollback(Exception):
+    pass
+
+
+def create_objects(row, factories, count, **factory_kwargs):
+    return factories[row["factory"]].build_batch(size=count, **factory_kwargs)
+
+
+class Command(BaseCommand):
+    help = """
+    Inject demo data into your database. Useful for load testing, or setting up a demo instance.
+
+    Use with caution and only if you know what you are doing.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--no-dry-run",
+            action="store_false",
+            dest="dry_run",
+            help="Commit the changes to the database",
+        )
+        parser.add_argument(
+            "--create-dependencies", action="store_true", dest="create_dependencies"
+        )
+        for row in CONFIG:
+            parser.add_argument(
+                "--{}".format(row["id"].replace("_", "-")),
+                dest=row["id"],
+                type=int,
+                help="Number of {} objects to create".format(row["id"]),
+            )
+            dependencies = row.get("depends_on", [])
+            for dependency in dependencies:
+                parser.add_argument(
+                    "--{}-{}-factor".format(row["id"], dependency["field"]),
+                    dest="{}_{}_factor".format(row["id"], dependency["field"]),
+                    type=float,
+                    help="Number of {} objects to create per {} object".format(
+                        dependency["id"], row["id"]
+                    ),
+                )
+
+    def handle(self, *args, **options):
+        from django.apps import apps
+        from funkwhale_api import factories
+
+        app_names = [app.name for app in apps.app_configs.values()]
+        factories.registry.autodiscover(app_names)
+        try:
+            return self.inner_handle(*args, **options)
+        except Rollback:
+            pass
+
+    @transaction.atomic
+    def inner_handle(self, *args, **options):
+        results = {}
+        for row in CONFIG:
+            self.create_batch(row, results, options, count=options.get(row["id"]))
+
+        self.stdout.write("\nFinal state of database:\n\n")
+        for row in CONFIG:
+            qs = row.get("queryset", row["model"].objects.all())
+            total = qs.count()
+            self.stdout.write("- {} {} objects".format(total, row["id"]))
+
+        self.stdout.write("")
+        if options["dry_run"]:
+
+            self.stdout.write(
+                "Run this command with --no-dry-run to commit the changes to the database"
+            )
+            raise Rollback()
+
+        self.stdout.write(self.style.SUCCESS("Done!"))
+
+    def create_batch(self, row, results, options, count):
+        from funkwhale_api import factories
+
+        if row["id"] in results:
+            # already generated
+            return results[row["id"]]
+        if not count:
+            return []
+        dependencies = row.get("depends_on", [])
+        create_dependencies = options.get("create_dependencies")
+        for dependency in dependencies:
+            dep_count = options.get(dependency["id"])
+            if not create_dependencies and dep_count is None:
+                continue
+            if dep_count is None:
+                factor = options[
+                    "{}_{}_factor".format(row["id"], dependency["field"])
+                ] or dependency.get("default_factor")
+                dep_count = math.ceil(factor * count)
+
+            results[dependency["id"]] = self.create_batch(
+                CONFIG_BY_ID[dependency["id"]], results, options, count=dep_count
+            )
+        self.stdout.write("Creating {} {}…".format(count, row["id"]))
+        handler = row.get("handler")
+        if handler:
+            objects = handler(factories.registry, count, dependencies=results)
+        else:
+            objects = create_objects(
+                row, factories.registry, count, **row.get("factory_kwargs", {})
+            )
+        for dependency in dependencies:
+            if not dependency.get("set", True):
+                continue
+            if create_dependencies:
+                candidates = results[dependency["id"]]
+            else:
+                # we use existing objects in the database
+                queryset = dependency.get(
+                    "queryset", CONFIG_BY_ID[dependency["id"]]["model"].objects.all()
+                )
+                candidates = list(queryset.values_list("pk", flat=True))
+                picked_pks = [random.choice(candidates) for _ in objects]
+                picked_objects = {o.pk: o for o in queryset.filter(pk__in=picked_pks)}
+            for i, obj in enumerate(objects):
+                if create_dependencies:
+                    value = random.choice(candidates)
+                else:
+                    value = picked_objects[picked_pks[i]]
+                setattr(obj, dependency["field"], value)
+        if not handler:
+            objects = row["model"].objects.bulk_create(objects, batch_size=BATCH_SIZE)
+        results[row["id"]] = objects
+        return objects

api/funkwhale_api/common/middleware.py

@@ -1,13 +1,16 @@
 import html
 import requests
+import time
 import xml.sax.saxutils
 
 from django import http
 from django.conf import settings
 from django.core.cache import caches
 from django import urls
+from rest_framework import views
 
 from . import preferences
+from . import throttling
 from . import utils
 
 EXCLUDED_PATHS = ["/api", "/federation", "/.well-known"]
@@ -176,3 +179,66 @@ class DevHttpsMiddleware:
                 lambda: request.__class__.get_host(request).replace(":80", ":443"),
             )
         return self.get_response(request)
+
+
+def monkey_patch_rest_initialize_request():
+    """
+    Rest framework use it's own APIRequest, meaning we can't easily
+    access our throttling info in the middleware. So me monkey patch the
+    `initialize_request` method from rest_framework to keep a link between both requests
+    """
+    original = views.APIView.initialize_request
+
+    def replacement(self, request, *args, **kwargs):
+        r = original(self, request, *args, **kwargs)
+        setattr(request, "_api_request", r)
+        return r
+
+    setattr(views.APIView, "initialize_request", replacement)
+
+
+monkey_patch_rest_initialize_request()
+
+
+class ThrottleStatusMiddleware:
+    """
+    Include useful information regarding throttling in API responses to
+    ensure clients can adapt.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        try:
+            response = self.get_response(request)
+        except throttling.TooManyRequests:
+            # manual throttling in non rest_framework view, we have to return
+            # the proper response ourselves
+            response = http.HttpResponse(status=429)
+        request_to_check = request
+        try:
+            request_to_check = request._api_request
+        except AttributeError:
+            pass
+        throttle_status = getattr(request_to_check, "_throttle_status", None)
+        if throttle_status:
+            response["X-RateLimit-Limit"] = str(throttle_status["num_requests"])
+            response["X-RateLimit-Scope"] = str(throttle_status["scope"])
+            response["X-RateLimit-Remaining"] = throttle_status["num_requests"] - len(
+                throttle_status["history"]
+            )
+            response["X-RateLimit-Duration"] = str(throttle_status["duration"])
+            if throttle_status["history"]:
+                now = int(time.time())
+                # At this point, the client can send additional requests
+                oldtest_request = throttle_status["history"][-1]
+                remaining = throttle_status["duration"] - (now - int(oldtest_request))
+                response["Retry-After"] = str(remaining)
+                # At this point, all Rate Limit is reset to 0
+                latest_request = throttle_status["history"][0]
+                remaining = throttle_status["duration"] - (now - int(latest_request))
+                response["X-RateLimit-Reset"] = str(now + remaining)
+                response["X-RateLimit-ResetSeconds"] = str(remaining)
+
+        return response

api/funkwhale_api/common/migrations/0003_cit_extension.py

@@ -0,0 +1,22 @@
+# Generated by Django 2.0.2 on 2018-02-27 18:43
+from django.db import migrations
+from django.contrib.postgres.operations import CITextExtension
+
+
+class CustomCITExtension(CITextExtension):
+    def database_forwards(self, app_label, schema_editor, from_state, to_state):
+        check_sql = "SELECT 1 FROM pg_extension WHERE extname = 'citext'"
+        with schema_editor.connection.cursor() as cursor:
+            cursor.execute(check_sql)
+            result = cursor.fetchall()
+
+        if result:
+            return
+        return super().database_forwards(app_label, schema_editor, from_state, to_state)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("common", "0002_mutation")]
+
+    operations = [CustomCITExtension()]

api/funkwhale_api/common/models.py

@@ -73,7 +73,7 @@ class LocalFromFidQuerySet:
             return self.filter(~query)
 
 
-class MutationQuerySet(models.QuerySet):
+class GenericTargetQuerySet(models.QuerySet):
     def get_for_target(self, target):
         content_type = ContentType.objects.get_for_model(target)
         return self.filter(target_content_type=content_type, target_id=target.pk)
@@ -119,7 +119,7 @@ class Mutation(models.Model):
     )
     target = GenericForeignKey("target_content_type", "target_id")
 
-    objects = MutationQuerySet.as_manager()
+    objects = GenericTargetQuerySet.as_manager()
 
     def get_federation_id(self):
         if self.fid:

api/funkwhale_api/common/mutations.py

@@ -86,6 +86,7 @@ class MutationSerializer(serializers.Serializer):
 
 class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
     serialized_relations = {}
+    previous_state_handlers = {}
 
     def __init__(self, *args, **kwargs):
         # we force partial mode, because update mutations are partial
@@ -139,16 +140,20 @@ class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
         return get_update_previous_state(
             obj,
             *list(validated_data.keys()),
-            serialized_relations=self.serialized_relations
+            serialized_relations=self.serialized_relations,
+            handlers=self.previous_state_handlers,
         )
 
 
-def get_update_previous_state(obj, *fields, serialized_relations={}):
+def get_update_previous_state(obj, *fields, serialized_relations={}, handlers={}):
     if not fields:
         raise ValueError("You need to provide at least one field")
 
     state = {}
     for field in fields:
+        if field in handlers:
+            state[field] = handlers[field](obj)
+            continue
         value = getattr(obj, field)
         if isinstance(value, models.Model):
             # we store the related object id and repr for better UX

api/funkwhale_api/common/preferences.py

@@ -14,6 +14,11 @@ def get(pref):
     return manager[pref]
 
 
+def all():
+    manager = global_preferences_registry.manager()
+    return manager.all()
+
+
 def set(pref, value):
     manager = global_preferences_registry.manager()
     manager[pref] = value

api/funkwhale_api/common/routers.py

@@ -0,0 +1,7 @@
+from rest_framework.routers import SimpleRouter
+
+
+class OptionalSlashRouter(SimpleRouter):
+    def __init__(self):
+        super().__init__()
+        self.trailing_slash = "/?"

api/funkwhale_api/common/throttling.py

@@ -0,0 +1,151 @@
+import collections
+
+from django.core.cache import cache
+from rest_framework import throttling as rest_throttling
+
+from django.conf import settings
+
+
+def get_ident(request):
+    if hasattr(request, "user") and request.user.is_authenticated:
+        return {"type": "authenticated", "id": request.user.pk}
+    ident = rest_throttling.BaseThrottle().get_ident(request)
+
+    return {"type": "anonymous", "id": ident}
+
+
+def get_cache_key(scope, ident):
+    parts = ["throttling", scope, ident["type"], str(ident["id"])]
+    return ":".join(parts)
+
+
+def get_scope_for_action_and_ident_type(action, ident_type, view_conf={}):
+    config = collections.ChainMap(view_conf, settings.THROTTLING_SCOPES)
+
+    try:
+        action_config = config[action]
+    except KeyError:
+        action_config = config.get("*", {})
+
+    try:
+        return action_config[ident_type]
+    except KeyError:
+        return
+
+
+def get_status(ident, now):
+    data = []
+    throttle = FunkwhaleThrottle()
+    for key in sorted(settings.THROTTLING_RATES.keys()):
+        conf = settings.THROTTLING_RATES[key]
+        row_data = {"id": key, "rate": conf["rate"], "description": conf["description"]}
+        if conf["rate"]:
+            num_requests, duration = throttle.parse_rate(conf["rate"])
+            history = cache.get(get_cache_key(key, ident)) or []
+
+            relevant_history = [h for h in history if h > now - duration]
+            row_data["limit"] = num_requests
+            row_data["duration"] = duration
+            row_data["remaining"] = num_requests - len(relevant_history)
+            if relevant_history and len(relevant_history) >= num_requests:
+                # At this point, the endpoint becomes available again
+                now_request = relevant_history[-1]
+                remaining = duration - (now - int(now_request))
+                row_data["available"] = int(now + remaining) or None
+                row_data["available_seconds"] = int(remaining) or None
+            else:
+                row_data["available"] = None
+                row_data["available_seconds"] = None
+
+            if relevant_history:
+                # At this point, all Rate Limit is reset to 0
+                latest_request = relevant_history[0]
+                remaining = duration - (now - int(latest_request))
+                row_data["reset"] = int(now + remaining)
+                row_data["reset_seconds"] = int(remaining)
+            else:
+                row_data["reset"] = None
+                row_data["reset_seconds"] = None
+        else:
+            row_data["limit"] = None
+            row_data["duration"] = None
+            row_data["remaining"] = None
+            row_data["available"] = None
+            row_data["available_seconds"] = None
+            row_data["reset"] = None
+            row_data["reset_seconds"] = None
+
+        data.append(row_data)
+
+    return data
+
+
+class FunkwhaleThrottle(rest_throttling.SimpleRateThrottle):
+    def __init__(self):
+        pass
+
+    def get_cache_key(self, request, view):
+        return get_cache_key(self.scope, self.ident)
+
+    def allow_request(self, request, view):
+        self.request = request
+        self.ident = get_ident(request)
+        action = getattr(view, "action", "*")
+        view_scopes = getattr(view, "throttling_scopes", {})
+        if view_scopes is None:
+            return True
+        self.scope = get_scope_for_action_and_ident_type(
+            action=action, ident_type=self.ident["type"], view_conf=view_scopes
+        )
+        if not self.scope or self.scope not in settings.THROTTLING_RATES:
+            return True
+        self.rate = settings.THROTTLING_RATES[self.scope].get("rate")
+        self.num_requests, self.duration = self.parse_rate(self.rate)
+        self.request = request
+
+        return super().allow_request(request, view)
+
+    def attach_info(self):
+        info = {
+            "num_requests": self.num_requests,
+            "duration": self.duration,
+            "scope": self.scope,
+            "history": self.history or [],
+            "wait": self.wait(),
+        }
+        setattr(self.request, "_throttle_status", info)
+
+    def throttle_success(self):
+        self.attach_info()
+        return super().throttle_success()
+
+    def throttle_failure(self):
+        self.attach_info()
+        return super().throttle_failure()
+
+
+class TooManyRequests(Exception):
+    pass
+
+
+DummyView = collections.namedtuple("DummyView", "action throttling_scopes")
+
+
+def check_request(request, scope):
+    """
+    A simple wrapper around FunkwhaleThrottle for views that aren't API views
+    or cannot use rest_framework automatic throttling.
+
+    Raise TooManyRequests if limit is reached.
+    """
+    if not settings.THROTTLING_ENABLED:
+        return True
+
+    view = DummyView(
+        action=scope,
+        throttling_scopes={scope: {"anonymous": scope, "authenticated": scope}},
+    )
+    throttle = FunkwhaleThrottle()
+    if not throttle.allow_request(request, view):
+        raise TooManyRequests()
+    return True

api/funkwhale_api/common/utils.py

@@ -154,7 +154,7 @@ def order_for_search(qs, field):
 
 def recursive_getattr(obj, key, permissive=False):
     """
-    Given a dictionary such as {'user': {'name': 'Bob'}} and
+    Given a dictionary such as {'user': {'name': 'Bob'}} or and object and
     a dotted string such as user.name, returns 'Bob'.
 
     If the value is not present, returns None
@@ -162,7 +162,10 @@ def recursive_getattr(obj, key, permissive=False):
     v = obj
     for k in key.split("."):
         try:
-            v = v.get(k)
+            if hasattr(v, "get"):
+                v = v.get(k)
+            else:
+                v = getattr(v, k)
         except (TypeError, AttributeError):
             if not permissive:
                 raise

api/funkwhale_api/common/views.py

@@ -1,3 +1,6 @@
+import time
+
+from django.conf import settings
 from django.db import transaction
 
 from rest_framework.decorators import action
@@ -5,6 +8,7 @@ from rest_framework import exceptions
 from rest_framework import mixins
 from rest_framework import permissions
 from rest_framework import response
+from rest_framework import views
 from rest_framework import viewsets
 
 from . import filters
@@ -13,6 +17,7 @@ from . import mutations
 from . import serializers
 from . import signals
 from . import tasks
+from . import throttling
 from . import utils
 
 
@@ -121,3 +126,17 @@ class MutationViewSet(
             new_is_approved=instance.is_approved,
         )
         return response.Response({}, status=200)
+
+
+class RateLimitView(views.APIView):
+    permission_classes = []
+    throttle_classes = []
+
+    def get(self, request, *args, **kwargs):
+        ident = throttling.get_ident(request)
+        data = {
+            "enabled": settings.THROTTLING_ENABLED,
+            "ident": ident,
+            "scopes": throttling.get_status(ident, time.time()),
+        }
+        return response.Response(data, status=200)

api/funkwhale_api/factories.py

@@ -1,5 +1,6 @@
 import uuid
 import factory
+import random
 import persisting_theory
 
 from django.conf import settings
@@ -46,6 +47,268 @@ class NoUpdateOnCreate:
         return
 
 
+TAGS_DATA = {
+    "type": [
+        "acoustic",
+        "acid",
+        "ambient",
+        "alternative",
+        "brutalist",
+        "chill",
+        "club",
+        "cold",
+        "cool",
+        "contemporary",
+        "dark",
+        "doom",
+        "electro",
+        "folk",
+        "freestyle",
+        "fusion",
+        "garage",
+        "glitch",
+        "hard",
+        "healing",
+        "industrial",
+        "instrumental",
+        "hardcore",
+        "holiday",
+        "hot",
+        "liquid",
+        "modern",
+        "minimalist",
+        "new",
+        "parody",
+        "postmodern",
+        "progressive",
+        "smooth",
+        "symphonic",
+        "traditional",
+        "tribal",
+        "metal",
+    ],
+    "genre": [
+        "blues",
+        "classical",
+        "chiptune",
+        "dance",
+        "disco",
+        "funk",
+        "jazz",
+        "house",
+        "hiphop",
+        "NewAge",
+        "pop",
+        "punk",
+        "rap",
+        "RnB",
+        "reggae",
+        "rock",
+        "soul",
+        "soundtrack",
+        "ska",
+        "swing",
+        "trance",
+    ],
+    "nationality": [
+        "Afghan",
+        "Albanian",
+        "Algerian",
+        "American",
+        "Andorran",
+        "Angolan",
+        "Antiguans",
+        "Argentinean",
+        "Armenian",
+        "Australian",
+        "Austrian",
+        "Azerbaijani",
+        "Bahamian",
+        "Bahraini",
+        "Bangladeshi",
+        "Barbadian",
+        "Barbudans",
+        "Batswana",
+        "Belarusian",
+        "Belgian",
+        "Belizean",
+        "Beninese",
+        "Bhutanese",
+        "Bolivian",
+        "Bosnian",
+        "Brazilian",
+        "British",
+        "Bruneian",
+        "Bulgarian",
+        "Burkinabe",
+        "Burmese",
+        "Burundian",
+        "Cambodian",
+        "Cameroonian",
+        "Canadian",
+        "Cape Verdean",
+        "Central African",
+        "Chadian",
+        "Chilean",
+        "Chinese",
+        "Colombian",
+        "Comoran",
+        "Congolese",
+        "Costa Rican",
+        "Croatian",
+        "Cuban",
+        "Cypriot",
+        "Czech",
+        "Danish",
+        "Djibouti",
+        "Dominican",
+        "Dutch",
+        "East Timorese",
+        "Ecuadorean",
+        "Egyptian",
+        "Emirian",
+        "Equatorial Guinean",
+        "Eritrean",
+        "Estonian",
+        "Ethiopian",
+        "Fijian",
+        "Filipino",
+        "Finnish",
+        "French",
+        "Gabonese",
+        "Gambian",
+        "Georgian",
+        "German",
+        "Ghanaian",
+        "Greek",
+        "Grenadian",
+        "Guatemalan",
+        "Guinea-Bissauan",
+        "Guinean",
+        "Guyanese",
+        "Haitian",
+        "Herzegovinian",
+        "Honduran",
+        "Hungarian",
+        "I-Kiribati",
+        "Icelander",
+        "Indian",
+        "Indonesian",
+        "Iranian",
+        "Iraqi",
+        "Irish",
+        "Israeli",
+        "Italian",
+        "Ivorian",
+        "Jamaican",
+        "Japanese",
+        "Jordanian",
+        "Kazakhstani",
+        "Kenyan",
+        "Kittian and Nevisian",
+        "Kuwaiti",
+        "Kyrgyz",
+        "Laotian",
+        "Latvian",
+        "Lebanese",
+        "Liberian",
+        "Libyan",
+        "Liechtensteiner",
+        "Lithuanian",
+        "Luxembourger",
+        "Macedonian",
+        "Malagasy",
+        "Malawian",
+        "Malaysian",
+        "Maldivian",
+        "Malian",
+        "Maltese",
+        "Marshallese",
+        "Mauritanian",
+        "Mauritian",
+        "Mexican",
+        "Micronesian",
+        "Moldovan",
+        "Monacan",
+        "Mongolian",
+        "Moroccan",
+        "Mosotho",
+        "Motswana",
+        "Mozambican",
+        "Namibian",
+        "Nauruan",
+        "Nepalese",
+        "New Zealander",
+        "Ni-Vanuatu",
+        "Nicaraguan",
+        "Nigerian",
+        "Nigerien",
+        "North Korean",
+        "Northern Irish",
+        "Norwegian",
+        "Omani",
+        "Pakistani",
+        "Palauan",
+        "Panamanian",
+        "Papua New Guinean",
+        "Paraguayan",
+        "Peruvian",
+        "Polish",
+        "Portuguese",
+        "Qatari",
+        "Romanian",
+        "Russian",
+        "Rwandan",
+        "Saint Lucian",
+        "Salvadoran",
+        "Samoan",
+        "San Marinese",
+        "Sao Tomean",
+        "Saudi",
+        "Scottish",
+        "Senegalese",
+        "Serbian",
+        "Seychellois",
+        "Sierra Leonean",
+        "Singaporean",
+        "Slovakian",
+        "Slovenian",
+        "Solomon Islander",
+        "Somali",
+        "South African",
+        "South Korean",
+        "Spanish",
+        "Sri Lankan",
+        "Sudanese",
+        "Surinamer",
+        "Swazi",
+        "Swedish",
+        "Swiss",
+        "Syrian",
+        "Taiwanese",
+        "Tajik",
+        "Tanzanian",
+        "Thai",
+        "Togolese",
+        "Tongan",
+        "Trinidadian",
+        "Tunisian",
+        "Turkish",
+        "Tuvaluan",
+        "Ugandan",
+        "Ukrainian",
+        "Uruguayan",
+        "Uzbekistani",
+        "Venezuelan",
+        "Vietnamese",
+        "Welsh",
+        "Yemenite",
+        "Zambian",
+        "Zimbabwean",
+    ],
+}
+
+
 class FunkwhaleProvider(internet_provider.Provider):
     """
     Our own faker data generator, since built-in ones are sometimes
@@ -61,5 +324,40 @@ class FunkwhaleProvider(internet_provider.Provider):
         path = path_generator()
         return "{}://{}/{}".format(protocol, domain, path)
 
+    def user_name(self):
+        u = super().user_name()
+        return "{}{}".format(u, random.randint(10, 999))
+
+    def music_genre(self):
+        return random.choice(TAGS_DATA["genre"])
+
+    def music_type(self):
+        return random.choice(TAGS_DATA["type"])
+
+    def music_nationality(self):
+        return random.choice(TAGS_DATA["nationality"])
+
+    def music_hashtag(self, prefix_length=4):
+        genre = self.music_genre()
+        prefixes = [genre]
+        nationality = False
+        while len(prefixes) < prefix_length:
+            if nationality:
+                t = "type"
+            else:
+                t = random.choice(["type", "nationality", "genre"])
+
+            if t == "nationality":
+                nationality = True
+
+            choice = random.choice(TAGS_DATA[t])
+            if choice in prefixes:
+                continue
+            prefixes.append(choice)
+
+        return "".join(
+            [p.capitalize().strip().replace(" ", "") for p in reversed(prefixes)]
+        )
+
 
 factory.Faker.add_provider(FunkwhaleProvider)

api/funkwhale_api/favorites/serializers.py

@@ -1,6 +1,7 @@
 from rest_framework import serializers
 
 from funkwhale_api.activity import serializers as activity_serializers
+from funkwhale_api.federation import serializers as federation_serializers
 from funkwhale_api.music.serializers import TrackActivitySerializer, TrackSerializer
 from funkwhale_api.users.serializers import UserActivitySerializer, UserBasicSerializer
 
@@ -27,10 +28,17 @@ class TrackFavoriteActivitySerializer(activity_serializers.ModelSerializer):
 class UserTrackFavoriteSerializer(serializers.ModelSerializer):
     track = TrackSerializer(read_only=True)
     user = UserBasicSerializer(read_only=True)
+    actor = serializers.SerializerMethodField()
 
     class Meta:
         model = models.TrackFavorite
-        fields = ("id", "user", "track", "creation_date")
+        fields = ("id", "user", "track", "creation_date", "actor")
+        actor = serializers.SerializerMethodField()
+
+    def get_actor(self, obj):
+        actor = obj.user.actor
+        if actor:
+            return federation_serializers.APIActorSerializer(actor).data
 
 
 class UserTrackFavoriteWriteSerializer(serializers.ModelSerializer):

api/funkwhale_api/favorites/urls.py

@@ -1,8 +1,8 @@
-from rest_framework import routers
+from funkwhale_api.common import routers
 
 from . import views
 
-router = routers.SimpleRouter()
+router = routers.OptionalSlashRouter()
 router.register(r"tracks", views.TrackFavoriteViewSet, "tracks")
 
 urlpatterns = router.urls

api/funkwhale_api/favorites/views.py

@@ -22,7 +22,7 @@ class TrackFavoriteViewSet(
 
     filterset_class = filters.TrackFavoriteFilter
     serializer_class = serializers.UserTrackFavoriteSerializer
-    queryset = models.TrackFavorite.objects.all().select_related("user")
+    queryset = models.TrackFavorite.objects.all().select_related("user__actor")
     permission_classes = [
         oauth_permissions.ScopePermission,
         permissions.OwnerPermission,
@@ -54,7 +54,7 @@ class TrackFavoriteViewSet(
         )
         tracks = Track.objects.with_playable_uploads(
             music_utils.get_actor_from_request(self.request)
-        ).select_related("artist", "album__artist")
+        ).select_related("artist", "album__artist", "attributed_to")
         queryset = queryset.prefetch_related(Prefetch("track", queryset=tracks))
         return queryset
 

api/funkwhale_api/federation/activity.py

@@ -1,5 +1,6 @@
 import uuid
 import logging
+import urllib.parse
 
 from django.core.cache import cache
 from django.conf import settings
@@ -122,32 +123,38 @@ def receive(activity, on_behalf_of):
     from . import serializers
     from . import tasks
     from .routes import inbox
+    from funkwhale_api.moderation import mrf
 
+    logger.debug(
+        "[federation] Received activity from %s : %s", on_behalf_of.fid, activity
+    )
     # we ensure the activity has the bare minimum structure before storing
     # it in our database
     serializer = serializers.BaseActivitySerializer(
         data=activity, context={"actor": on_behalf_of, "local_recipients": True}
     )
     serializer.is_valid(raise_exception=True)
-    if not inbox.get_matching_handlers(activity):
-        # discard unhandlable activity
-        return
 
-    if should_reject(
-        fid=serializer.validated_data.get("id"),
-        actor_id=serializer.validated_data["actor"].fid,
-        payload=activity,
-    ):
+    payload, updated = mrf.inbox.apply(activity, sender_id=on_behalf_of.fid)
+    if not payload:
         logger.info(
-            "[federation] Discarding activity due to instance policies %s",
+            "[federation] Discarding activity due to mrf %s",
             serializer.validated_data.get("id"),
         )
         return
+
+    if not inbox.get_matching_handlers(payload):
+        # discard unhandlable activity
+        logger.debug(
+            "[federation] No matching route found for activity, discarding: %s", payload
+        )
+        return
+
     try:
-        copy = serializer.save()
+        copy = serializer.save(payload=payload, type=payload["type"])
     except IntegrityError:
         logger.warning(
-            "[federation] Discarding already elivered activity %s",
+            "[federation] Discarding already delivered activity %s",
             serializer.validated_data.get("id"),
         )
         return
@@ -283,9 +290,19 @@ class OutboxRouter(Router):
         and may yield data that should be persisted in the Activity model
         for further delivery.
         """
+        from funkwhale_api.common import preferences
         from . import models
         from . import tasks
 
+        allow_list_enabled = preferences.get("moderation__allow_list_enabled")
+        allowed_domains = None
+        if allow_list_enabled:
+            allowed_domains = set(
+                models.Domain.objects.filter(allowed=True).values_list(
+                    "name", flat=True
+                )
+            )
+
         for route, handler in self.routes:
             if not match_route(route, routing):
                 continue
@@ -314,10 +331,10 @@ class OutboxRouter(Router):
                 a = models.Activity(**activity_data)
                 a.uuid = uuid.uuid4()
                 to_inbox_items, to_deliveries, new_to = prepare_deliveries_and_inbox_items(
-                    to, "to"
+                    to, "to", allowed_domains=allowed_domains
                 )
                 cc_inbox_items, cc_deliveries, new_cc = prepare_deliveries_and_inbox_items(
-                    cc, "cc"
+                    cc, "cc", allowed_domains=allowed_domains
                 )
                 if not any(
                     [to_inbox_items, to_deliveries, cc_inbox_items, cc_deliveries]
@@ -368,13 +385,23 @@ class OutboxRouter(Router):
 def match_route(route, payload):
     for key, value in route.items():
         payload_value = recursive_getattr(payload, key, permissive=True)
-        if payload_value != value:
+        if isinstance(value, list):
+            if payload_value not in value:
+                return False
+        elif payload_value != value:
             return False
 
     return True
 
 
-def prepare_deliveries_and_inbox_items(recipient_list, type):
+def is_allowed_url(url, allowed_domains):
+    return (
+        allowed_domains is None
+        or urllib.parse.urlparse(url).hostname in allowed_domains
+    )
+
+
+def prepare_deliveries_and_inbox_items(recipient_list, type, allowed_domains=None):
     """
     Given a list of recipients (
         either actor instances, public adresses, a dictionnary with a "type" and "target"
@@ -384,10 +411,12 @@ def prepare_deliveries_and_inbox_items(recipient_list, type):
     """
     from . import models
 
+    if allowed_domains is not None:
+        allowed_domains = set(allowed_domains)
+        allowed_domains.add(settings.FEDERATION_HOSTNAME)
     local_recipients = set()
     remote_inbox_urls = set()
     urls = []
-
     for r in recipient_list:
         if isinstance(r, models.Actor):
             if r.is_local:
@@ -424,15 +453,39 @@ def prepare_deliveries_and_inbox_items(recipient_list, type):
                     .exclude(actor__domain=None)
                 )
             )
+            followed_domains = list(follows.values_list("actor__domain_id", flat=True))
             actors = models.Actor.objects.filter(
-                managed_domains__name__in=follows.values_list(
-                    "actor__domain_id", flat=True
-                )
+                managed_domains__name__in=followed_domains
             )
-            values = actors.values("shared_inbox_url", "inbox_url")
+            values = actors.values("shared_inbox_url", "inbox_url", "domain_id")
+            handled_domains = set()
             for v in values:
                 remote_inbox_urls.add(v["shared_inbox_url"] or v["inbox_url"])
-    deliveries = [models.Delivery(inbox_url=url) for url in remote_inbox_urls]
+                handled_domains.add(v["domain_id"])
+
+            if len(handled_domains) >= len(followed_domains):
+                continue
+
+            # for all remaining domains (probably non-funkwhale instances, with no
+            # service actors), we also pick the latest known actor per domain and send the message
+            # there instead
+            remaining_domains = models.Domain.objects.exclude(name__in=handled_domains)
+            remaining_domains = remaining_domains.filter(name__in=followed_domains)
+            actors = models.Actor.objects.filter(domain__in=remaining_domains)
+            actors = (
+                actors.order_by("domain_id", "-last_fetch_date")
+                .distinct("domain_id")
+                .values("shared_inbox_url", "inbox_url")
+            )
+            for v in actors:
+                remote_inbox_urls.add(v["shared_inbox_url"] or v["inbox_url"])
+
+    deliveries = [
+        models.Delivery(inbox_url=url)
+        for url in remote_inbox_urls
+        if is_allowed_url(url, allowed_domains)
+    ]
+    urls = [url for url in urls if is_allowed_url(url, allowed_domains)]
     inbox_items = [
         models.InboxItem(actor=actor, type=type) for actor in local_recipients
     ]

api/funkwhale_api/federation/admin.py

@@ -26,7 +26,8 @@ redeliver_activities.short_description = "Redeliver"
 
 @admin.register(models.Domain)
 class DomainAdmin(admin.ModelAdmin):
-    list_display = ["name", "creation_date"]
+    list_display = ["name", "allowed", "creation_date"]
+    list_filter = ["allowed"]
     search_fields = ["name"]
 
 
@@ -40,7 +41,7 @@ class FetchAdmin(admin.ModelAdmin):
 
 @admin.register(models.Activity)
 class ActivityAdmin(admin.ModelAdmin):
-    list_display = ["type", "fid", "url", "actor", "creation_date"]
+    list_display = ["uuid", "type", "fid", "url", "actor", "creation_date"]
     search_fields = ["payload", "fid", "url", "actor__domain__name"]
     list_filter = ["type", "actor__domain__name"]
     actions = [redeliver_activities]

api/funkwhale_api/federation/api_urls.py

@@ -1,8 +1,8 @@
-from rest_framework import routers
+from funkwhale_api.common import routers
 
 from . import api_views
 
-router = routers.SimpleRouter()
+router = routers.OptionalSlashRouter()
 router.register(r"fetches", api_views.FetchViewSet, "fetches")
 router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows")
 router.register(r"inbox", api_views.InboxItemViewSet, "inbox")

api/funkwhale_api/federation/authentication.py

@@ -1,13 +1,14 @@
 import cryptography
 import logging
 import datetime
-
+import urllib.parse
 from django.contrib.auth.models import AnonymousUser
 from django.utils import timezone
 
 from rest_framework import authentication, exceptions as rest_exceptions
+from funkwhale_api.common import preferences
 from funkwhale_api.moderation import models as moderation_models
-from . import actors, exceptions, keys, signing, tasks, utils
+from . import actors, exceptions, keys, models, signing, tasks, utils
 
 
 logger = logging.getLogger(__name__)
@@ -37,6 +38,16 @@ class SignatureAuthentication(authentication.BaseAuthentication):
         if policies.exists():
             raise exceptions.BlockedActorOrDomain()
 
+        if request.method.lower() == "get" and preferences.get(
+            "moderation__allow_list_enabled"
+        ):
+            # Only GET requests because POST requests with messages will be handled through
+            # MRF
+            domain = urllib.parse.urlparse(actor_url).hostname
+            allowed = models.Domain.objects.filter(name=domain, allowed=True).exists()
+            if not allowed:
+                raise exceptions.BlockedActorOrDomain()
+
         try:
             actor = actors.get_actor(actor_url)
         except Exception as e:

api/funkwhale_api/federation/contexts.py

@@ -214,6 +214,7 @@ CONTEXTS = [
                 "shares": {"@id": "as:shares", "@type": "@id"},
                 # Added manually
                 "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+                "Hashtag": "as:Hashtag",
             }
         },
     },

api/funkwhale_api/federation/dynamic_preferences_registry.py

@@ -14,7 +14,7 @@ class MusicCacheDuration(types.IntPreference):
     default = 60 * 24 * 2
     verbose_name = "Music cache duration"
     help_text = (
-        "How much minutes do you want to keep a copy of federated tracks"
+        "How many minutes do you want to keep a copy of federated tracks "
         "locally? Federated files that were not listened in this interval "
         "will be erased and refetched from the remote on the next listening."
     )
@@ -38,7 +38,7 @@ class CollectionPageSize(preferences.DefaultFromSettingMixin, types.IntPreferenc
     name = "collection_page_size"
     setting = "FEDERATION_COLLECTION_PAGE_SIZE"
     verbose_name = "Federation collection page size"
-    help_text = "How much items to display in ActivityPub collections."
+    help_text = "How many items to display in ActivityPub collections."
     field_kwargs = {"required": False}
 
 
@@ -49,7 +49,7 @@ class ActorFetchDelay(preferences.DefaultFromSettingMixin, types.IntPreference):
     setting = "FEDERATION_ACTOR_FETCH_DELAY"
     verbose_name = "Federation actor fetch delay"
     help_text = (
-        "How much minutes to wait before refetching actors on "
+        "How many minutes to wait before refetching actors on "
         "request authentication."
     )
     field_kwargs = {"required": False}

api/funkwhale_api/federation/factories.py

@@ -70,6 +70,7 @@ def create_user(actor):
 class DomainFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     name = factory.Faker("domain_name")
     nodeinfo_fetch_date = factory.LazyFunction(lambda: timezone.now())
+    allowed = None
 
     class Meta:
         model = "federation.Domain"

api/funkwhale_api/federation/migrations/0019_auto_20190611_0851.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.2 on 2019-06-11 08:51
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0018_fetch")]
+
+    operations = [
+        migrations.AddField(
+            model_name="domain",
+            name="allowed",
+            field=models.BooleanField(default=None, null=True),
+        )
+    ]

api/funkwhale_api/federation/migrations/0020_auto_20190730_0846.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.2.3 on 2019-07-30 08:46
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0019_auto_20190611_0851'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='activity',
+            name='payload',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000),
+        ),
+        migrations.AlterField(
+            model_name='fetch',
+            name='detail',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000),
+        ),
+        migrations.AlterField(
+            model_name='librarytrack',
+            name='metadata',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=10000),
+        ),
+    ]

api/funkwhale_api/federation/models.py

@@ -21,6 +21,7 @@ from . import utils as federation_utils
 
 TYPE_CHOICES = [
     ("Person", "Person"),
+    ("Tombstone", "Tombstone"),
     ("Application", "Application"),
     ("Group", "Group"),
     ("Organization", "Organization"),
@@ -118,6 +119,9 @@ class Domain(models.Model):
         null=True,
         blank=True,
     )
+    # are interactions with this domain allowed (only applies when allow-listing is on)
+    allowed = models.BooleanField(default=None, null=True)
+
     objects = DomainQuerySet.as_manager()
 
     def __str__(self):
@@ -201,6 +205,10 @@ class Actor(models.Model):
 
     class Meta:
         unique_together = ["domain", "preferred_username"]
+        verbose_name = "Account"
+
+    def get_moderation_url(self):
+        return "/manage/moderation/accounts/{}".format(self.full_username)
 
     @property
     def webfinger_subject(self):
@@ -245,6 +253,7 @@ class Actor(models.Model):
 
     def get_stats(self):
         from funkwhale_api.music import models as music_models
+        from funkwhale_api.moderation import models as moderation_models
 
         data = Actor.objects.filter(pk=self.pk).aggregate(
             outbox_activities=models.Count("outbox_activities", distinct=True),
@@ -257,6 +266,7 @@ class Actor(models.Model):
         data["artists"] = music_models.Artist.objects.filter(
             from_activity__actor=self.pk
         ).count()
+        data["reports"] = moderation_models.Report.objects.get_for_target(self).count()
         data["albums"] = music_models.Album.objects.filter(
             from_activity__actor=self.pk
         ).count()

api/funkwhale_api/federation/mrf_policies.py

@@ -0,0 +1,15 @@
+from funkwhale_api.moderation import mrf
+
+
+from . import activity
+
+
+@mrf.inbox.register(name="instance_policies")
+def instance_policies(payload, **kwargs):
+    reject = activity.should_reject(
+        fid=payload.get("id"),
+        actor_id=kwargs.get("sender_id", payload.get("id")),
+        payload=payload,
+    )
+    if reject:
+        raise mrf.Discard()

api/funkwhale_api/federation/routes.py

@@ -4,6 +4,7 @@ from funkwhale_api.music import models as music_models
 
 from . import activity
 from . import actors
+from . import models
 from . import serializers
 
 logger = logging.getLogger(__name__)
@@ -380,3 +381,63 @@ def outbox_update_artist(context):
             to=[activity.PUBLIC_ADDRESS, {"type": "instances_with_followers"}],
         ),
     }
+
+
+@outbox.register(
+    {
+        "type": "Delete",
+        "object.type": [
+            "Tombstone",
+            "Actor",
+            "Person",
+            "Application",
+            "Organization",
+            "Service",
+            "Group",
+        ],
+    }
+)
+def outbox_delete_actor(context):
+    actor = context["actor"]
+    serializer = serializers.ActivitySerializer(
+        {"type": "Delete", "object": {"type": actor.type, "id": actor.fid}}
+    )
+    yield {
+        "type": "Delete",
+        "actor": actor,
+        "payload": with_recipients(
+            serializer.data,
+            to=[activity.PUBLIC_ADDRESS, {"type": "instances_with_followers"}],
+        ),
+    }
+
+
+@inbox.register(
+    {
+        "type": "Delete",
+        "object.type": [
+            "Tombstone",
+            "Actor",
+            "Person",
+            "Application",
+            "Organization",
+            "Service",
+            "Group",
+        ],
+    }
+)
+def inbox_delete_actor(payload, context):
+    actor = context["actor"]
+    serializer = serializers.ActorDeleteSerializer(data=payload)
+    if not serializer.is_valid():
+        logger.info("Skipped actor %s deletion, invalid payload", actor.fid)
+        return
+
+    deleted_fid = serializer.validated_data["fid"]
+    try:
+        # ensure the actor only can delete itself, and is a remote one
+        actor = models.Actor.objects.local(False).get(fid=deleted_fid, pk=actor.pk)
+    except models.Actor.DoesNotExist:
+        logger.warn("Cannot delete actor %s, no matching object found", actor.fid)
+        return
+    actor.delete()

api/funkwhale_api/federation/serializers.py

@@ -11,6 +11,7 @@ from funkwhale_api.common import utils as funkwhale_utils
 from funkwhale_api.music import licenses
 from funkwhale_api.music import models as music_models
 from funkwhale_api.music import tasks as music_tasks
+from funkwhale_api.tags import models as tags_models
 
 from . import activity, actors, contexts, jsonld, models, tasks, utils
 
@@ -778,9 +779,24 @@ MUSIC_ENTITY_JSONLD_MAPPING = {
     "published": jsonld.first_val(contexts.AS.published),
     "musicbrainzId": jsonld.first_val(contexts.FW.musicbrainzId),
     "attributedTo": jsonld.first_id(contexts.AS.attributedTo),
+    "tags": jsonld.raw(contexts.AS.tag),
 }
 
 
+class TagSerializer(jsonld.JsonLdSerializer):
+    type = serializers.ChoiceField(choices=[contexts.AS.Hashtag])
+    name = serializers.CharField(max_length=100)
+
+    class Meta:
+        jsonld_mapping = {"name": jsonld.first_val(contexts.AS.name)}
+
+    def validate_name(self, value):
+        if value.startswith("#"):
+            # remove trailing #
+            value = value[1:]
+        return value
+
+
 class MusicEntitySerializer(jsonld.JsonLdSerializer):
     id = serializers.URLField(max_length=500)
     published = serializers.DateTimeField()
@@ -788,6 +804,9 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
     name = serializers.CharField(max_length=1000)
     attributedTo = serializers.URLField(max_length=500, allow_null=True, required=False)
     updateable_fields = []
+    tags = serializers.ListField(
+        child=TagSerializer(), min_length=0, required=False, allow_null=True
+    )
 
     def update(self, instance, validated_data):
         attributed_to_fid = validated_data.get("attributedTo")
@@ -797,10 +816,18 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
             self.updateable_fields, validated_data, instance
         )
         if updated_fields:
-            return music_tasks.update_library_entity(instance, updated_fields)
+            music_tasks.update_library_entity(instance, updated_fields)
 
+        tags = [t["name"] for t in validated_data.get("tags", []) or []]
+        tags_models.set_tags(instance, *tags)
         return instance
 
+    def get_tags_repr(self, instance):
+        return [
+            {"type": "Hashtag", "name": "#{}".format(tag)}
+            for tag in sorted(instance.tagged_items.values_list("tag__name", flat=True))
+        ]
+
 
 class ArtistSerializer(MusicEntitySerializer):
     updateable_fields = [
@@ -823,6 +850,7 @@ class ArtistSerializer(MusicEntitySerializer):
             "attributedTo": instance.attributed_to.fid
             if instance.attributed_to
             else None,
+            "tag": self.get_tags_repr(instance),
         }
 
         if self.context.get("include_ap_context", self.parent is None):
@@ -872,6 +900,7 @@ class AlbumSerializer(MusicEntitySerializer):
             "attributedTo": instance.attributed_to.fid
             if instance.attributed_to
             else None,
+            "tag": self.get_tags_repr(instance),
         }
         if instance.cover:
             d["cover"] = {
@@ -941,6 +970,7 @@ class TrackSerializer(MusicEntitySerializer):
             "attributedTo": instance.attributed_to.fid
             if instance.attributed_to
             else None,
+            "tag": self.get_tags_repr(instance),
         }
 
         if self.context.get("include_ap_context", self.parent is None):
@@ -981,7 +1011,6 @@ class TrackSerializer(MusicEntitySerializer):
             if not url:
                 continue
             references[url] = actors.get_actor(url)
-
         metadata = music_tasks.federation_audio_track_to_metadata(
             validated_data, references
         )
@@ -990,6 +1019,7 @@ class TrackSerializer(MusicEntitySerializer):
         if from_activity:
             metadata["from_activity_id"] = from_activity.pk
         track = music_tasks.get_track_from_import_metadata(metadata, update_cover=True)
+
         return track
 
     def update(self, obj, validated_data):
@@ -1108,6 +1138,13 @@ class UploadSerializer(jsonld.JsonLdSerializer):
         return d
 
 
+class ActorDeleteSerializer(jsonld.JsonLdSerializer):
+    fid = serializers.URLField(max_length=500)
+
+    class Meta:
+        jsonld_mapping = {"fid": jsonld.first_id(contexts.AS.object)}
+
+
 class NodeInfoLinkSerializer(serializers.Serializer):
     href = serializers.URLField()
     rel = serializers.URLField()

api/funkwhale_api/federation/tasks.py

@@ -190,7 +190,11 @@ def update_domain_nodeinfo(domain):
     now = timezone.now()
     try:
         nodeinfo = {"status": "ok", "payload": fetch_nodeinfo(domain.name)}
-    except (requests.RequestException, serializers.serializers.ValidationError) as e:
+    except (
+        requests.RequestException,
+        serializers.serializers.ValidationError,
+        ValueError,
+    ) as e:
         nodeinfo = {"status": "error", "error": str(e)}
 
     service_actor_id = common_utils.recursive_getattr(

api/funkwhale_api/federation/urls.py

@@ -9,6 +9,7 @@ music_router = routers.SimpleRouter(trailing_slash=False)
 router.register(r"federation/shared", views.SharedViewSet, "shared")
 router.register(r"federation/actors", views.ActorViewSet, "actors")
 router.register(r"federation/edits", views.EditViewSet, "edits")
+router.register(r"federation/reports", views.ReportViewSet, "reports")
 router.register(r".well-known", views.WellKnownViewSet, "well-known")
 
 music_router.register(r"libraries", views.MusicLibraryViewSet, "libraries")

api/funkwhale_api/federation/utils.py

@@ -4,7 +4,7 @@ from django.conf import settings
 from django.db.models import Q
 
 from funkwhale_api.common import session
-from funkwhale_api.moderation import models as moderation_models
+from funkwhale_api.moderation import mrf
 
 from . import exceptions
 from . import signing
@@ -64,10 +64,10 @@ def slugify_username(username):
 def retrieve_ap_object(
     fid, actor, serializer_class=None, queryset=None, apply_instance_policies=True
 ):
-    from . import activity
-
-    policies = moderation_models.InstancePolicy.objects.active().filter(block_all=True)
-    if apply_instance_policies and policies.matching_url(fid):
+    # we have a duplicate check here because it's less expensive to do those checks
+    # twice than to trigger a HTTP request
+    payload, updated = mrf.inbox.apply({"id": fid})
+    if not payload:
         raise exceptions.BlockedActorOrDomain()
     if queryset:
         try:
@@ -94,15 +94,12 @@ def retrieve_ap_object(
     response.raise_for_status()
     data = response.json()
 
-    # we match against moderation policies here again, because the FID of the returned
-    # object may not be the same as the URL used to access it
-    try:
-        id = data["id"]
-    except KeyError:
-        pass
-    else:
-        if apply_instance_policies and activity.should_reject(fid=id, payload=data):
-            raise exceptions.BlockedActorOrDomain()
+    # we match against mrf here again, because new data may yield different
+    # results
+    data, updated = mrf.inbox.apply(data)
+    if not data:
+        raise exceptions.BlockedActorOrDomain()
+
     if not serializer_class:
         return data
     serializer = serializer_class(data=data, context={"fetch_actor": actor})
@@ -131,3 +128,32 @@ def is_local(url):
     return url.startswith("http://{}/".format(d)) or url.startswith(
         "https://{}/".format(d)
     )
+
+
+def get_actor_data_from_username(username):
+
+    parts = username.split("@")
+
+    return {
+        "username": parts[0],
+        "domain": parts[1] if len(parts) > 1 else settings.FEDERATION_HOSTNAME,
+    }
+
+
+def get_actor_from_username_data_query(field, data):
+    if not data:
+        return Q(**{field: None})
+    if field:
+        return Q(
+            **{
+                "{}__preferred_username__iexact".format(field): data["username"],
+                "{}__domain__name__iexact".format(field): data["domain"],
+            }
+        )
+    else:
+        return Q(
+            **{
+                "preferred_username__iexact": data["username"],
+                "domain__name__iexact": data["domain"],
+            }
+        )

api/funkwhale_api/federation/views.py

@@ -2,17 +2,28 @@ from django import forms
 from django.core import paginator
 from django.http import HttpResponse
 from django.urls import reverse
-from rest_framework import exceptions, mixins, response, viewsets
+from rest_framework import exceptions, mixins, permissions, response, viewsets
 from rest_framework.decorators import action
 
 from funkwhale_api.common import preferences
+from funkwhale_api.moderation import models as moderation_models
 from funkwhale_api.music import models as music_models
 from funkwhale_api.music import utils as music_utils
 
 from . import activity, authentication, models, renderers, serializers, utils, webfinger
 
 
+class AuthenticatedIfAllowListEnabled(permissions.BasePermission):
+    def has_permission(self, request, view):
+        allow_list_enabled = preferences.get("moderation__allow_list_enabled")
+        if not allow_list_enabled:
+            return True
+        return bool(request.actor)
+
+
 class FederationMixin(object):
+    permission_classes = [AuthenticatedIfAllowListEnabled]
+
     def dispatch(self, request, *args, **kwargs):
         if not preferences.get("federation__enabled"):
             return HttpResponse(status=405)
@@ -20,7 +31,6 @@ class FederationMixin(object):
 
 
 class SharedViewSet(FederationMixin, viewsets.GenericViewSet):
-    permission_classes = []
     authentication_classes = [authentication.SignatureAuthentication]
     renderer_classes = renderers.get_ap_renderers()
 
@@ -38,7 +48,6 @@ class SharedViewSet(FederationMixin, viewsets.GenericViewSet):
 class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
     lookup_field = "preferred_username"
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     queryset = models.Actor.objects.local().select_related("user")
     serializer_class = serializers.ActorSerializer
@@ -73,12 +82,20 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
 class EditViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
     lookup_field = "uuid"
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     # queryset = common_models.Mutation.objects.local().select_related()
     # serializer_class = serializers.ActorSerializer
 
 
+class ReportViewSet(
+    FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+):
+    lookup_field = "uuid"
+    authentication_classes = [authentication.SignatureAuthentication]
+    renderer_classes = renderers.get_ap_renderers()
+    queryset = moderation_models.Report.objects.none()
+
+
 class WellKnownViewSet(viewsets.GenericViewSet):
     authentication_classes = []
     permission_classes = []
@@ -146,7 +163,6 @@ class MusicLibraryViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     serializer_class = serializers.LibrarySerializer
     queryset = music_models.Library.objects.all().select_related("actor")
@@ -201,7 +217,6 @@ class MusicUploadViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     queryset = music_models.Upload.objects.local().select_related(
         "library__actor", "track__artist", "track__album__artist"
@@ -219,7 +234,6 @@ class MusicArtistViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     queryset = music_models.Artist.objects.local()
     serializer_class = serializers.ArtistSerializer
@@ -230,7 +244,6 @@ class MusicAlbumViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     queryset = music_models.Album.objects.local().select_related("artist")
     serializer_class = serializers.AlbumSerializer
@@ -241,7 +254,6 @@ class MusicTrackViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     queryset = music_models.Track.objects.local().select_related(
         "album__artist", "artist"

api/funkwhale_api/history/serializers.py

@@ -1,6 +1,7 @@
 from rest_framework import serializers
 
 from funkwhale_api.activity import serializers as activity_serializers
+from funkwhale_api.federation import serializers as federation_serializers
 from funkwhale_api.music.serializers import TrackActivitySerializer, TrackSerializer
 from funkwhale_api.users.serializers import UserActivitySerializer, UserBasicSerializer
 
@@ -27,16 +28,22 @@ class ListeningActivitySerializer(activity_serializers.ModelSerializer):
 class ListeningSerializer(serializers.ModelSerializer):
     track = TrackSerializer(read_only=True)
     user = UserBasicSerializer(read_only=True)
+    actor = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Listening
-        fields = ("id", "user", "track", "creation_date")
+        fields = ("id", "user", "track", "creation_date", "actor")
 
     def create(self, validated_data):
         validated_data["user"] = self.context["user"]
 
         return super().create(validated_data)
 
+    def get_actor(self, obj):
+        actor = obj.user.actor
+        if actor:
+            return federation_serializers.APIActorSerializer(actor).data
+
 
 class ListeningWriteSerializer(serializers.ModelSerializer):
     class Meta:

api/funkwhale_api/history/urls.py

@@ -1,8 +1,8 @@
-from rest_framework import routers
+from funkwhale_api.common import routers
 
 from . import views
 
-router = routers.SimpleRouter()
+router = routers.OptionalSlashRouter()
 router.register(r"listenings", views.ListeningViewSet, "listenings")
 
 urlpatterns = router.urls

api/funkwhale_api/history/views.py

@@ -19,7 +19,7 @@ class ListeningViewSet(
 ):
 
     serializer_class = serializers.ListeningSerializer
-    queryset = models.Listening.objects.all().select_related("user")
+    queryset = models.Listening.objects.all().select_related("user__actor")
 
     permission_classes = [
         oauth_permissions.ScopePermission,
@@ -47,7 +47,7 @@ class ListeningViewSet(
         )
         tracks = Track.objects.with_playable_uploads(
             music_utils.get_actor_from_request(self.request)
-        ).select_related("artist", "album__artist")
+        ).select_related("artist", "album__artist", "attributed_to")
         return queryset.prefetch_related(Prefetch("track", queryset=tracks))
 
     def get_serializer_context(self):

api/funkwhale_api/instance/dynamic_preferences_registry.py

@@ -1,4 +1,6 @@
 from django.forms import widgets
+from django.core.validators import FileExtensionValidator
+
 from dynamic_preferences import types
 from dynamic_preferences.registries import global_preferences_registry
 
@@ -43,6 +45,72 @@ class InstanceLongDescription(types.StringPreference):
     field_kwargs = {"required": False}
 
 
+@global_preferences_registry.register
+class InstanceTerms(types.StringPreference):
+    show_in_api = True
+    section = instance
+    name = "terms"
+    verbose_name = "Terms of service"
+    default = ""
+    help_text = (
+        "Terms of service and privacy policy for your instance (markdown allowed)."
+    )
+    widget = widgets.Textarea
+    field_kwargs = {"required": False}
+
+
+@global_preferences_registry.register
+class InstanceRules(types.StringPreference):
+    show_in_api = True
+    section = instance
+    name = "rules"
+    verbose_name = "Rules"
+    default = ""
+    help_text = "Rules/Code of Conduct (markdown allowed)."
+    widget = widgets.Textarea
+    field_kwargs = {"required": False}
+
+
+@global_preferences_registry.register
+class InstanceContactEmail(types.StringPreference):
+    show_in_api = True
+    section = instance
+    name = "contact_email"
+    verbose_name = "Contact email"
+    default = ""
+    help_text = "A contact email for visitors who need to contact an admin or moderator"
+    field_kwargs = {"required": False}
+
+
+@global_preferences_registry.register
+class InstanceSupportMessage(types.StringPreference):
+    show_in_api = True
+    section = instance
+    name = "support_message"
+    verbose_name = "Support message"
+    default = ""
+    help_text = (
+        "A short message that will be displayed periodically to local users. "
+        "Use it to ask for financial support or anything else you might need. "
+        "(markdown allowed)."
+    )
+    widget = widgets.Textarea
+    field_kwargs = {"required": False}
+
+
+@global_preferences_registry.register
+class InstanceFunkwhaleSupportMessageEnabled(types.BooleanPreference):
+    show_in_api = True
+    section = instance
+    name = "funkwhale_support_message_enabled"
+    verbose_name = "Funkwhale Support message"
+    default = True
+    help_text = (
+        "If this is enabled, we will periodically display a message to encourage "
+        "local users to support Funkwhale."
+    )
+
+
 @global_preferences_registry.register
 class RavenDSN(types.StringPreference):
     show_in_api = True
@@ -115,3 +183,27 @@ class CustomCSS(types.StringPreference):
     )
     widget = widgets.Textarea
     field_kwargs = {"required": False}
+
+
+class ImageWidget(widgets.ClearableFileInput):
+    pass
+
+
+class ImagePreference(types.FilePreference):
+    widget = ImageWidget
+    field_kwargs = {
+        "validators": [
+            FileExtensionValidator(allowed_extensions=["png", "jpg", "jpeg", "webp"])
+        ]
+    }
+
+
+@global_preferences_registry.register
+class Banner(ImagePreference):
+    show_in_api = True
+    section = instance
+    name = "banner"
+    verbose_name = "Banner image"
+    default = None
+    help_text = "This banner will be displayed on your pod's landing and about page. At least 600x100px recommended."
+    field_kwargs = {"required": False}

api/funkwhale_api/instance/nodeinfo.py

@@ -2,7 +2,9 @@ import memoize.djangocache
 
 import funkwhale_api
 from funkwhale_api.common import preferences
-from funkwhale_api.federation import actors
+from funkwhale_api.federation import actors, models as federation_models
+from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.moderation import models as moderation_models
 from funkwhale_api.music import utils as music_utils
 
 from . import stats
@@ -12,32 +14,62 @@ memo = memoize.Memoizer(store, namespace="instance:stats")
 
 
 def get():
-    share_stats = preferences.get("instance__nodeinfo_stats_enabled")
+    all_preferences = preferences.all()
+    share_stats = all_preferences.get("instance__nodeinfo_stats_enabled")
+    allow_list_enabled = all_preferences.get("moderation__allow_list_enabled")
+    allow_list_public = all_preferences.get("moderation__allow_list_public")
+    banner = all_preferences.get("instance__banner")
+    unauthenticated_report_types = all_preferences.get(
+        "moderation__unauthenticated_report_types"
+    )
+    if allow_list_enabled and allow_list_public:
+        allowed_domains = list(
+            federation_models.Domain.objects.filter(allowed=True)
+            .order_by("name")
+            .values_list("name", flat=True)
+        )
+    else:
+        allowed_domains = None
     data = {
         "version": "2.0",
         "software": {"name": "funkwhale", "version": funkwhale_api.__version__},
         "protocols": ["activitypub"],
         "services": {"inbound": [], "outbound": []},
-        "openRegistrations": preferences.get("users__registration_enabled"),
+        "openRegistrations": all_preferences.get("users__registration_enabled"),
         "usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}},
         "metadata": {
             "actorId": actors.get_service_actor().fid,
-            "private": preferences.get("instance__nodeinfo_private"),
-            "shortDescription": preferences.get("instance__short_description"),
-            "longDescription": preferences.get("instance__long_description"),
-            "nodeName": preferences.get("instance__name"),
+            "private": all_preferences.get("instance__nodeinfo_private"),
+            "shortDescription": all_preferences.get("instance__short_description"),
+            "longDescription": all_preferences.get("instance__long_description"),
+            "rules": all_preferences.get("instance__rules"),
+            "contactEmail": all_preferences.get("instance__contact_email"),
+            "terms": all_preferences.get("instance__terms"),
+            "nodeName": all_preferences.get("instance__name"),
+            "banner": federation_utils.full_url(banner.url) if banner else None,
+            "defaultUploadQuota": all_preferences.get("users__upload_quota"),
             "library": {
-                "federationEnabled": preferences.get("federation__enabled"),
-                "federationNeedsApproval": preferences.get(
+                "federationEnabled": all_preferences.get("federation__enabled"),
+                "federationNeedsApproval": all_preferences.get(
                     "federation__music_needs_approval"
                 ),
-                "anonymousCanListen": not preferences.get(
+                "anonymousCanListen": not all_preferences.get(
                     "common__api_authentication_required"
                 ),
             },
             "supportedUploadExtensions": music_utils.SUPPORTED_EXTENSIONS,
+            "allowList": {"enabled": allow_list_enabled, "domains": allowed_domains},
+            "reportTypes": [
+                {"type": t, "label": l, "anonymous": t in unauthenticated_report_types}
+                for t, l in moderation_models.REPORT_TYPES
+            ],
+            "funkwhaleSupportMessageEnabled": all_preferences.get(
+                "instance__funkwhale_support_message_enabled"
+            ),
+            "instanceSupportMessage": all_preferences.get("instance__support_message"),
         },
     }
+
     if share_stats:
         getter = memo(lambda: stats.get(), max_age=600)
         statistics = getter()

api/funkwhale_api/instance/urls.py

@@ -1,12 +1,12 @@
 from django.conf.urls import url
-from rest_framework import routers
+from funkwhale_api.common import routers
 
 from . import views
 
-admin_router = routers.SimpleRouter()
+admin_router = routers.OptionalSlashRouter()
 admin_router.register(r"admin/settings", views.AdminSettings, "admin-settings")
 
 urlpatterns = [
-    url(r"^nodeinfo/2.0/$", views.NodeInfo.as_view(), name="nodeinfo-2.0"),
-    url(r"^settings/$", views.InstanceSettings.as_view(), name="settings"),
+    url(r"^nodeinfo/2.0/?$", views.NodeInfo.as_view(), name="nodeinfo-2.0"),
+    url(r"^settings/?$", views.InstanceSettings.as_view(), name="settings"),
 ] + admin_router.urls

api/funkwhale_api/manage/filters.py

@@ -1,18 +1,21 @@
 from django import forms
 from django.db.models import Q
-from django.conf import settings
 
 import django_filters
 from django_filters import rest_framework as filters
 
 from funkwhale_api.common import fields
+from funkwhale_api.common import filters as common_filters
 from funkwhale_api.common import search
 
 from funkwhale_api.federation import models as federation_models
 from funkwhale_api.federation import utils as federation_utils
 from funkwhale_api.moderation import models as moderation_models
+from funkwhale_api.moderation import serializers as moderation_serializers
+from funkwhale_api.moderation import utils as moderation_utils
 from funkwhale_api.music import models as music_models
 from funkwhale_api.users import models as users_models
+from funkwhale_api.tags import models as tags_models
 
 
 class ActorField(forms.CharField):
@@ -21,24 +24,12 @@ class ActorField(forms.CharField):
         if not value:
             return value
 
-        parts = value.split("@")
-
-        return {
-            "username": parts[0],
-            "domain": parts[1] if len(parts) > 1 else settings.FEDERATION_HOSTNAME,
-        }
+        return federation_utils.get_actor_data_from_username(value)
 
 
 def get_actor_filter(actor_field):
     def handler(v):
-        if not v:
-            return Q(**{actor_field: None})
-        return Q(
-            **{
-                "{}__preferred_username__iexact".format(actor_field): v["username"],
-                "{}__domain__name__iexact".format(actor_field): v["domain"],
-            }
-        )
+        return federation_utils.get_actor_from_username_data_query(actor_field, v)
 
     return {"field": ActorField(), "handler": handler}
 
@@ -61,6 +52,7 @@ class ManageArtistFilterSet(filters.FilterSet):
                     "field": forms.IntegerField(),
                     "distinct": True,
                 },
+                "tag": {"to": "tagged_items__tag__name", "distinct": True},
             },
         )
     )
@@ -90,6 +82,7 @@ class ManageAlbumFilterSet(filters.FilterSet):
                     "field": forms.IntegerField(),
                     "distinct": True,
                 },
+                "tag": {"to": "tagged_items__tag__name", "distinct": True},
             },
         )
     )
@@ -128,6 +121,7 @@ class ManageTrackFilterSet(filters.FilterSet):
                     "field": forms.IntegerField(),
                     "distinct": True,
                 },
+                "tag": {"to": "tagged_items__tag__name", "distinct": True},
             },
         )
     )
@@ -235,12 +229,23 @@ class ManageUploadFilterSet(filters.FilterSet):
         ]
 
 
+def filter_allowed(queryset, name, value):
+    """
+    If value=false, we want to include object with value=null as well
+    """
+    if value:
+        return queryset.filter(allowed=True)
+    else:
+        return queryset.filter(Q(allowed=False) | Q(allowed__isnull=True))
+
+
 class ManageDomainFilterSet(filters.FilterSet):
     q = fields.SearchFilter(search_fields=["name"])
+    allowed = filters.BooleanFilter(method=filter_allowed)
 
     class Meta:
         model = federation_models.Domain
-        fields = ["name"]
+        fields = ["name", "allowed"]
 
 
 class ManageActorFilterSet(filters.FilterSet):
@@ -320,6 +325,10 @@ class ManageInstancePolicyFilterSet(filters.FilterSet):
         ]
     )
 
+    target_domain = filters.CharFilter("target_domain__name")
+    target_account_domain = filters.CharFilter("target_actor__domain__name")
+    target_account_username = filters.CharFilter("target_actor__preferred_username")
+
     class Meta:
         model = moderation_models.InstancePolicy
         fields = [
@@ -328,4 +337,60 @@ class ManageInstancePolicyFilterSet(filters.FilterSet):
             "silence_activity",
             "silence_notifications",
             "reject_media",
+            "target_domain",
+            "target_account_domain",
+            "target_account_username",
         ]
+
+
+class ManageTagFilterSet(filters.FilterSet):
+    q = fields.SearchFilter(search_fields=["name"])
+
+    class Meta:
+        model = tags_models.Tag
+        fields = ["q"]
+
+
+class ManageReportFilterSet(filters.FilterSet):
+    q = fields.SmartSearchFilter(
+        config=search.SearchConfig(
+            search_fields={"summary": {"to": "summary"}},
+            filter_fields={
+                "uuid": {"to": "uuid"},
+                "id": {"to": "id"},
+                "resolved": common_filters.get_boolean_filter("is_handled"),
+                "domain": {"to": "target_owner__domain_id"},
+                "category": {"to": "type"},
+                "submitter": get_actor_filter("submitter"),
+                "assigned_to": get_actor_filter("assigned_to"),
+                "target_owner": get_actor_filter("target_owner"),
+                "submitter_email": {"to": "submitter_email"},
+                "target": common_filters.get_generic_relation_filter(
+                    "target", moderation_serializers.TARGET_CONFIG
+                ),
+            },
+        )
+    )
+
+    class Meta:
+        model = moderation_models.Report
+        fields = ["q", "is_handled", "type", "submitter_email"]
+
+
+class ManageNoteFilterSet(filters.FilterSet):
+    q = fields.SmartSearchFilter(
+        config=search.SearchConfig(
+            search_fields={"summary": {"to": "summary"}},
+            filter_fields={
+                "uuid": {"to": "uuid"},
+                "author": get_actor_filter("author"),
+                "target": common_filters.get_generic_relation_filter(
+                    "target", moderation_utils.NOTE_TARGET_FIELDS
+                ),
+            },
+        )
+    )
+
+    class Meta:
+        model = moderation_models.Note
+        fields = ["q"]

api/funkwhale_api/manage/serializers.py

@@ -1,15 +1,20 @@
+from django.conf import settings
 from django.db import transaction
 
 from rest_framework import serializers
 
+from funkwhale_api.common import fields as common_fields
 from funkwhale_api.common import serializers as common_serializers
 from funkwhale_api.common import utils as common_utils
 from funkwhale_api.federation import models as federation_models
 from funkwhale_api.federation import fields as federation_fields
 from funkwhale_api.federation import tasks as federation_tasks
 from funkwhale_api.moderation import models as moderation_models
+from funkwhale_api.moderation import serializers as moderation_serializers
+from funkwhale_api.moderation import utils as moderation_utils
 from funkwhale_api.music import models as music_models
 from funkwhale_api.music import serializers as music_serializers
+from funkwhale_api.tags import models as tags_models
 from funkwhale_api.users import models as users_models
 
 from . import filters
@@ -136,6 +141,7 @@ class ManageDomainSerializer(serializers.ModelSerializer):
             "nodeinfo",
             "nodeinfo_fetch_date",
             "instance_policy",
+            "allowed",
         ]
         read_only_fields = [
             "creation_date",
@@ -151,8 +157,17 @@ class ManageDomainSerializer(serializers.ModelSerializer):
         return getattr(o, "outbox_activities_count", 0)
 
 
+class ManageDomainUpdateSerializer(ManageDomainSerializer):
+    class Meta(ManageDomainSerializer.Meta):
+        read_only_fields = ["name"] + ManageDomainSerializer.Meta.read_only_fields
+
+
 class ManageDomainActionSerializer(common_serializers.ActionSerializer):
-    actions = [common_serializers.Action("purge", allow_all=False)]
+    actions = [
+        common_serializers.Action("purge", allow_all=False),
+        common_serializers.Action("allow_list_add", allow_all=True),
+        common_serializers.Action("allow_list_remove", allow_all=True),
+    ]
     filterset_class = filters.ManageDomainFilterSet
     pk_field = "name"
 
@@ -161,8 +176,18 @@ class ManageDomainActionSerializer(common_serializers.ActionSerializer):
         ids = objects.values_list("pk", flat=True)
         common_utils.on_commit(federation_tasks.purge_actors.delay, domains=list(ids))
 
+    @transaction.atomic
+    def handle_allow_list_add(self, objects):
+        objects.update(allowed=True)
+
+    @transaction.atomic
+    def handle_allow_list_remove(self, objects):
+        objects.update(allowed=False)
+
 
 class ManageBaseActorSerializer(serializers.ModelSerializer):
+    is_local = serializers.SerializerMethodField()
+
     class Meta:
         model = federation_models.Actor
         fields = [
@@ -181,9 +206,13 @@ class ManageBaseActorSerializer(serializers.ModelSerializer):
             "outbox_url",
             "shared_inbox_url",
             "manually_approves_followers",
+            "is_local",
         ]
         read_only_fields = ["creation_date", "instance_policy"]
 
+    def get_is_local(self, o):
+        return o.domain_id == settings.FEDERATION_HOSTNAME
+
 
 class ManageActorSerializer(ManageBaseActorSerializer):
     uploads_count = serializers.SerializerMethodField()
@@ -358,6 +387,7 @@ class ManageArtistSerializer(ManageBaseArtistSerializer):
     albums = ManageNestedAlbumSerializer(many=True)
     tracks = ManageNestedTrackSerializer(many=True)
     attributed_to = ManageBaseActorSerializer()
+    tags = serializers.SerializerMethodField()
 
     class Meta:
         model = music_models.Artist
@@ -365,8 +395,13 @@ class ManageArtistSerializer(ManageBaseArtistSerializer):
             "albums",
             "tracks",
             "attributed_to",
+            "tags",
         ]
 
+    def get_tags(self, obj):
+        tagged_items = getattr(obj, "_prefetched_tagged_items", [])
+        return [ti.tag.name for ti in tagged_items]
+
 
 class ManageNestedArtistSerializer(ManageBaseArtistSerializer):
     pass
@@ -376,6 +411,7 @@ class ManageAlbumSerializer(ManageBaseAlbumSerializer):
     tracks = ManageNestedTrackSerializer(many=True)
     attributed_to = ManageBaseActorSerializer()
     artist = ManageNestedArtistSerializer()
+    tags = serializers.SerializerMethodField()
 
     class Meta:
         model = music_models.Album
@@ -383,8 +419,13 @@ class ManageAlbumSerializer(ManageBaseAlbumSerializer):
             "artist",
             "tracks",
             "attributed_to",
+            "tags",
         ]
 
+    def get_tags(self, obj):
+        tagged_items = getattr(obj, "_prefetched_tagged_items", [])
+        return [ti.tag.name for ti in tagged_items]
+
 
 class ManageTrackAlbumSerializer(ManageBaseAlbumSerializer):
     artist = ManageNestedArtistSerializer()
@@ -399,6 +440,7 @@ class ManageTrackSerializer(ManageNestedTrackSerializer):
     album = ManageTrackAlbumSerializer()
     attributed_to = ManageBaseActorSerializer()
     uploads_count = serializers.SerializerMethodField()
+    tags = serializers.SerializerMethodField()
 
     class Meta:
         model = music_models.Track
@@ -407,11 +449,16 @@ class ManageTrackSerializer(ManageNestedTrackSerializer):
             "album",
             "attributed_to",
             "uploads_count",
+            "tags",
         ]
 
     def get_uploads_count(self, obj):
         return getattr(obj, "uploads_count", None)
 
+    def get_tags(self, obj):
+        tagged_items = getattr(obj, "_prefetched_tagged_items", [])
+        return [ti.tag.name for ti in tagged_items]
+
 
 class ManageTrackActionSerializer(common_serializers.ActionSerializer):
     actions = [common_serializers.Action("delete", allow_all=False)]
@@ -482,6 +529,15 @@ class ManageLibrarySerializer(serializers.ModelSerializer):
             "followers_url",
             "actor",
         ]
+        read_only_fields = [
+            "fid",
+            "uuid",
+            "id",
+            "url",
+            "domain",
+            "actor",
+            "creation_date",
+        ]
 
     def get_uploads_count(self, obj):
         return getattr(obj, "_uploads_count", obj.uploads_count)
@@ -546,3 +602,101 @@ class ManageUploadSerializer(serializers.ModelSerializer):
             "track",
             "library",
         )
+
+
+class ManageTagSerializer(ManageBaseAlbumSerializer):
+
+    tracks_count = serializers.SerializerMethodField()
+    albums_count = serializers.SerializerMethodField()
+    artists_count = serializers.SerializerMethodField()
+
+    class Meta:
+        model = tags_models.Tag
+        fields = [
+            "id",
+            "name",
+            "creation_date",
+            "tracks_count",
+            "albums_count",
+            "artists_count",
+        ]
+
+    def get_tracks_count(self, obj):
+        return getattr(obj, "_tracks_count", None)
+
+    def get_albums_count(self, obj):
+        return getattr(obj, "_albums_count", None)
+
+    def get_artists_count(self, obj):
+        return getattr(obj, "_artists_count", None)
+
+
+class ManageTagActionSerializer(common_serializers.ActionSerializer):
+    actions = [common_serializers.Action("delete", allow_all=False)]
+    filterset_class = filters.ManageTagFilterSet
+    pk_field = "name"
+
+    @transaction.atomic
+    def handle_delete(self, objects):
+        return objects.delete()
+
+
+class ManageBaseNoteSerializer(serializers.ModelSerializer):
+    author = ManageBaseActorSerializer(required=False, read_only=True)
+
+    class Meta:
+        model = moderation_models.Note
+        fields = ["id", "uuid", "creation_date", "summary", "author"]
+        read_only_fields = ["uuid", "creation_date", "author"]
+
+
+class ManageNoteSerializer(ManageBaseNoteSerializer):
+    target = common_fields.GenericRelation(moderation_utils.NOTE_TARGET_FIELDS)
+
+    class Meta(ManageBaseNoteSerializer.Meta):
+        fields = ManageBaseNoteSerializer.Meta.fields + ["target"]
+
+
+class ManageReportSerializer(serializers.ModelSerializer):
+    assigned_to = ManageBaseActorSerializer()
+    target_owner = ManageBaseActorSerializer()
+    submitter = ManageBaseActorSerializer()
+    target = moderation_serializers.TARGET_FIELD
+    notes = serializers.SerializerMethodField()
+
+    class Meta:
+        model = moderation_models.Report
+        fields = [
+            "id",
+            "uuid",
+            "fid",
+            "creation_date",
+            "handled_date",
+            "summary",
+            "type",
+            "target",
+            "target_state",
+            "is_handled",
+            "assigned_to",
+            "target_owner",
+            "submitter",
+            "submitter_email",
+            "notes",
+        ]
+        read_only_fields = [
+            "id",
+            "uuid",
+            "fid",
+            "submitter",
+            "submitter_email",
+            "creation_date",
+            "handled_date",
+            "target",
+            "target_state",
+            "target_owner",
+            "summary",
+        ]
+
+    def get_notes(self, o):
+        notes = getattr(o, "_prefetched_notes", [])
+        return ManageBaseNoteSerializer(notes, many=True).data

api/funkwhale_api/manage/urls.py

@@ -1,29 +1,32 @@
 from django.conf.urls import include, url
-from rest_framework import routers
+from funkwhale_api.common import routers
 
 from . import views
 
-federation_router = routers.SimpleRouter()
+federation_router = routers.OptionalSlashRouter()
 federation_router.register(r"domains", views.ManageDomainViewSet, "domains")
 
-library_router = routers.SimpleRouter()
+library_router = routers.OptionalSlashRouter()
 library_router.register(r"albums", views.ManageAlbumViewSet, "albums")
 library_router.register(r"artists", views.ManageArtistViewSet, "artists")
 library_router.register(r"libraries", views.ManageLibraryViewSet, "libraries")
 library_router.register(r"tracks", views.ManageTrackViewSet, "tracks")
 library_router.register(r"uploads", views.ManageUploadViewSet, "uploads")
 
-moderation_router = routers.SimpleRouter()
+moderation_router = routers.OptionalSlashRouter()
 moderation_router.register(
     r"instance-policies", views.ManageInstancePolicyViewSet, "instance-policies"
 )
+moderation_router.register(r"reports", views.ManageReportViewSet, "reports")
+moderation_router.register(r"notes", views.ManageNoteViewSet, "notes")
 
-users_router = routers.SimpleRouter()
+users_router = routers.OptionalSlashRouter()
 users_router.register(r"users", views.ManageUserViewSet, "users")
 users_router.register(r"invitations", views.ManageInvitationViewSet, "invitations")
 
-other_router = routers.SimpleRouter()
+other_router = routers.OptionalSlashRouter()
 other_router.register(r"accounts", views.ManageActorViewSet, "accounts")
+other_router.register(r"tags", views.ManageTagViewSet, "tags")
 
 urlpatterns = [
     url(

api/funkwhale_api/manage/views.py

@@ -2,7 +2,7 @@ from rest_framework import mixins, response, viewsets
 from rest_framework import decorators as rest_decorators
 
 from django.db.models import Count, Prefetch, Q, Sum, OuterRef, Subquery
-from django.db.models.functions import Coalesce
+from django.db.models.functions import Coalesce, Length
 from django.shortcuts import get_object_or_404
 
 from funkwhale_api.common import models as common_models
@@ -12,8 +12,10 @@ from funkwhale_api.federation import models as federation_models
 from funkwhale_api.federation import tasks as federation_tasks
 from funkwhale_api.history import models as history_models
 from funkwhale_api.music import models as music_models
+from funkwhale_api.music import views as music_views
 from funkwhale_api.moderation import models as moderation_models
 from funkwhale_api.playlists import models as playlists_models
+from funkwhale_api.tags import models as tags_models
 from funkwhale_api.users import models as users_models
 
 
@@ -39,6 +41,7 @@ def get_stats(tracks, target):
     ).count()
     data["libraries"] = uploads.values_list("library", flat=True).distinct().count()
     data["uploads"] = uploads.count()
+    data["reports"] = moderation_models.Report.objects.get_for_target(target).count()
     data.update(get_media_stats(uploads))
     return data
 
@@ -70,6 +73,7 @@ class ManageArtistViewSet(
                     tracks_count=Count("tracks")
                 ),
             ),
+            music_views.TAG_PREFETCH,
         )
     )
     serializer_class = serializers.ManageArtistSerializer
@@ -107,7 +111,7 @@ class ManageAlbumViewSet(
         music_models.Album.objects.all()
         .order_by("-id")
         .select_related("attributed_to", "artist")
-        .prefetch_related("tracks")
+        .prefetch_related("tracks", music_views.TAG_PREFETCH)
     )
     serializer_class = serializers.ManageAlbumSerializer
     filterset_class = filters.ManageAlbumFilterSet
@@ -151,6 +155,7 @@ class ManageTrackViewSet(
         .order_by("-id")
         .select_related("attributed_to", "artist", "album__artist")
         .annotate(uploads_count=Coalesce(Subquery(uploads_subquery), 0))
+        .prefetch_related(music_views.TAG_PREFETCH)
     )
     serializer_class = serializers.ManageTrackSerializer
     filterset_class = filters.ManageTrackFilterSet
@@ -200,6 +205,7 @@ follows_subquery = (
 class ManageLibraryViewSet(
     mixins.ListModelMixin,
     mixins.RetrieveModelMixin,
+    mixins.UpdateModelMixin,
     mixins.DestroyModelMixin,
     viewsets.GenericViewSet,
 ):
@@ -243,6 +249,7 @@ class ManageLibraryViewSet(
             "tracks": tracks.count(),
             "albums": albums.count(),
             "artists": len(artists),
+            "reports": moderation_models.Report.objects.get_for_target(library).count(),
         }
         data.update(get_media_stats(uploads.all()))
         return response.Response(data, status=200)
@@ -339,6 +346,7 @@ class ManageDomainViewSet(
     mixins.CreateModelMixin,
     mixins.ListModelMixin,
     mixins.RetrieveModelMixin,
+    mixins.UpdateModelMixin,
     viewsets.GenericViewSet,
 ):
     lookup_value_regex = r"[a-zA-Z0-9\-\.]+"
@@ -361,6 +369,13 @@ class ManageDomainViewSet(
         "instance_policy",
     ]
 
+    def get_serializer_class(self):
+        if self.action in ["update", "partial_update"]:
+            # A dedicated serializer for update
+            # to ensure domain name can't be changed
+            return serializers.ManageDomainUpdateSerializer
+        return super().get_serializer_class()
+
     def perform_create(self, serializer):
         domain = serializer.save()
         federation_tasks.update_domain_nodeinfo(domain_name=domain.name)
@@ -444,3 +459,115 @@ class ManageInstancePolicyViewSet(
 
     def perform_create(self, serializer):
         serializer.save(actor=self.request.user.actor)
+
+
+class ManageReportViewSet(
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.UpdateModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    queryset = (
+        moderation_models.Report.objects.all()
+        .order_by("-creation_date")
+        .select_related(
+            "submitter", "target_owner", "assigned_to", "target_content_type"
+        )
+        .prefetch_related("target")
+        .prefetch_related(
+            Prefetch(
+                "notes",
+                queryset=moderation_models.Note.objects.order_by(
+                    "creation_date"
+                ).select_related("author"),
+                to_attr="_prefetched_notes",
+            )
+        )
+    )
+    serializer_class = serializers.ManageReportSerializer
+    filterset_class = filters.ManageReportFilterSet
+    required_scope = "instance:reports"
+    ordering_fields = ["id", "creation_date", "handled_date"]
+
+    def perform_update(self, serializer):
+        is_handled = serializer.instance.is_handled
+        if not is_handled and serializer.validated_data.get("is_handled") is True:
+            # report was resolved, we assign to the mod making the request
+            serializer.save(assigned_to=self.request.user.actor)
+        else:
+            serializer.save()
+
+
+class ManageNoteViewSet(
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    mixins.CreateModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    queryset = (
+        moderation_models.Note.objects.all()
+        .order_by("-creation_date")
+        .select_related("author", "target_content_type")
+        .prefetch_related("target")
+    )
+    serializer_class = serializers.ManageNoteSerializer
+    filterset_class = filters.ManageNoteFilterSet
+    required_scope = "instance:notes"
+    ordering_fields = ["id", "creation_date"]
+
+    def perform_create(self, serializer):
+        author = self.request.user.actor
+        return serializer.save(author=author)
+
+
+class ManageTagViewSet(
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    mixins.CreateModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "name"
+    queryset = (
+        tags_models.Tag.objects.all()
+        .order_by("-creation_date")
+        .annotate(items_count=Count("tagged_items"))
+        .annotate(length=Length("name"))
+    )
+    serializer_class = serializers.ManageTagSerializer
+    filterset_class = filters.ManageTagFilterSet
+    required_scope = "instance:libraries"
+    ordering_fields = ["id", "creation_date", "name", "items_count", "length"]
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        from django.contrib.contenttypes.models import ContentType
+
+        album_ct = ContentType.objects.get_for_model(music_models.Album)
+        track_ct = ContentType.objects.get_for_model(music_models.Track)
+        artist_ct = ContentType.objects.get_for_model(music_models.Artist)
+        queryset = queryset.annotate(
+            _albums_count=Count(
+                "tagged_items", filter=Q(tagged_items__content_type=album_ct)
+            ),
+            _tracks_count=Count(
+                "tagged_items", filter=Q(tagged_items__content_type=track_ct)
+            ),
+            _artists_count=Count(
+                "tagged_items", filter=Q(tagged_items__content_type=artist_ct)
+            ),
+        )
+        return queryset
+
+    @rest_decorators.action(methods=["post"], detail=False)
+    def action(self, request, *args, **kwargs):
+        queryset = self.get_queryset()
+        serializer = serializers.ManageTagActionSerializer(
+            request.data, queryset=queryset
+        )
+        serializer.is_valid(raise_exception=True)
+        result = serializer.save()
+        return response.Response(result, status=200)

api/funkwhale_api/moderation/admin.py

@@ -30,6 +30,22 @@ class InstancePolicyAdmin(admin.ModelAdmin):
     list_select_related = True
 
 
+@admin.register(models.Report)
+class ReportAdmin(admin.ModelAdmin):
+    list_display = [
+        "uuid",
+        "submitter",
+        "type",
+        "assigned_to",
+        "is_handled",
+        "creation_date",
+        "handled_date",
+    ]
+    list_filter = ["type", "is_handled"]
+    search_fields = ["summary"]
+    list_select_related = True
+
+
 @admin.register(models.UserFilter)
 class UserFilterAdmin(admin.ModelAdmin):
     list_display = ["uuid", "user", "target_artist", "creation_date"]

api/funkwhale_api/moderation/apps.py

@@ -0,0 +1,13 @@
+from django.apps import AppConfig, apps
+
+from . import mrf
+
+
+class ModerationConfig(AppConfig):
+    name = "funkwhale_api.moderation"
+
+    def ready(self):
+        super().ready()
+
+        app_names = [app.name for app in apps.app_configs.values()]
+        mrf.inbox.autodiscover(app_names)

api/funkwhale_api/moderation/dynamic_preferences_registry.py

@@ -0,0 +1,42 @@
+from dynamic_preferences import types
+from dynamic_preferences.registries import global_preferences_registry
+
+from funkwhale_api.common import preferences as common_preferences
+
+from . import models
+
+moderation = types.Section("moderation")
+
+
+@global_preferences_registry.register
+class AllowListEnabled(types.BooleanPreference):
+    section = moderation
+    name = "allow_list_enabled"
+    verbose_name = "Enable allow-listing"
+    help_text = "If enabled, only interactions with explicitely allowed domains will be authorized."
+    default = False
+
+
+@global_preferences_registry.register
+class AllowListPublic(types.BooleanPreference):
+    section = moderation
+    name = "allow_list_public"
+    verbose_name = "Publish your allowed-domains list"
+    help_text = (
+        "If enabled, everyone will be able to retrieve the list of domains you allowed. "
+        "This is useful on open setups, to help people decide if they want to join your pod, or to "
+        "make your moderation policy public."
+    )
+    default = False
+
+
+@global_preferences_registry.register
+class UnauthenticatedReportTypes(common_preferences.StringListPreference):
+    show_in_api = True
+    section = moderation
+    name = "unauthenticated_report_types"
+    default = ["takedown_request", "illegal_content"]
+    verbose_name = "Accountless report categories"
+    help_text = "A list of categories for which external users (without an account) can submit a report"
+    choices = models.REPORT_TYPES
+    field_kwargs = {"choices": choices, "required": False}

api/funkwhale_api/moderation/factories.py

@@ -5,6 +5,8 @@ from funkwhale_api.federation import factories as federation_factories
 from funkwhale_api.music import factories as music_factories
 from funkwhale_api.users import factories as users_factories
 
+from . import serializers
+
 
 @registry.register
 class InstancePolicyFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
@@ -37,3 +39,37 @@ class UserFilterFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
         for_artist = factory.Trait(
             target_artist=factory.SubFactory(music_factories.ArtistFactory)
         )
+
+
+@registry.register
+class NoteFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
+    author = factory.SubFactory(federation_factories.ActorFactory)
+    target = None
+    summary = factory.Faker("paragraph")
+
+    class Meta:
+        model = "moderation.Note"
+
+
+@registry.register
+class ReportFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
+    submitter = factory.SubFactory(federation_factories.ActorFactory)
+    target = factory.SubFactory(music_factories.ArtistFactory)
+    summary = factory.Faker("paragraph")
+    type = "other"
+
+    class Meta:
+        model = "moderation.Report"
+
+    class Params:
+        anonymous = factory.Trait(actor=None, submitter_email=factory.Faker("email"))
+        assigned = factory.Trait(
+            assigned_to=factory.SubFactory(federation_factories.ActorFactory)
+        )
+
+    @factory.post_generation
+    def _set_target_owner(self, create, extracted, **kwargs):
+        if not self.target:
+            return
+
+        self.target_owner = serializers.get_target_owner(self.target)

api/funkwhale_api/moderation/management/commands/mrf_check.py

@@ -0,0 +1,117 @@
+import json
+import sys
+import uuid
+import logging
+
+from django.core.management.base import BaseCommand, CommandError
+from django.core import validators
+
+from django.conf import settings
+
+from funkwhale_api.common import session
+from funkwhale_api.federation import models
+from funkwhale_api.moderation import mrf
+
+
+def is_uuid(v):
+    try:
+        uuid.UUID(v)
+    except ValueError:
+        return False
+    return True
+
+
+def is_url(v):
+    validator = validators.URLValidator()
+    try:
+        validator(v)
+    except (ValueError, validators.ValidationError):
+        return False
+
+    return True
+
+
+class Command(BaseCommand):
+    help = "Check a given message against all or a specific MRF rule"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "type",
+            type=str,
+            choices=["inbox"],
+            help=("The type of MRF. Only inbox is supported at the moment"),
+        )
+        parser.add_argument(
+            "input",
+            nargs="?",
+            help=(
+                "The path to a file containing JSON data. Use - to read from stdin. "
+                "If no input is provided, registered MRF policies will be listed "
+                "instead.",
+            ),
+        )
+        parser.add_argument(
+            "--policy",
+            "-p",
+            dest="policies",
+            nargs="+",
+            default=False,
+            help="Restrict to a list of MRF policies that will be applied, in that order",
+        )
+
+    def handle(self, *args, **options):
+        logger = logging.getLogger("funkwhale.mrf")
+        logger.setLevel(logging.DEBUG)
+        logger.addHandler(logging.StreamHandler(stream=sys.stderr))
+
+        input = options["input"]
+        if not input:
+            registry = getattr(mrf, options["type"])
+            self.stdout.write(
+                "No input given, listing registered policies for '{}' MRF:".format(
+                    options["type"]
+                )
+            )
+            for name in registry.keys():
+                self.stdout.write("- {}".format(name))
+            return
+        raw_content = None
+        content = None
+        if input == "-":
+            raw_content = sys.stdin.read()
+        elif is_uuid(input):
+            self.stderr.write("UUID provided, retrieving payload from db")
+            content = models.Activity.objects.get(uuid=input).payload
+        elif is_url(input):
+            response = session.get_session().get(
+                input,
+                timeout=5,
+                verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+                headers={"Content-Type": "application/activity+json"},
+            )
+            response.raise_for_status()
+            content = response.json()
+        else:
+            with open(input, "rb") as f:
+                raw_content = f.read()
+        content = json.loads(raw_content) if content is None else content
+
+        policies = options["policies"] or []
+        registry = getattr(mrf, options["type"])
+        for policy in policies:
+            if policy not in registry:
+                raise CommandError(
+                    "Unknown policy '{}' for MRF '{}'".format(policy, options["type"])
+                )
+
+        payload, updated = registry.apply(content, policies=policies)
+        if not payload:
+            self.stderr.write("Payload was discarded by MRF")
+        elif updated:
+            self.stderr.write("Payload was modified by MRF")
+            self.stderr.write("Initial payload:\n")
+            self.stdout.write(json.dumps(content, indent=2, sort_keys=True))
+            self.stderr.write("Modified payload:\n")
+            self.stdout.write(json.dumps(payload, indent=2, sort_keys=True))
+        else:
+            self.stderr.write("Payload left untouched by MRF")

api/funkwhale_api/moderation/migrations/0003_report.py

@@ -0,0 +1,100 @@
+# Generated by Django 2.2.3 on 2019-08-01 08:34
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("federation", "0020_auto_20190730_0846"),
+        ("moderation", "0002_auto_20190213_0927"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Report",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("fid", models.URLField(db_index=True, max_length=500, unique=True)),
+                ("url", models.URLField(blank=True, max_length=500, null=True)),
+                ("uuid", models.UUIDField(default=uuid.uuid4, unique=True)),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("summary", models.TextField(max_length=50000, null=True)),
+                ("handled_date", models.DateTimeField(null=True)),
+                ("is_handled", models.BooleanField(default=False)),
+                (
+                    "type",
+                    models.CharField(
+                        choices=[
+                            ("takedown_request", "Takedown request"),
+                            ("invalid_metadata", "Invalid metadata"),
+                            ("illegal_content", "Illegal content"),
+                            ("offensive_content", "Offensive content"),
+                            ("other", "Other"),
+                        ],
+                        max_length=40,
+                    ),
+                ),
+                ("submitter_email", models.EmailField(max_length=254, null=True)),
+                ("target_id", models.IntegerField(null=True)),
+                (
+                    "target_state",
+                    django.contrib.postgres.fields.jsonb.JSONField(null=True),
+                ),
+                (
+                    "submitter",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="reports",
+                        to="federation.Actor",
+                    ),
+                ),
+                (
+                    "assigned_to",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="assigned_reports",
+                        to="federation.Actor",
+                    ),
+                ),
+                (
+                    "target_content_type",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="contenttypes.ContentType",
+                    ),
+                ),
+                (
+                    "target_owner",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="federation.Actor",
+                    ),
+                ),
+            ],
+            options={"abstract": False},
+        )
+    ]

api/funkwhale_api/moderation/migrations/0004_note.py

@@ -0,0 +1,30 @@
+# Generated by Django 2.2.4 on 2019-08-29 09:08
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0020_auto_20190730_0846'),
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('moderation', '0003_report'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Note',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('summary', models.TextField(max_length=50000)),
+                ('target_id', models.IntegerField(null=True)),
+                ('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='moderation_notes', to='federation.Actor')),
+                ('target_content_type', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+            ],
+        ),
+    ]

api/funkwhale_api/moderation/models.py

@@ -1,9 +1,19 @@
 import urllib.parse
 import uuid
 
+from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
+from django.contrib.contenttypes.models import ContentType
+from django.contrib.postgres.fields import JSONField
 from django.db import models
+from django.db.models.signals import pre_save
+from django.dispatch import receiver
+from django.urls import reverse
 from django.utils import timezone
 
+from funkwhale_api.common import models as common_models
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import utils as federation_utils
+
 
 class InstancePolicyQuerySet(models.QuerySet):
     def active(self):
@@ -92,3 +102,92 @@ class UserFilter(models.Model):
     def target(self):
         if self.target_artist:
             return {"type": "artist", "obj": self.target_artist}
+
+
+REPORT_TYPES = [
+    ("takedown_request", "Takedown request"),
+    ("invalid_metadata", "Invalid metadata"),
+    ("illegal_content", "Illegal content"),
+    ("offensive_content", "Offensive content"),
+    ("other", "Other"),
+]
+
+
+class Report(federation_models.FederationMixin):
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    creation_date = models.DateTimeField(default=timezone.now)
+    summary = models.TextField(null=True, blank=True, max_length=50000)
+    handled_date = models.DateTimeField(null=True)
+    is_handled = models.BooleanField(default=False)
+    type = models.CharField(max_length=40, choices=REPORT_TYPES)
+    submitter_email = models.EmailField(null=True)
+    submitter = models.ForeignKey(
+        "federation.Actor",
+        related_name="reports",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+    )
+
+    assigned_to = models.ForeignKey(
+        "federation.Actor",
+        related_name="assigned_reports",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+    )
+
+    target_id = models.IntegerField(null=True)
+    target_content_type = models.ForeignKey(
+        ContentType, null=True, on_delete=models.CASCADE
+    )
+    target = GenericForeignKey("target_content_type", "target_id")
+    target_owner = models.ForeignKey(
+        "federation.Actor", on_delete=models.SET_NULL, null=True, blank=True
+    )
+    # frozen state of the target being reported, to ensure we still have info in the event of a
+    # delete
+    target_state = JSONField(null=True)
+
+    notes = GenericRelation(
+        "Note", content_type_field="target_content_type", object_id_field="target_id"
+    )
+
+    objects = common_models.GenericTargetQuerySet.as_manager()
+
+    def get_federation_id(self):
+        if self.fid:
+            return self.fid
+
+        return federation_utils.full_url(
+            reverse("federation:reports-detail", kwargs={"uuid": self.uuid})
+        )
+
+    def save(self, **kwargs):
+        if not self.pk and not self.fid:
+            self.fid = self.get_federation_id()
+
+        return super().save(**kwargs)
+
+
+class Note(models.Model):
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    creation_date = models.DateTimeField(default=timezone.now)
+    summary = models.TextField(max_length=50000)
+    author = models.ForeignKey(
+        "federation.Actor", related_name="moderation_notes", on_delete=models.CASCADE
+    )
+
+    target_id = models.IntegerField(null=True)
+    target_content_type = models.ForeignKey(
+        ContentType, null=True, on_delete=models.CASCADE
+    )
+    target = GenericForeignKey("target_content_type", "target_id")
+
+
+@receiver(pre_save, sender=Report)
+def set_handled_date(sender, instance, **kwargs):
+    if instance.is_handled is True and not instance.handled_date:
+        instance.handled_date = timezone.now()
+    elif not instance.is_handled:
+        instance.handled_date = None

api/funkwhale_api/moderation/mrf.py

@@ -0,0 +1,78 @@
+"""
+Inspired from the MRF logic from Pleroma, see https://docs-develop.pleroma.social/mrf.html
+To support pluggable / customizable moderation using a programming language if
+our exposed features aren't enough.
+"""
+
+import logging
+
+import persisting_theory
+
+logger = logging.getLogger("funkwhale.mrf")
+
+
+class MRFException(Exception):
+    pass
+
+
+class Discard(MRFException):
+    pass
+
+
+class Skip(MRFException):
+    pass
+
+
+class Registry(persisting_theory.Registry):
+    look_into = "mrf_policies"
+
+    def __init__(self, name=""):
+        self.name = name
+
+        super().__init__()
+
+    def apply(self, payload, **kwargs):
+        policy_names = kwargs.pop("policies", [])
+        if not policy_names:
+            policies = self.items()
+        else:
+            logger.debug(
+                "[MRF.%s] Running restricted list of policies %s…",
+                self.name,
+                ", ".join(policy_names),
+            )
+            policies = [(name, self[name]) for name in policy_names]
+        updated = False
+        for policy_name, policy in policies:
+            logger.debug("[MRF.%s] Applying mrf policy '%s'…", self.name, policy_name)
+            try:
+                new_payload = policy(payload, **kwargs)
+            except Skip as e:
+                logger.debug(
+                    "[MRF.%s] Skipped policy %s because '%s'",
+                    self.name,
+                    policy_name,
+                    str(e),
+                )
+                continue
+            except Discard as e:
+                logger.info(
+                    "[MRF.%s] Discarded message per policy '%s' because '%s'",
+                    self.name,
+                    policy_name,
+                    str(e),
+                )
+                return (None, False)
+            except Exception:
+                logger.exception(
+                    "[MRF.%s] Error while applying policy '%s'!", self.name, policy_name
+                )
+                continue
+            if new_payload:
+                updated = True
+                payload = new_payload
+
+        return payload, updated
+
+
+inbox = Registry("inbox")

api/funkwhale_api/moderation/mrf_policies.py

@@ -0,0 +1,47 @@
+import urllib.parse
+
+from funkwhale_api.common import preferences
+from funkwhale_api.common import utils
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.moderation import mrf
+
+
+@mrf.inbox.register(name="allow_list")
+def check_allow_list(payload, **kwargs):
+    """
+    A MRF policy that only works when the moderation__allow_list_enabled
+    setting is on.
+
+    It will extract domain names from the activity ID, actor ID and activity object ID
+    and discard the activity if any of those domain names isn't on the allow list.
+    """
+    if not preferences.get("moderation__allow_list_enabled"):
+        raise mrf.Skip("Allow-listing is disabled")
+
+    allowed_domains = set(
+        federation_models.Domain.objects.filter(allowed=True).values_list(
+            "name", flat=True
+        )
+    )
+
+    relevant_ids = [
+        payload.get("actor"),
+        kwargs.get("sender_id", payload.get("id")),
+        utils.recursive_getattr(payload, "object.id", permissive=True),
+    ]
+
+    relevant_domains = set(
+        [
+            domain
+            for domain in [urllib.parse.urlparse(i).hostname for i in relevant_ids if i]
+            if domain
+        ]
+    )
+
+    if relevant_domains - allowed_domains:
+
+        raise mrf.Discard(
+            "These domains are not allowed: {}".format(
+                ", ".join(relevant_domains - allowed_domains)
+            )
+        )

api/funkwhale_api/moderation/serializers.py

@@ -1,7 +1,20 @@
+import json
+import urllib.parse
+
+from django.conf import settings
+from django.core.serializers.json import DjangoJSONEncoder
+import persisting_theory
 from rest_framework import serializers
 
+from funkwhale_api.common import fields as common_fields
+from funkwhale_api.common import preferences
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import utils as federation_utils
 from funkwhale_api.music import models as music_models
+from funkwhale_api.playlists import models as playlists_models
+
 from . import models
+from . import tasks
 
 
 class FilteredArtistSerializer(serializers.ModelSerializer):
@@ -43,3 +56,208 @@ class UserFilterSerializer(serializers.ModelSerializer):
             data["target_artist"] = target["obj"]
 
         return data
+
+
+state_serializers = persisting_theory.Registry()
+
+
+TAGS_FIELD = serializers.ListField(source="get_tags")
+
+
+@state_serializers.register(name="music.Artist")
+class ArtistStateSerializer(serializers.ModelSerializer):
+    tags = TAGS_FIELD
+
+    class Meta:
+        model = music_models.Artist
+        fields = ["id", "name", "mbid", "fid", "creation_date", "uuid", "tags"]
+
+
+@state_serializers.register(name="music.Album")
+class AlbumStateSerializer(serializers.ModelSerializer):
+    tags = TAGS_FIELD
+    artist = ArtistStateSerializer()
+
+    class Meta:
+        model = music_models.Album
+        fields = [
+            "id",
+            "title",
+            "mbid",
+            "fid",
+            "creation_date",
+            "uuid",
+            "artist",
+            "release_date",
+            "tags",
+        ]
+
+
+@state_serializers.register(name="music.Track")
+class TrackStateSerializer(serializers.ModelSerializer):
+    tags = TAGS_FIELD
+    artist = ArtistStateSerializer()
+    album = AlbumStateSerializer()
+
+    class Meta:
+        model = music_models.Track
+        fields = [
+            "id",
+            "title",
+            "mbid",
+            "fid",
+            "creation_date",
+            "uuid",
+            "artist",
+            "album",
+            "disc_number",
+            "position",
+            "license",
+            "copyright",
+            "tags",
+        ]
+
+
+@state_serializers.register(name="music.Library")
+class LibraryStateSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = music_models.Library
+        fields = [
+            "id",
+            "uuid",
+            "fid",
+            "name",
+            "description",
+            "creation_date",
+            "privacy_level",
+        ]
+
+
+@state_serializers.register(name="playlists.Playlist")
+class PlaylistStateSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = playlists_models.Playlist
+        fields = ["id", "name", "creation_date", "privacy_level"]
+
+
+@state_serializers.register(name="federation.Actor")
+class ActorStateSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = federation_models.Actor
+        fields = [
+            "fid",
+            "name",
+            "preferred_username",
+            "full_username",
+            "summary",
+            "domain",
+            "type",
+            "creation_date",
+        ]
+
+
+def get_actor_query(attr, value):
+    data = federation_utils.get_actor_data_from_username(value)
+    return federation_utils.get_actor_from_username_data_query(None, data)
+
+
+def get_target_owner(target):
+    mapping = {
+        music_models.Artist: lambda t: t.attributed_to,
+        music_models.Album: lambda t: t.attributed_to,
+        music_models.Track: lambda t: t.attributed_to,
+        music_models.Library: lambda t: t.actor,
+        playlists_models.Playlist: lambda t: t.user.actor,
+        federation_models.Actor: lambda t: t,
+    }
+
+    return mapping[target.__class__](target)
+
+
+TARGET_CONFIG = {
+    "artist": {"queryset": music_models.Artist.objects.all()},
+    "album": {"queryset": music_models.Album.objects.all()},
+    "track": {"queryset": music_models.Track.objects.all()},
+    "library": {
+        "queryset": music_models.Library.objects.all(),
+        "id_attr": "uuid",
+        "id_field": serializers.UUIDField(),
+    },
+    "playlist": {"queryset": playlists_models.Playlist.objects.all()},
+    "account": {
+        "queryset": federation_models.Actor.objects.all(),
+        "id_attr": "full_username",
+        "id_field": serializers.EmailField(),
+        "get_query": get_actor_query,
+    },
+}
+TARGET_FIELD = common_fields.GenericRelation(TARGET_CONFIG)
+
+
+class ReportSerializer(serializers.ModelSerializer):
+    target = TARGET_FIELD
+
+    class Meta:
+        model = models.Report
+        fields = [
+            "uuid",
+            "summary",
+            "creation_date",
+            "handled_date",
+            "is_handled",
+            "submitter_email",
+            "target",
+            "type",
+        ]
+        read_only_fields = ["uuid", "is_handled", "creation_date", "handled_date"]
+
+    def validate(self, validated_data):
+        validated_data = super().validate(validated_data)
+        submitter = self.context.get("submitter")
+        if submitter:
+            # we have an authenticated actor so no need to check further
+            return validated_data
+
+        unauthenticated_report_types = preferences.get(
+            "moderation__unauthenticated_report_types"
+        )
+        if validated_data["type"] not in unauthenticated_report_types:
+            raise serializers.ValidationError(
+                "You need an account to submit this report"
+            )
+
+        if not validated_data.get("submitter_email"):
+            raise serializers.ValidationError(
+                "You need to provide an email address to submit this report"
+            )
+
+        return validated_data
+
+    def create(self, validated_data):
+        target_state_serializer = state_serializers[
+            validated_data["target"]._meta.label
+        ]
+
+        validated_data["target_state"] = target_state_serializer(
+            validated_data["target"]
+        ).data
+        # freeze target type/id in JSON so even if the corresponding object is deleted
+        # we can have the info and display it in the frontend
+        target_data = self.fields["target"].to_representation(validated_data["target"])
+        validated_data["target_state"]["_target"] = json.loads(
+            json.dumps(target_data, cls=DjangoJSONEncoder)
+        )
+
+        if "fid" in validated_data["target_state"]:
+            validated_data["target_state"]["domain"] = urllib.parse.urlparse(
+                validated_data["target_state"]["fid"]
+            ).hostname
+
+        validated_data["target_state"]["is_local"] = (
+            validated_data["target_state"].get("domain", settings.FEDERATION_HOSTNAME)
+            == settings.FEDERATION_HOSTNAME
+        )
+        validated_data["target_owner"] = get_target_owner(validated_data["target"])
+        r = super().create(validated_data)
+        tasks.signals.report_created.send(sender=None, report=r)
+        return r

api/funkwhale_api/moderation/signals.py

@@ -0,0 +1,3 @@
+import django.dispatch
+
+report_created = django.dispatch.Signal(providing_args=["report"])

api/funkwhale_api/moderation/tasks.py

@@ -0,0 +1,116 @@
+import logging
+from django.core import mail
+from django.dispatch import receiver
+from django.conf import settings
+
+from funkwhale_api.common import channels
+from funkwhale_api.common import utils
+from funkwhale_api.taskapp import celery
+from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.users import models as users_models
+
+from . import models
+from . import signals
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(signals.report_created)
+def broadcast_report_created(report, **kwargs):
+    from . import serializers
+
+    channels.group_send(
+        "admin.moderation",
+        {
+            "type": "event.send",
+            "text": "",
+            "data": {
+                "type": "report.created",
+                "report": serializers.ReportSerializer(report).data,
+                "unresolved_count": models.Report.objects.filter(
+                    is_handled=False
+                ).count(),
+            },
+        },
+    )
+
+
+@receiver(signals.report_created)
+def trigger_moderator_email(report, **kwargs):
+    if settings.MODERATION_EMAIL_NOTIFICATIONS_ENABLED:
+        utils.on_commit(send_new_report_email_to_moderators.delay, report_id=report.pk)
+
+
+@celery.app.task(name="moderation.send_new_report_email_to_moderators")
+@celery.require_instance(
+    models.Report.objects.select_related("submitter").filter(is_handled=False), "report"
+)
+def send_new_report_email_to_moderators(report):
+    moderators = users_models.User.objects.filter(
+        is_active=True, permission_moderation=True
+    )
+    if not moderators:
+        # we fallback on superusers
+        moderators = users_models.User.objects.filter(is_superuser=True)
+    moderators = sorted(moderators, key=lambda m: m.pk)
+    submitter_repr = (
+        report.submitter.full_username if report.submitter else report.submitter_email
+    )
+    subject = "[{} moderation - {}] New report from {}".format(
+        settings.FUNKWHALE_HOSTNAME, report.get_type_display(), submitter_repr
+    )
+    detail_url = federation_utils.full_url(
+        "/manage/moderation/reports/{}".format(report.uuid)
+    )
+    unresolved_reports_url = federation_utils.full_url(
+        "/manage/moderation/reports?q=resolved:no"
+    )
+    unresolved_reports = models.Report.objects.filter(is_handled=False).count()
+    body = [
+        '{} just submitted a report in the "{}" category.'.format(
+            submitter_repr, report.get_type_display()
+        ),
+        "",
+        "Reported object: {} - {}".format(
+            report.target._meta.verbose_name.title(), str(report.target)
+        ),
+    ]
+    if hasattr(report.target, "get_absolute_url"):
+        body.append(
+            "Open public page: {}".format(
+                federation_utils.full_url(report.target.get_absolute_url())
+            )
+        )
+    if hasattr(report.target, "get_moderation_url"):
+        body.append(
+            "Open moderation page: {}".format(
+                federation_utils.full_url(report.target.get_moderation_url())
+            )
+        )
+    if report.summary:
+        body += ["", "Report content:", "", report.summary]
+
+    body += [
+        "",
+        "- To handle this report, please visit {}".format(detail_url),
+        "- To view all unresolved reports (currently {}), please visit {}".format(
+            unresolved_reports, unresolved_reports_url
+        ),
+        "",
+        "—",
+        "",
+        "You are receiving this email because you are a moderator for {}.".format(
+            settings.FUNKWHALE_HOSTNAME
+        ),
+    ]
+
+    for moderator in moderators:
+        if not moderator.email:
+            logger.warning("Moderator %s has no email configured", moderator.username)
+            continue
+        mail.send_mail(
+            subject,
+            message="\n".join(body),
+            recipient_list=[moderator.email],
+            from_email=settings.DEFAULT_FROM_EMAIL,
+        )

api/funkwhale_api/moderation/urls.py

@@ -1,8 +1,9 @@
-from rest_framework import routers
+from funkwhale_api.common import routers
 
 from . import views
 
-router = routers.SimpleRouter()
+router = routers.OptionalSlashRouter()
 router.register(r"content-filters", views.UserFilterViewSet, "content-filters")
+router.register(r"reports", views.ReportsViewSet, "reports")
 
 urlpatterns = router.urls

api/funkwhale_api/moderation/utils.py

@@ -0,0 +1,21 @@
+from rest_framework import serializers
+
+from funkwhale_api.federation import models as federation_models
+
+from . import models
+from . import serializers as moderation_serializers
+
+
+NOTE_TARGET_FIELDS = {
+    "report": {
+        "queryset": models.Report.objects.all(),
+        "id_attr": "uuid",
+        "id_field": serializers.UUIDField(),
+    },
+    "account": {
+        "queryset": federation_models.Actor.objects.all(),
+        "id_attr": "full_username",
+        "id_field": serializers.EmailField(),
+        "get_query": moderation_serializers.get_actor_query,
+    },
+}

api/funkwhale_api/moderation/views.py

@@ -39,3 +39,31 @@ class UserFilterViewSet(
 
     def perform_create(self, serializer):
         serializer.save(user=self.request.user)
+
+
+class ReportsViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):
+    lookup_field = "uuid"
+    queryset = models.Report.objects.all().order_by("-creation_date")
+    serializer_class = serializers.ReportSerializer
+    required_scope = "reports"
+    ordering_fields = ("creation_date",)
+    anonymous_policy = "setting"
+    anonymous_scopes = {"write:reports"}
+    throttling_scopes = {
+        "create": {
+            "anonymous": "anonymous-reports",
+            "authenticated": "authenticated-reports",
+        }
+    }
+
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        if self.request.user.is_authenticated:
+            context["submitter"] = self.request.user.actor
+        return context
+
+    def perform_create(self, serializer):
+        submitter = None
+        if self.request.user.is_authenticated:
+            submitter = self.request.user.actor
+        serializer.save(submitter=submitter)

api/funkwhale_api/music/factories.py

@@ -2,10 +2,11 @@ import os
 
 import factory
 
-from funkwhale_api.factories import ManyToManyFromList, registry, NoUpdateOnCreate
+from funkwhale_api.factories import registry, NoUpdateOnCreate
 
 from funkwhale_api.federation import factories as federation_factories
 from funkwhale_api.music import licenses
+from funkwhale_api.tags import factories as tags_factories
 from funkwhale_api.users import factories as users_factories
 
 SAMPLES_PATH = os.path.join(
@@ -55,7 +56,9 @@ class LicenseFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
 
 
 @registry.register
-class ArtistFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+class ArtistFactory(
+    tags_factories.TaggableFactory, NoUpdateOnCreate, factory.django.DjangoModelFactory
+):
     name = factory.Faker("name")
     mbid = factory.Faker("uuid4")
     fid = factory.Faker("federation_url")
@@ -72,7 +75,9 @@ class ArtistFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
 
 
 @registry.register
-class AlbumFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+class AlbumFactory(
+    tags_factories.TaggableFactory, NoUpdateOnCreate, factory.django.DjangoModelFactory
+):
     title = factory.Faker("sentence", nb_words=3)
     mbid = factory.Faker("uuid4")
     release_date = factory.Faker("date_object")
@@ -96,14 +101,14 @@ class AlbumFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
 
 
 @registry.register
-class TrackFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+class TrackFactory(
+    tags_factories.TaggableFactory, NoUpdateOnCreate, factory.django.DjangoModelFactory
+):
     fid = factory.Faker("federation_url")
     title = factory.Faker("sentence", nb_words=3)
     mbid = factory.Faker("uuid4")
     album = factory.SubFactory(AlbumFactory)
-    artist = factory.SelfAttribute("album.artist")
     position = 1
-    tags = ManyToManyFromList("tags")
     playable = playable_factory("track")
 
     class Meta:
@@ -118,6 +123,26 @@ class TrackFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
             fid=factory.Faker("federation_url", local=True), album__local=True
         )
 
+    @factory.post_generation
+    def artist(self, created, extracted, **kwargs):
+        """
+        A bit intricated, because we want to be able to specify a different
+        track artist with a fallback on album artist if nothing is specified.
+
+        And handle cases where build or build_batch are used (so no db calls)
+        """
+        if extracted:
+            self.artist = extracted
+        elif kwargs:
+            if created:
+                self.artist = ArtistFactory(**kwargs)
+            else:
+                self.artist = ArtistFactory.build(**kwargs)
+        elif self.album:
+            self.artist = self.album.artist
+        if created:
+            self.save()
+
     @factory.post_generation
     def license(self, created, extracted, **kwargs):
         if not created:
@@ -164,18 +189,6 @@ class UploadVersionFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
         model = "music.UploadVersion"
 
 
-@registry.register
-class TagFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
-    name = factory.SelfAttribute("slug")
-    slug = factory.Faker("slug")
-
-    class Meta:
-        model = "taggit.Tag"
-
-
-# XXX To remove
-
-
 class ImportBatchFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     submitted_by = factory.SubFactory(users_factories.UserFactory)
 

api/funkwhale_api/music/filters.py

@@ -9,9 +9,20 @@ from . import models
 from . import utils
 
 
+def filter_tags(queryset, name, value):
+    non_empty_tags = [v.lower() for v in value if v]
+    for tag in non_empty_tags:
+        queryset = queryset.filter(tagged_items__tag__name=tag).distinct()
+    return queryset
+
+
+TAG_FILTER = common_filters.MultipleQueryFilter(method=filter_tags)
+
+
 class ArtistFilter(moderation_filters.HiddenContentFilterSet):
     q = fields.SearchFilter(search_fields=["name"])
     playable = filters.BooleanFilter(field_name="_", method="filter_playable")
+    tag = TAG_FILTER
 
     class Meta:
         model = models.Artist
@@ -29,6 +40,7 @@ class ArtistFilter(moderation_filters.HiddenContentFilterSet):
 class TrackFilter(moderation_filters.HiddenContentFilterSet):
     q = fields.SearchFilter(search_fields=["title", "album__title", "artist__name"])
     playable = filters.BooleanFilter(field_name="_", method="filter_playable")
+    tag = TAG_FILTER
     id = common_filters.MultipleQueryFilter(coerce=int)
 
     class Meta:
@@ -94,6 +106,7 @@ class UploadFilter(filters.FilterSet):
 class AlbumFilter(moderation_filters.HiddenContentFilterSet):
     playable = filters.BooleanFilter(field_name="_", method="filter_playable")
     q = fields.SearchFilter(search_fields=["title", "artist__name"])
+    tag = TAG_FILTER
 
     class Meta:
         model = models.Album

api/funkwhale_api/music/metadata.py

@@ -1,4 +1,5 @@
 import base64
+from collections.abc import Mapping
 import datetime
 import logging
 import pendulum
@@ -9,12 +10,13 @@ import mutagen.oggvorbis
 import mutagen.flac
 
 from rest_framework import serializers
-from rest_framework.compat import Mapping
+
+from funkwhale_api.tags import models as tags_models
 
 logger = logging.getLogger(__name__)
 NODEFAULT = object()
 # default title used when imported tracks miss the `Album` tag, see #122
-UNKWOWN_ALBUM = "[Unknown Album]"
+UNKNOWN_ALBUM = "[Unknown Album]"
 
 
 class TagNotFound(KeyError):
@@ -68,6 +70,45 @@ def clean_id3_pictures(apic):
     return pictures
 
 
+def get_mp4_tag(f, k):
+    if k == "pictures":
+        return f.get("covr")
+    raw_value = f.get(k, None)
+
+    if not raw_value:
+        raise TagNotFound(k)
+
+    value = raw_value[0]
+    try:
+        return value.decode()
+    except AttributeError:
+        return value
+
+
+def get_mp4_position(raw_value):
+    return raw_value[0]
+
+
+def clean_mp4_pictures(raw_pictures):
+    pictures = []
+    for p in list(raw_pictures):
+        if p.imageformat == p.FORMAT_JPEG:
+            mimetype = "image/jpeg"
+        elif p.imageformat == p.FORMAT_PNG:
+            mimetype = "image/png"
+        else:
+            continue
+        pictures.append(
+            {
+                "mimetype": mimetype,
+                "content": bytes(p),
+                "description": "",
+                "type": mutagen.id3.PictureType.COVER_FRONT,
+            }
+        )
+    return pictures
+
+
 def get_flac_tag(f, k):
     if k == "pictures":
         return f.pictures
@@ -144,6 +185,11 @@ CONF = {
             "mbid": {"field": "musicbrainz_trackid"},
             "license": {},
             "copyright": {},
+            "genre": {},
+            "pictures": {
+                "field": "metadata_block_picture",
+                "to_application": clean_ogg_pictures,
+            },
         },
     },
     "OggVorbis": {
@@ -162,6 +208,7 @@ CONF = {
             "mbid": {"field": "musicbrainz_trackid"},
             "license": {},
             "copyright": {},
+            "genre": {},
             "pictures": {
                 "field": "metadata_block_picture",
                 "to_application": clean_ogg_pictures,
@@ -184,6 +231,7 @@ CONF = {
             "mbid": {"field": "MusicBrainz Track Id"},
             "license": {},
             "copyright": {},
+            "genre": {},
         },
     },
     "MP3": {
@@ -199,6 +247,7 @@ CONF = {
             "date": {"field": "TDRC"},
             "musicbrainz_albumid": {"field": "MusicBrainz Album Id"},
             "musicbrainz_artistid": {"field": "MusicBrainz Artist Id"},
+            "genre": {"field": "TCON"},
             "musicbrainz_albumartistid": {"field": "MusicBrainz Album Artist Id"},
             "mbid": {"field": "UFID", "getter": get_mp3_recording_id},
             "pictures": {},
@@ -206,6 +255,33 @@ CONF = {
             "copyright": {"field": "TCOP"},
         },
     },
+    "MP4": {
+        "getter": get_mp4_tag,
+        "clean_pictures": clean_mp4_pictures,
+        "fields": {
+            "position": {"field": "trkn", "to_application": get_mp4_position},
+            "disc_number": {"field": "disk", "to_application": get_mp4_position},
+            "title": {"field": "©nam"},
+            "artist": {"field": "©ART"},
+            "album_artist": {"field": "aART"},
+            "album": {"field": "©alb"},
+            "date": {"field": "©day"},
+            "musicbrainz_albumid": {
+                "field": "----:com.apple.iTunes:MusicBrainz Album Id"
+            },
+            "musicbrainz_artistid": {
+                "field": "----:com.apple.iTunes:MusicBrainz Artist Id"
+            },
+            "genre": {"field": "©gen"},
+            "musicbrainz_albumartistid": {
+                "field": "----:com.apple.iTunes:MusicBrainz Album Artist Id"
+            },
+            "mbid": {"field": "----:com.apple.iTunes:MusicBrainz Track Id"},
+            "pictures": {},
+            "license": {"field": "----:com.apple.iTunes:LICENSE"},
+            "copyright": {"field": "cprt"},
+        },
+    },
     "FLAC": {
         "getter": get_flac_tag,
         "clean_pictures": clean_flac_pictures,
@@ -220,6 +296,7 @@ CONF = {
             "musicbrainz_albumid": {},
             "musicbrainz_artistid": {},
             "musicbrainz_albumartistid": {},
+            "genre": {},
             "mbid": {"field": "musicbrainz_trackid"},
             "test": {},
             "pictures": {},
@@ -431,7 +508,7 @@ class AlbumField(serializers.Field):
         except TagNotFound:
             title = ""
 
-        title = title.strip() or UNKWOWN_ALBUM
+        title = title.strip() or UNKNOWN_ALBUM
         final = {
             "title": title,
             "release_date": data.get("date", None),
@@ -485,6 +562,58 @@ class PermissiveDateField(serializers.CharField):
         return None
 
 
+def extract_tags_from_genre(string):
+    tags = []
+    delimiter = "@@@@@"
+    for d in [" - ", ",", ";", "/"]:
+        # Replace common tags separators by a custom delimiter
+        string = string.replace(d, delimiter)
+
+    # loop on the parts (splitting on our custom delimiter)
+    for tag in string.split(delimiter):
+        tag = tag.strip()
+        for d in ["-"]:
+            # preparation for replacement so that Pop-Rock becomes Pop Rock, then PopRock
+            # (step 1, step 2 happens below)
+            tag = tag.replace(d, " ")
+        if not tag:
+            continue
+        final_tag = ""
+        if not tags_models.TAG_REGEX.match(tag.replace(" ", "")):
+            # the string contains some non words chars ($, €, etc.), right now
+            # we simply skip such tags
+            continue
+        # concatenate the parts and uppercase them so that 'pop rock' becomes 'PopRock'
+        if len(tag.split(" ")) == 1:
+            # we append the tag "as is", because it doesn't contain any space
+            tags.append(tag)
+            continue
+        for part in tag.split(" "):
+            # the tag contains space, there's work to do to have consistent case
+            # 'pop rock' -> 'PopRock'
+            # (step 2)
+            if not part:
+                continue
+            final_tag += part[0].upper() + part[1:]
+        if final_tag:
+            tags.append(final_tag)
+    return tags
+
+
+class TagsField(serializers.CharField):
+    def get_value(self, data):
+        return data
+
+    def to_internal_value(self, data):
+        try:
+            value = data.get("genre") or ""
+        except TagNotFound:
+            return []
+        value = super().to_internal_value(str(value))
+
+        return extract_tags_from_genre(value)
+
+
 class MBIDField(serializers.UUIDField):
     def __init__(self, *args, **kwargs):
         kwargs.setdefault("allow_null", True)
@@ -533,6 +662,7 @@ class TrackMetadataSerializer(serializers.Serializer):
     copyright = serializers.CharField(allow_blank=True, allow_null=True, required=False)
     license = serializers.CharField(allow_blank=True, allow_null=True, required=False)
     mbid = MBIDField()
+    tags = TagsField(allow_blank=True, allow_null=True, required=False)
 
     album = AlbumField()
     artists = ArtistField()
@@ -544,6 +674,7 @@ class TrackMetadataSerializer(serializers.Serializer):
         "position",
         "disc_number",
         "mbid",
+        "tags",
     ]
 
     def validate(self, validated_data):
@@ -553,7 +684,7 @@ class TrackMetadataSerializer(serializers.Serializer):
                 v = validated_data[field]
             except KeyError:
                 continue
-            if v in ["", None]:
+            if v in ["", None, []]:
                 validated_data.pop(field)
         return validated_data
 

api/funkwhale_api/music/migrations/0004_track_tags.py

@@ -2,25 +2,10 @@
 from __future__ import unicode_literals
 
 from django.db import migrations
-import taggit.managers
 
 
 class Migration(migrations.Migration):
 
-    dependencies = [
-        ("taggit", "0002_auto_20150616_2121"),
-        ("music", "0003_auto_20151222_2233"),
-    ]
+    dependencies = [("music", "0003_auto_20151222_2233")]
 
-    operations = [
-        migrations.AddField(
-            model_name="track",
-            name="tags",
-            field=taggit.managers.TaggableManager(
-                verbose_name="Tags",
-                help_text="A comma-separated list of tags.",
-                through="taggit.TaggedItem",
-                to="taggit.Tag",
-            ),
-        )
-    ]
+    operations = []

api/funkwhale_api/music/migrations/0027_auto_20180515_1808.py

@@ -1,7 +1,6 @@
 # Generated by Django 2.0.3 on 2018-05-15 18:08
 
 from django.db import migrations, models
-import taggit.managers
 
 
 class Migration(migrations.Migration):
@@ -19,15 +18,4 @@ class Migration(migrations.Migration):
             name="size",
             field=models.IntegerField(blank=True, null=True),
         ),
-        migrations.AlterField(
-            model_name="track",
-            name="tags",
-            field=taggit.managers.TaggableManager(
-                blank=True,
-                help_text="A comma-separated list of tags.",
-                through="taggit.TaggedItem",
-                to="taggit.Tag",
-                verbose_name="Tags",
-            ),
-        ),
     ]

api/funkwhale_api/music/models.py

@@ -9,6 +9,7 @@ import uuid
 import pendulum
 import pydub
 from django.conf import settings
+from django.contrib.contenttypes.fields import GenericRelation
 from django.contrib.postgres.fields import JSONField
 from django.core.files.base import ContentFile
 from django.core.serializers.json import DjangoJSONEncoder
@@ -17,7 +18,6 @@ from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.urls import reverse
 from django.utils import timezone
-from taggit.managers import TaggableManager
 
 from versatileimagefield.fields import VersatileImageField
 from versatileimagefield.image_warmer import VersatileImageFieldWarmer
@@ -29,10 +29,18 @@ from funkwhale_api.common import session
 from funkwhale_api.common import utils as common_utils
 from funkwhale_api.federation import models as federation_models
 from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.tags import models as tags_models
 from . import importers, metadata, utils
 
 logger = logging.getLogger(__name__)
 
+MAX_LENGTHS = {
+    "ARTIST_NAME": 255,
+    "ALBUM_TITLE": 255,
+    "TRACK_TITLE": 255,
+    "COPYRIGHT": 500,
+}
+
 
 def empty_dict():
     return {}
@@ -126,6 +134,9 @@ class APIModelMixin(models.Model):
         parsed = urllib.parse.urlparse(self.fid)
         return parsed.hostname
 
+    def get_tags(self):
+        return list(sorted(self.tagged_items.values_list("tag__name", flat=True)))
+
 
 class License(models.Model):
     code = models.CharField(primary_key=True, max_length=100)
@@ -183,7 +194,7 @@ class ArtistQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
 
 
 class Artist(APIModelMixin):
-    name = models.CharField(max_length=255)
+    name = models.CharField(max_length=MAX_LENGTHS["ARTIST_NAME"])
     federation_namespace = "artists"
     musicbrainz_model = "artist"
     musicbrainz_mapping = {
@@ -200,19 +211,24 @@ class Artist(APIModelMixin):
         on_delete=models.SET_NULL,
         related_name="attributed_artists",
     )
+    tagged_items = GenericRelation(tags_models.TaggedItem)
+    fetches = GenericRelation(
+        "federation.Fetch",
+        content_type_field="object_content_type",
+        object_id_field="object_id",
+    )
+
     api = musicbrainz.api.artists
     objects = ArtistQuerySet.as_manager()
 
     def __str__(self):
         return self.name
 
-    @property
-    def tags(self):
-        t = []
-        for album in self.albums.all():
-            for tag in album.tags:
-                t.append(tag)
-        return set(t)
+    def get_absolute_url(self):
+        return "/library/artists/{}".format(self.pk)
+
+    def get_moderation_url(self):
+        return "/manage/library/artists/{}".format(self.pk)
 
     @classmethod
     def get_or_create_from_name(cls, name, **kwargs):
@@ -266,7 +282,7 @@ class AlbumQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
 
 
 class Album(APIModelMixin):
-    title = models.CharField(max_length=255)
+    title = models.CharField(max_length=MAX_LENGTHS["ALBUM_TITLE"])
     artist = models.ForeignKey(Artist, related_name="albums", on_delete=models.CASCADE)
     release_date = models.DateField(null=True, blank=True)
     release_group_id = models.UUIDField(null=True, blank=True)
@@ -286,6 +302,13 @@ class Album(APIModelMixin):
         on_delete=models.SET_NULL,
         related_name="attributed_albums",
     )
+    tagged_items = GenericRelation(tags_models.TaggedItem)
+    fetches = GenericRelation(
+        "federation.Fetch",
+        content_type_field="object_content_type",
+        object_id_field="object_id",
+    )
+
     api_includes = ["artist-credits", "recordings", "media", "release-groups"]
     api = musicbrainz.api.releases
     federation_namespace = "albums"
@@ -314,6 +337,7 @@ class Album(APIModelMixin):
         if data:
             extensions = {"image/jpeg": "jpg", "image/png": "png", "image/gif": "gif"}
             extension = extensions.get(data["mimetype"], "jpg")
+            f = None
             if data.get("content"):
                 # we have to cover itself
                 f = ContentFile(data["content"])
@@ -333,19 +357,27 @@ class Album(APIModelMixin):
                     return
                 else:
                     f = ContentFile(response.content)
-            self.cover.save("{}.{}".format(self.uuid, extension), f, save=False)
-            self.save(update_fields=["cover"])
-            return self.cover.file
+            if f:
+                self.cover.save("{}.{}".format(self.uuid, extension), f, save=False)
+                self.save(update_fields=["cover"])
+                return self.cover.file
         if self.mbid:
             image_data = musicbrainz.api.images.get_front(str(self.mbid))
             f = ContentFile(image_data)
             self.cover.save("{0}.jpg".format(self.mbid), f, save=False)
             self.save(update_fields=["cover"])
-        return self.cover.file
+        if self.cover:
+            return self.cover.file
 
     def __str__(self):
         return self.title
 
+    def get_absolute_url(self):
+        return "/library/albums/{}".format(self.pk)
+
+    def get_moderation_url(self):
+        return "/manage/library/albums/{}".format(self.pk)
+
     @property
     def cover_path(self):
         if not self.cover:
@@ -356,14 +388,6 @@ class Album(APIModelMixin):
             # external storage
             return self.cover.name
 
-    @property
-    def tags(self):
-        t = []
-        for track in self.tracks.all():
-            for tag in track.tags.all():
-                t.append(tag)
-        return set(t)
-
     @classmethod
     def get_or_create_from_title(cls, title, **kwargs):
         kwargs.update({"title": title})
@@ -380,7 +404,8 @@ def import_tags(instance, cleaned_data, raw_data):
         except ValueError:
             continue
         tags_to_add.append(tag_data["name"])
-    instance.tags.add(*tags_to_add)
+
+    tags_models.add_tags(instance, *tags_to_add)
 
 
 def import_album(v):
@@ -430,7 +455,7 @@ def get_artist(release_list):
 
 
 class Track(APIModelMixin):
-    title = models.CharField(max_length=255)
+    title = models.CharField(max_length=MAX_LENGTHS["TRACK_TITLE"])
     artist = models.ForeignKey(Artist, related_name="tracks", on_delete=models.CASCADE)
     disc_number = models.PositiveIntegerField(null=True, blank=True)
     position = models.PositiveIntegerField(null=True, blank=True)
@@ -454,7 +479,9 @@ class Track(APIModelMixin):
         on_delete=models.SET_NULL,
         related_name="attributed_tracks",
     )
-    copyright = models.CharField(max_length=500, null=True, blank=True)
+    copyright = models.CharField(
+        max_length=MAX_LENGTHS["COPYRIGHT"], null=True, blank=True
+    )
     federation_namespace = "tracks"
     musicbrainz_model = "recording"
     api = musicbrainz.api.recordings
@@ -472,7 +499,12 @@ class Track(APIModelMixin):
     }
     import_hooks = [import_tags]
     objects = TrackQuerySet.as_manager()
-    tags = TaggableManager(blank=True)
+    tagged_items = GenericRelation(tags_models.TaggedItem)
+    fetches = GenericRelation(
+        "federation.Fetch",
+        content_type_field="object_content_type",
+        object_id_field="object_id",
+    )
 
     class Meta:
         ordering = ["album", "disc_number", "position"]
@@ -480,6 +512,12 @@ class Track(APIModelMixin):
     def __str__(self):
         return self.title
 
+    def get_absolute_url(self):
+        return "/library/tracks/{}".format(self.pk)
+
+    def get_moderation_url(self):
+        return "/manage/library/tracks/{}".format(self.pk)
+
     def save(self, **kwargs):
         try:
             self.artist
@@ -1043,6 +1081,12 @@ class Library(federation_models.FederationMixin):
     uploads_count = models.PositiveIntegerField(default=0)
     objects = LibraryQuerySet.as_manager()
 
+    def __str__(self):
+        return self.name
+
+    def get_moderation_url(self):
+        return "/manage/library/libraries/{}".format(self.uuid)
+
     def get_federation_id(self):
         return federation_utils.full_url(
             reverse("federation:music:libraries-detail", kwargs={"uuid": self.uuid})

api/funkwhale_api/music/mutations.py

@@ -1,5 +1,7 @@
 from funkwhale_api.common import mutations
 from funkwhale_api.federation import routes
+from funkwhale_api.tags import models as tags_models
+from funkwhale_api.tags import serializers as tags_serializers
 
 from . import models
 
@@ -9,7 +11,27 @@ def can_suggest(obj, actor):
 
 
 def can_approve(obj, actor):
-    return obj.is_local and actor.user and actor.user.get_permissions()["library"]
+    if not obj.is_local or not actor.user:
+        return False
+
+    return (
+        actor.id is not None and actor.id == obj.attributed_to_id
+    ) or actor.user.get_permissions()["library"]
+
+
+class TagMutation(mutations.UpdateMutationSerializer):
+    tags = tags_serializers.TagsListField()
+    previous_state_handlers = {
+        "tags": lambda obj: list(
+            sorted(obj.tagged_items.values_list("tag__name", flat=True))
+        )
+    }
+
+    def update(self, instance, validated_data):
+        tags = validated_data.pop("tags", [])
+        r = super().update(instance, validated_data)
+        tags_models.set_tags(instance, *tags)
+        return r
 
 
 @mutations.registry.connect(
@@ -17,12 +39,12 @@ def can_approve(obj, actor):
     models.Track,
     perm_checkers={"suggest": can_suggest, "approve": can_approve},
 )
-class TrackMutationSerializer(mutations.UpdateMutationSerializer):
+class TrackMutationSerializer(TagMutation):
     serialized_relations = {"license": "code"}
 
     class Meta:
         model = models.Track
-        fields = ["license", "title", "position", "copyright"]
+        fields = ["license", "title", "position", "copyright", "tags"]
 
     def post_apply(self, obj, validated_data):
         routes.outbox.dispatch(
@@ -35,10 +57,10 @@ class TrackMutationSerializer(mutations.UpdateMutationSerializer):
     models.Artist,
     perm_checkers={"suggest": can_suggest, "approve": can_approve},
 )
-class ArtistMutationSerializer(mutations.UpdateMutationSerializer):
+class ArtistMutationSerializer(TagMutation):
     class Meta:
         model = models.Artist
-        fields = ["name"]
+        fields = ["name", "tags"]
 
     def post_apply(self, obj, validated_data):
         routes.outbox.dispatch(
@@ -51,10 +73,10 @@ class ArtistMutationSerializer(mutations.UpdateMutationSerializer):
     models.Album,
     perm_checkers={"suggest": can_suggest, "approve": can_approve},
 )
-class AlbumMutationSerializer(mutations.UpdateMutationSerializer):
+class AlbumMutationSerializer(TagMutation):
     class Meta:
         model = models.Album
-        fields = ["title", "release_date"]
+        fields = ["title", "release_date", "tags"]
 
     def post_apply(self, obj, validated_data):
         routes.outbox.dispatch(

api/funkwhale_api/music/serializers.py

@@ -4,7 +4,6 @@ from django.db import transaction
 from django import urls
 from django.conf import settings
 from rest_framework import serializers
-from taggit.models import Tag
 from versatileimagefield.serializers import VersatileImageFieldSerializer
 
 from funkwhale_api.activity import serializers as activity_serializers
@@ -12,6 +11,8 @@ from funkwhale_api.common import serializers as common_serializers
 from funkwhale_api.common import utils as common_utils
 from funkwhale_api.federation import routes
 from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.playlists import models as playlists_models
+from funkwhale_api.tags.models import Tag
 
 from . import filters, models, tasks
 
@@ -19,6 +20,16 @@ from . import filters, models, tasks
 cover_field = VersatileImageFieldSerializer(allow_null=True, sizes="square")
 
 
+def serialize_attributed_to(self, obj):
+    # Import at runtime to avoid a circular import issue
+    from funkwhale_api.federation import serializers as federation_serializers
+
+    if not obj.attributed_to_id:
+        return
+
+    return federation_serializers.APIActorSerializer(obj.attributed_to).data
+
+
 class LicenseSerializer(serializers.Serializer):
     id = serializers.SerializerMethodField()
     url = serializers.URLField()
@@ -67,63 +78,80 @@ class ArtistAlbumSerializer(serializers.ModelSerializer):
 
 class ArtistWithAlbumsSerializer(serializers.ModelSerializer):
     albums = ArtistAlbumSerializer(many=True, read_only=True)
+    tags = serializers.SerializerMethodField()
+    attributed_to = serializers.SerializerMethodField()
+    tracks_count = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Artist
-        fields = ("id", "fid", "mbid", "name", "creation_date", "albums", "is_local")
-
-
-class ArtistSimpleSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = models.Artist
-        fields = ("id", "fid", "mbid", "name", "creation_date", "is_local")
-
-
-class AlbumTrackSerializer(serializers.ModelSerializer):
-    artist = ArtistSimpleSerializer(read_only=True)
-    uploads = serializers.SerializerMethodField()
-    listen_url = serializers.SerializerMethodField()
-    duration = serializers.SerializerMethodField()
-
-    class Meta:
-        model = models.Track
         fields = (
             "id",
             "fid",
             "mbid",
-            "title",
-            "album",
-            "artist",
+            "name",
             "creation_date",
-            "position",
-            "disc_number",
-            "uploads",
-            "listen_url",
-            "duration",
-            "copyright",
-            "license",
+            "albums",
             "is_local",
+            "tags",
+            "attributed_to",
+            "tracks_count",
         )
 
-    def get_uploads(self, obj):
-        uploads = getattr(obj, "playable_uploads", [])
-        return TrackUploadSerializer(uploads, many=True).data
+    def get_tags(self, obj):
+        tagged_items = getattr(obj, "_prefetched_tagged_items", [])
+        return [ti.tag.name for ti in tagged_items]
 
-    def get_listen_url(self, obj):
-        return obj.listen_url
+    get_attributed_to = serialize_attributed_to
 
-    def get_duration(self, obj):
-        try:
-            return obj.duration
-        except AttributeError:
-            return None
+    def get_tracks_count(self, o):
+        tracks = getattr(o, "_prefetched_tracks", None)
+        return len(tracks) if tracks else None
+
+
+def serialize_artist_simple(artist):
+    return {
+        "id": artist.id,
+        "fid": artist.fid,
+        "mbid": str(artist.mbid),
+        "name": artist.name,
+        "creation_date": serializers.DateTimeField().to_representation(
+            artist.creation_date
+        ),
+        "is_local": artist.is_local,
+    }
+
+
+def serialize_album_track(track):
+    return {
+        "id": track.id,
+        "fid": track.fid,
+        "mbid": str(track.mbid),
+        "title": track.title,
+        "artist": serialize_artist_simple(track.artist),
+        "album": track.album_id,
+        "creation_date": serializers.DateTimeField().to_representation(
+            track.creation_date
+        ),
+        "position": track.position,
+        "disc_number": track.disc_number,
+        "uploads": [
+            serialize_upload(u) for u in getattr(track, "playable_uploads", [])
+        ],
+        "listen_url": track.listen_url,
+        "duration": getattr(track, "duration", None),
+        "copyright": track.copyright,
+        "license": track.license_id,
+        "is_local": track.is_local,
+    }
 
 
 class AlbumSerializer(serializers.ModelSerializer):
     tracks = serializers.SerializerMethodField()
-    artist = ArtistSimpleSerializer(read_only=True)
+    artist = serializers.SerializerMethodField()
     cover = cover_field
     is_playable = serializers.SerializerMethodField()
+    tags = serializers.SerializerMethodField()
+    attributed_to = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Album
@@ -139,11 +167,18 @@ class AlbumSerializer(serializers.ModelSerializer):
             "creation_date",
             "is_playable",
             "is_local",
+            "tags",
+            "attributed_to",
         )
 
+    get_attributed_to = serialize_attributed_to
+
+    def get_artist(self, o):
+        return serialize_artist_simple(o.artist)
+
     def get_tracks(self, o):
         ordered_tracks = o.tracks.all()
-        return AlbumTrackSerializer(ordered_tracks, many=True).data
+        return [serialize_album_track(track) for track in ordered_tracks]
 
     def get_is_playable(self, obj):
         try:
@@ -153,9 +188,13 @@ class AlbumSerializer(serializers.ModelSerializer):
         except AttributeError:
             return None
 
+    def get_tags(self, obj):
+        tagged_items = getattr(obj, "_prefetched_tagged_items", [])
+        return [ti.tag.name for ti in tagged_items]
+
 
 class TrackAlbumSerializer(serializers.ModelSerializer):
-    artist = ArtistSimpleSerializer(read_only=True)
+    artist = serializers.SerializerMethodField()
     cover = cover_field
 
     class Meta:
@@ -172,26 +211,29 @@ class TrackAlbumSerializer(serializers.ModelSerializer):
             "is_local",
         )
 
+    def get_artist(self, o):
+        return serialize_artist_simple(o.artist)
 
-class TrackUploadSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = models.Upload
-        fields = (
-            "uuid",
-            "listen_url",
-            "size",
-            "duration",
-            "bitrate",
-            "mimetype",
-            "extension",
-        )
+
+def serialize_upload(upload):
+    return {
+        "uuid": str(upload.uuid),
+        "listen_url": upload.listen_url,
+        "size": upload.size,
+        "duration": upload.duration,
+        "bitrate": upload.bitrate,
+        "mimetype": upload.mimetype,
+        "extension": upload.extension,
+    }
 
 
 class TrackSerializer(serializers.ModelSerializer):
-    artist = ArtistSimpleSerializer(read_only=True)
+    artist = serializers.SerializerMethodField()
     album = TrackAlbumSerializer(read_only=True)
     uploads = serializers.SerializerMethodField()
     listen_url = serializers.SerializerMethodField()
+    tags = serializers.SerializerMethodField()
+    attributed_to = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Track
@@ -210,14 +252,24 @@ class TrackSerializer(serializers.ModelSerializer):
             "copyright",
             "license",
             "is_local",
+            "tags",
+            "attributed_to",
         )
 
+    get_attributed_to = serialize_attributed_to
+
+    def get_artist(self, o):
+        return serialize_artist_simple(o.artist)
+
     def get_listen_url(self, obj):
         return obj.listen_url
 
     def get_uploads(self, obj):
-        uploads = getattr(obj, "playable_uploads", [])
-        return TrackUploadSerializer(uploads, many=True).data
+        return [serialize_upload(u) for u in getattr(obj, "playable_uploads", [])]
+
+    def get_tags(self, obj):
+        tagged_items = getattr(obj, "_prefetched_tagged_items", [])
+        return [ti.tag.name for ti in tagged_items]
 
 
 @common_serializers.track_fields_for_update("name", "description", "privacy_level")
@@ -361,7 +413,7 @@ class UploadActionSerializer(common_serializers.ActionSerializer):
 class TagSerializer(serializers.ModelSerializer):
     class Meta:
         model = Tag
-        fields = ("id", "name", "slug")
+        fields = ("id", "name", "creation_date")
 
 
 class SimpleAlbumSerializer(serializers.ModelSerializer):
@@ -499,6 +551,38 @@ class OembedSerializer(serializers.Serializer):
             data["author_url"] = federation_utils.full_url(
                 common_utils.spa_reverse("library_artist", kwargs={"pk": artist.pk})
             )
+        elif match.url_name == "library_playlist":
+            qs = playlists_models.Playlist.objects.filter(
+                pk=int(match.kwargs["pk"]), privacy_level="everyone"
+            )
+            try:
+                obj = qs.get()
+            except playlists_models.Playlist.DoesNotExist:
+                raise serializers.ValidationError(
+                    "No artist matching id {}".format(match.kwargs["pk"])
+                )
+            embed_type = "playlist"
+            embed_id = obj.pk
+            playlist_tracks = obj.playlist_tracks.exclude(track__album__cover="")
+            playlist_tracks = playlist_tracks.exclude(track__album__cover=None)
+            playlist_tracks = playlist_tracks.select_related("track__album").order_by(
+                "index"
+            )
+            first_playlist_track = playlist_tracks.first()
+
+            if first_playlist_track:
+                data["thumbnail_url"] = federation_utils.full_url(
+                    first_playlist_track.track.album.cover.crop["400x400"].url
+                )
+                data["thumbnail_width"] = 400
+                data["thumbnail_height"] = 400
+            data["title"] = obj.name
+            data["description"] = obj.name
+            data["author_name"] = obj.name
+            data["height"] = 400
+            data["author_url"] = federation_utils.full_url(
+                common_utils.spa_reverse("library_playlist", kwargs={"pk": obj.pk})
+            )
         else:
             raise serializers.ValidationError(
                 "Unsupported url: {}".format(validated_data["url"])

api/funkwhale_api/music/spa_views.py

@@ -5,6 +5,7 @@ from django.urls import reverse
 from django.db.models import Q
 
 from funkwhale_api.common import utils
+from funkwhale_api.playlists import models as playlists_models
 
 from . import models
 from . import serializers
@@ -203,3 +204,59 @@ def library_artist(request, pk):
         # twitter player is also supported in various software
         metas += get_twitter_card_metas(type="artist", id=obj.pk)
     return metas
+
+
+def library_playlist(request, pk):
+    queryset = playlists_models.Playlist.objects.filter(pk=pk, privacy_level="everyone")
+    try:
+        obj = queryset.get()
+    except playlists_models.Playlist.DoesNotExist:
+        return []
+    obj_url = utils.join_url(
+        settings.FUNKWHALE_URL,
+        utils.spa_reverse("library_playlist", kwargs={"pk": obj.pk}),
+    )
+    # we use the first playlist track's album's cover as image
+    playlist_tracks = obj.playlist_tracks.exclude(track__album__cover="")
+    playlist_tracks = playlist_tracks.exclude(track__album__cover=None)
+    playlist_tracks = playlist_tracks.select_related("track__album").order_by("index")
+    first_playlist_track = playlist_tracks.first()
+    metas = [
+        {"tag": "meta", "property": "og:url", "content": obj_url},
+        {"tag": "meta", "property": "og:title", "content": obj.name},
+        {"tag": "meta", "property": "og:type", "content": "music.playlist"},
+    ]
+
+    if first_playlist_track:
+        metas.append(
+            {
+                "tag": "meta",
+                "property": "og:image",
+                "content": utils.join_url(
+                    settings.FUNKWHALE_URL,
+                    first_playlist_track.track.album.cover.crop["400x400"].url,
+                ),
+            }
+        )
+
+    if (
+        models.Upload.objects.filter(
+            track__pk__in=[obj.playlist_tracks.values("track")]
+        )
+        .playable_by(None)
+        .exists()
+    ):
+        metas.append(
+            {
+                "tag": "link",
+                "rel": "alternate",
+                "type": "application/json+oembed",
+                "href": (
+                    utils.join_url(settings.FUNKWHALE_URL, reverse("api:v1:oembed"))
+                    + "?format=json&url={}".format(urllib.parse.quote_plus(obj_url))
+                ),
+            }
+        )
+        # twitter player is also supported in various software
+        metas += get_twitter_card_metas(type="playlist", id=obj.pk)
+    return metas

api/funkwhale_api/music/tasks.py

@@ -14,6 +14,7 @@ from requests.exceptions import RequestException
 from funkwhale_api.common import channels, preferences
 from funkwhale_api.federation import routes
 from funkwhale_api.federation import library as lb
+from funkwhale_api.tags import models as tags_models
 from funkwhale_api.taskapp import celery
 
 from . import licenses
@@ -54,19 +55,21 @@ def update_album_cover(
             )
 
 
-IMAGE_TYPES = [("jpg", "image/jpeg"), ("png", "image/png")]
+IMAGE_TYPES = [("jpg", "image/jpeg"), ("jpeg", "image/jpeg"), ("png", "image/png")]
+FOLDER_IMAGE_NAMES = ["cover", "folder"]
 
 
 def get_cover_from_fs(dir_path):
     if os.path.exists(dir_path):
-        for e, m in IMAGE_TYPES:
-            cover_path = os.path.join(dir_path, "cover.{}".format(e))
-            if not os.path.exists(cover_path):
-                logger.debug("Cover %s does not exists", cover_path)
-                continue
-            with open(cover_path, "rb") as c:
-                logger.info("Found cover at %s", cover_path)
-                return {"mimetype": m, "content": c.read()}
+        for name in FOLDER_IMAGE_NAMES:
+            for e, m in IMAGE_TYPES:
+                cover_path = os.path.join(dir_path, "{}.{}".format(name, e))
+                if not os.path.exists(cover_path):
+                    logger.debug("Cover %s does not exists", cover_path)
+                    continue
+                with open(cover_path, "rb") as c:
+                    logger.info("Found cover at %s", cover_path)
+                    return {"mimetype": m, "content": c.read()}
 
 
 @celery.app.task(name="music.start_library_scan")
@@ -297,6 +300,7 @@ def federation_audio_track_to_metadata(payload, references):
             if payload["album"].get("musicbrainzId")
             else None,
             "release_date": payload["album"].get("released"),
+            "tags": [t["name"] for t in payload["album"].get("tags", []) or []],
             "artists": [
                 {
                     "fid": a["id"],
@@ -304,6 +308,7 @@ def federation_audio_track_to_metadata(payload, references):
                     "fdate": a["published"],
                     "attributed_to": references.get(a.get("attributedTo")),
                     "mbid": str(a["musicbrainzId"]) if a.get("musicbrainzId") else None,
+                    "tags": [t["name"] for t in a.get("tags", []) or []],
                 }
                 for a in payload["album"]["artists"]
             ],
@@ -315,12 +320,14 @@ def federation_audio_track_to_metadata(payload, references):
                 "fdate": a["published"],
                 "attributed_to": references.get(a.get("attributedTo")),
                 "mbid": str(a["musicbrainzId"]) if a.get("musicbrainzId") else None,
+                "tags": [t["name"] for t in a.get("tags", []) or []],
             }
             for a in payload["artists"]
         ],
         # federation
         "fid": payload["id"],
         "fdate": payload["published"],
+        "tags": [t["name"] for t in payload.get("tags", []) or []],
     }
     cover = payload["album"].get("cover")
     if cover:
@@ -399,6 +406,12 @@ def get_track_from_import_metadata(data, update_cover=False, attributed_to=None)
     return track
 
 
+def truncate(v, length):
+    if v is None:
+        return v
+    return v[:length]
+
+
 def _get_track(data, attributed_to=None):
     track_uuid = getter(data, "funkwhale", "track", "uuid")
 
@@ -437,10 +450,10 @@ def _get_track(data, attributed_to=None):
 
     # get / create artist and album artist
     artists = getter(data, "artists", default=[])
-    artist = artists[0]
-    artist_mbid = artist.get("mbid", None)
-    artist_fid = artist.get("fid", None)
-    artist_name = artist["name"]
+    artist_data = artists[0]
+    artist_mbid = artist_data.get("mbid", None)
+    artist_fid = artist_data.get("fid", None)
+    artist_name = truncate(artist_data["name"], models.MAX_LENGTHS["ARTIST_NAME"])
 
     if artist_mbid:
         query = Q(mbid=artist_mbid)
@@ -453,24 +466,28 @@ def _get_track(data, attributed_to=None):
         "mbid": artist_mbid,
         "fid": artist_fid,
         "from_activity_id": from_activity_id,
-        "attributed_to": artist.get("attributed_to", attributed_to),
+        "attributed_to": artist_data.get("attributed_to", attributed_to),
     }
-    if artist.get("fdate"):
-        defaults["creation_date"] = artist.get("fdate")
+    if artist_data.get("fdate"):
+        defaults["creation_date"] = artist_data.get("fdate")
 
-    artist = get_best_candidate_or_create(
+    artist, created = get_best_candidate_or_create(
         models.Artist, query, defaults=defaults, sort_fields=["mbid", "fid"]
-    )[0]
+    )
+    if created:
+        tags_models.add_tags(artist, *artist_data.get("tags", []))
 
     album_artists = getter(data, "album", "artists", default=artists) or artists
-    album_artist = album_artists[0]
-    album_artist_name = album_artist.get("name")
+    album_artist_data = album_artists[0]
+    album_artist_name = truncate(
+        album_artist_data.get("name"), models.MAX_LENGTHS["ARTIST_NAME"]
+    )
     if album_artist_name == artist_name:
         album_artist = artist
     else:
         query = Q(name__iexact=album_artist_name)
-        album_artist_mbid = album_artist.get("mbid", None)
-        album_artist_fid = album_artist.get("fid", None)
+        album_artist_mbid = album_artist_data.get("mbid", None)
+        album_artist_fid = album_artist_data.get("fid", None)
         if album_artist_mbid:
             query |= Q(mbid=album_artist_mbid)
         if album_artist_fid:
@@ -480,19 +497,21 @@ def _get_track(data, attributed_to=None):
             "mbid": album_artist_mbid,
             "fid": album_artist_fid,
             "from_activity_id": from_activity_id,
-            "attributed_to": album_artist.get("attributed_to", attributed_to),
+            "attributed_to": album_artist_data.get("attributed_to", attributed_to),
         }
-        if album_artist.get("fdate"):
-            defaults["creation_date"] = album_artist.get("fdate")
+        if album_artist_data.get("fdate"):
+            defaults["creation_date"] = album_artist_data.get("fdate")
 
-        album_artist = get_best_candidate_or_create(
+        album_artist, created = get_best_candidate_or_create(
             models.Artist, query, defaults=defaults, sort_fields=["mbid", "fid"]
-        )[0]
+        )
+        if created:
+            tags_models.add_tags(album_artist, *album_artist_data.get("tags", []))
 
     # get / create album
-    album = data["album"]
-    album_title = album["title"]
-    album_fid = album.get("fid", None)
+    album_data = data["album"]
+    album_title = truncate(album_data["title"], models.MAX_LENGTHS["ALBUM_TITLE"])
+    album_fid = album_data.get("fid", None)
 
     if album_mbid:
         query = Q(mbid=album_mbid)
@@ -505,20 +524,22 @@ def _get_track(data, attributed_to=None):
         "title": album_title,
         "artist": album_artist,
         "mbid": album_mbid,
-        "release_date": album.get("release_date"),
+        "release_date": album_data.get("release_date"),
         "fid": album_fid,
         "from_activity_id": from_activity_id,
-        "attributed_to": album.get("attributed_to", attributed_to),
+        "attributed_to": album_data.get("attributed_to", attributed_to),
     }
-    if album.get("fdate"):
-        defaults["creation_date"] = album.get("fdate")
+    if album_data.get("fdate"):
+        defaults["creation_date"] = album_data.get("fdate")
 
-    album = get_best_candidate_or_create(
+    album, created = get_best_candidate_or_create(
         models.Album, query, defaults=defaults, sort_fields=["mbid", "fid"]
-    )[0]
+    )
+    if created:
+        tags_models.add_tags(album, *album_data.get("tags", []))
 
     # get / create track
-    track_title = data["title"]
+    track_title = truncate(data["title"], models.MAX_LENGTHS["TRACK_TITLE"])
     position = data.get("position", 1)
     query = Q(title__iexact=track_title, artist=artist, album=album, position=position)
     if track_mbid:
@@ -536,15 +557,17 @@ def _get_track(data, attributed_to=None):
         "from_activity_id": from_activity_id,
         "attributed_to": data.get("attributed_to", attributed_to),
         "license": licenses.match(data.get("license"), data.get("copyright")),
-        "copyright": data.get("copyright"),
+        "copyright": truncate(data.get("copyright"), models.MAX_LENGTHS["COPYRIGHT"]),
     }
     if data.get("fdate"):
         defaults["creation_date"] = data.get("fdate")
 
-    track = get_best_candidate_or_create(
+    track, created = get_best_candidate_or_create(
         models.Track, query, defaults=defaults, sort_fields=["mbid", "fid"]
-    )[0]
+    )
 
+    if created:
+        tags_models.add_tags(track, *data.get("tags", []))
     return track
 
 

api/funkwhale_api/music/utils.py

@@ -34,6 +34,8 @@ AUDIO_EXTENSIONS_AND_MIMETYPE = [
     ("ogg", "audio/ogg"),
     ("opus", "audio/opus"),
     ("mp3", "audio/mpeg"),
+    ("aac", "audio/x-m4a"),
+    ("m4a", "audio/x-m4a"),
     ("flac", "audio/x-flac"),
     ("flac", "audio/flac"),
 ]

api/funkwhale_api/music/views.py

@@ -1,10 +1,10 @@
+import datetime
 import logging
-import urllib
+import urllib.parse
 
 from django.conf import settings
 from django.db import transaction
 from django.db.models import Count, Prefetch, Sum, F, Q
-from django.db.models.functions import Length
 from django.utils import timezone
 
 from rest_framework import mixins
@@ -12,7 +12,6 @@ from rest_framework import settings as rest_settings
 from rest_framework import views, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
-from taggit.models import Tag
 
 from funkwhale_api.common import decorators as common_decorators
 from funkwhale_api.common import permissions as common_permissions
@@ -23,13 +22,23 @@ from funkwhale_api.federation.authentication import SignatureAuthentication
 from funkwhale_api.federation import actors
 from funkwhale_api.federation import api_serializers as federation_api_serializers
 from funkwhale_api.federation import decorators as federation_decorators
+from funkwhale_api.federation import models as federation_models
 from funkwhale_api.federation import routes
+from funkwhale_api.federation import tasks as federation_tasks
+from funkwhale_api.tags.models import Tag, TaggedItem
+from funkwhale_api.tags.serializers import TagSerializer
 from funkwhale_api.users.oauth import permissions as oauth_permissions
 
 from . import filters, licenses, models, serializers, tasks, utils
 
 logger = logging.getLogger(__name__)
 
+TAG_PREFETCH = Prefetch(
+    "tagged_items",
+    queryset=TaggedItem.objects.all().select_related().order_by("tag__name"),
+    to_attr="_prefetched_tagged_items",
+)
+
 
 def get_libraries(filter_uploads):
     def libraries(self, request, *args, **kwargs):
@@ -53,17 +62,49 @@ def get_libraries(filter_uploads):
     return libraries
 
 
-class TagViewSetMixin(object):
-    def get_queryset(self):
-        queryset = super().get_queryset()
-        tag = self.request.query_params.get("tag")
-        if tag:
-            queryset = queryset.filter(tags__pk=tag)
-        return queryset
+def refetch_obj(obj, queryset):
+    """
+    Given an Artist/Album/Track instance, if the instance is from a remote pod,
+    will attempt to update local data with the latest ActivityPub representation.
+    """
+    if obj.is_local:
+        return obj
+
+    now = timezone.now()
+    limit = now - datetime.timedelta(minutes=settings.FEDERATION_OBJECT_FETCH_DELAY)
+    last_fetch = obj.fetches.order_by("-creation_date").first()
+    if last_fetch is not None and last_fetch.creation_date > limit:
+        # we fetched recently, no need to do it again
+        return obj
+
+    logger.info("Refetching %s:%s at %s…", obj._meta.label, obj.pk, obj.fid)
+    actor = actors.get_service_actor()
+    fetch = federation_models.Fetch.objects.create(actor=actor, url=obj.fid, object=obj)
+    try:
+        federation_tasks.fetch(fetch_id=fetch.pk)
+    except Exception:
+        logger.exception(
+            "Error while refetching %s:%s at %s…", obj._meta.label, obj.pk, obj.fid
+        )
+    else:
+        fetch.refresh_from_db()
+        if fetch.status == "finished":
+            obj = queryset.get(pk=obj.pk)
+    return obj
 
 
 class ArtistViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelViewSet):
-    queryset = models.Artist.objects.all()
+    queryset = (
+        models.Artist.objects.all()
+        .prefetch_related("attributed_to")
+        .prefetch_related(
+            Prefetch(
+                "tracks",
+                queryset=models.Track.objects.all(),
+                to_attr="_prefetched_tracks",
+            )
+        )
+    )
     serializer_class = serializers.ArtistWithAlbumsSerializer
     permission_classes = [oauth_permissions.ScopePermission]
     required_scope = "libraries"
@@ -74,13 +115,25 @@ class ArtistViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelV
     fetches = federation_decorators.fetches_route()
     mutations = common_decorators.mutations_route(types=["update"])
 
+    def get_object(self):
+        obj = super().get_object()
+
+        if (
+            self.action == "retrieve"
+            and self.request.GET.get("refresh", "").lower() == "true"
+        ):
+            obj = refetch_obj(obj, self.get_queryset())
+        return obj
+
     def get_queryset(self):
         queryset = super().get_queryset()
         albums = models.Album.objects.with_tracks_count()
         albums = albums.annotate_playable_by_actor(
             utils.get_actor_from_request(self.request)
         )
-        return queryset.prefetch_related(Prefetch("albums", queryset=albums))
+        return queryset.prefetch_related(
+            Prefetch("albums", queryset=albums), TAG_PREFETCH
+        )
 
     libraries = action(methods=["get"], detail=True)(
         get_libraries(
@@ -93,7 +146,9 @@ class ArtistViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelV
 
 class AlbumViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelViewSet):
     queryset = (
-        models.Album.objects.all().order_by("artist", "release_date").select_related()
+        models.Album.objects.all()
+        .order_by("artist", "release_date")
+        .select_related("artist", "attributed_to")
     )
     serializer_class = serializers.AlbumSerializer
     permission_classes = [oauth_permissions.ScopePermission]
@@ -105,6 +160,16 @@ class AlbumViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelVi
     fetches = federation_decorators.fetches_route()
     mutations = common_decorators.mutations_route(types=["update"])
 
+    def get_object(self):
+        obj = super().get_object()
+
+        if (
+            self.action == "retrieve"
+            and self.request.GET.get("refresh", "").lower() == "true"
+        ):
+            obj = refetch_obj(obj, self.get_queryset())
+        return obj
+
     def get_queryset(self):
         queryset = super().get_queryset()
         tracks = (
@@ -112,7 +177,9 @@ class AlbumViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelVi
             .with_playable_uploads(utils.get_actor_from_request(self.request))
             .order_for_album()
         )
-        qs = queryset.prefetch_related(Prefetch("tracks", queryset=tracks))
+        qs = queryset.prefetch_related(
+            Prefetch("tracks", queryset=tracks), TAG_PREFETCH
+        )
         return qs
 
     libraries = action(methods=["get"], detail=True)(
@@ -182,14 +249,16 @@ class LibraryViewSet(
         return Response(serializer.data)
 
 
-class TrackViewSet(
-    common_views.SkipFilterForGetObject, TagViewSetMixin, viewsets.ReadOnlyModelViewSet
-):
+class TrackViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelViewSet):
     """
     A simple ViewSet for viewing and editing accounts.
     """
 
-    queryset = models.Track.objects.all().for_nested_serialization()
+    queryset = (
+        models.Track.objects.all()
+        .for_nested_serialization()
+        .select_related("attributed_to")
+    )
     serializer_class = serializers.TrackSerializer
     permission_classes = [oauth_permissions.ScopePermission]
     required_scope = "libraries"
@@ -207,6 +276,16 @@ class TrackViewSet(
     fetches = federation_decorators.fetches_route()
     mutations = common_decorators.mutations_route(types=["update"])
 
+    def get_object(self):
+        obj = super().get_object()
+
+        if (
+            self.action == "retrieve"
+            and self.request.GET.get("refresh", "").lower() == "true"
+        ):
+            obj = refetch_obj(obj, self.get_queryset())
+        return obj
+
     def get_queryset(self):
         queryset = super().get_queryset()
         filter_favorites = self.request.GET.get("favorites", None)
@@ -217,7 +296,7 @@ class TrackViewSet(
         queryset = queryset.with_playable_uploads(
             utils.get_actor_from_request(self.request)
         )
-        return queryset
+        return queryset.prefetch_related(TAG_PREFETCH)
 
     libraries = action(methods=["get"], detail=True)(
         get_libraries(filter_uploads=lambda o, uploads: uploads.filter(track=o))
@@ -242,6 +321,8 @@ def get_file_path(audio_file):
             path = "/music" + audio_file.replace(prefix, "", 1)
         if path.startswith("http://") or path.startswith("https://"):
             return (settings.PROTECT_FILES_PATH + "/media/" + path).encode("utf-8")
+        # needed to serve files with % or ? chars
+        path = urllib.parse.quote(path)
         return (settings.PROTECT_FILES_PATH + path).encode("utf-8")
     if t == "apache2":
         try:
@@ -334,7 +415,7 @@ def handle_serve(upload, user, format=None, max_bitrate=None, proxy_media=True):
         f = transcoded_version
         file_path = get_file_path(f.audio_file)
         mt = f.mimetype
-    if not proxy_media:
+    if not proxy_media and f.audio_file:
         # we simply issue a 302 redirect to the real URL
         response = Response(status=302)
         response["Location"] = f.audio_file.url
@@ -458,14 +539,6 @@ class UploadViewSet(
         instance.delete()
 
 
-class TagViewSet(viewsets.ReadOnlyModelViewSet):
-    queryset = Tag.objects.all().order_by("name")
-    serializer_class = serializers.TagSerializer
-    permission_classes = [oauth_permissions.ScopePermission]
-    required_scope = "libraries"
-    anonymous_policy = "setting"
-
-
 class Search(views.APIView):
     max_results = 3
     permission_classes = [oauth_permissions.ScopePermission]
@@ -485,6 +558,7 @@ class Search(views.APIView):
             "albums": serializers.AlbumSerializer(
                 self.get_albums(query), many=True
             ).data,
+            "tags": TagSerializer(self.get_tags(query), many=True).data,
         }
         return Response(results, status=200)
 
@@ -521,17 +595,10 @@ class Search(views.APIView):
         return common_utils.order_for_search(qs, "name")[: self.max_results]
 
     def get_tags(self, query):
-        search_fields = ["slug", "name__unaccent"]
+        search_fields = ["name__unaccent"]
         query_obj = utils.get_query(query, search_fields)
-
-        # We want the shortest tag first
-        qs = (
-            Tag.objects.all()
-            .annotate(slug_length=Length("slug"))
-            .order_by("slug_length")
-        )
-
-        return qs.filter(query_obj)[: self.max_results]
+        qs = Tag.objects.all().filter(query_obj)
+        return common_utils.order_for_search(qs, "name")[: self.max_results]
 
 
 class LicenseViewSet(viewsets.ReadOnlyModelViewSet):

api/funkwhale_api/musicbrainz/urls.py

@@ -1,9 +1,9 @@
 from django.conf.urls import url
-from rest_framework import routers
+from funkwhale_api.common import routers
 
 from . import views
 
-router = routers.SimpleRouter()
+router = routers.OptionalSlashRouter()
 router.register(r"search", views.SearchViewSet, "search")
 urlpatterns = [
     url(

api/funkwhale_api/playlists/models.py

@@ -69,6 +69,9 @@ class Playlist(models.Model):
     def __str__(self):
         return self.name
 
+    def get_absolute_url(self):
+        return "/library/playlists/{}".format(self.pk)
+
     @transaction.atomic
     def insert(self, plt, index=None, allow_duplicates=True):
         """

api/funkwhale_api/playlists/serializers.py

@@ -2,6 +2,7 @@ from django.db import transaction
 from rest_framework import serializers
 
 from funkwhale_api.common import preferences
+from funkwhale_api.federation import serializers as federation_serializers
 from funkwhale_api.music.models import Track
 from funkwhale_api.music.serializers import TrackSerializer
 from funkwhale_api.users.serializers import UserBasicSerializer
@@ -79,6 +80,7 @@ class PlaylistSerializer(serializers.ModelSerializer):
     album_covers = serializers.SerializerMethodField(read_only=True)
     user = UserBasicSerializer(read_only=True)
     is_playable = serializers.SerializerMethodField()
+    actor = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Playlist
@@ -93,9 +95,15 @@ class PlaylistSerializer(serializers.ModelSerializer):
             "album_covers",
             "duration",
             "is_playable",
+            "actor",
         )
         read_only_fields = ["id", "modification_date", "creation_date"]
 
+    def get_actor(self, obj):
+        actor = obj.user.actor
+        if actor:
+            return federation_serializers.APIActorSerializer(actor).data
+
     def get_is_playable(self, obj):
         try:
             return bool(obj.playable_plts)

api/funkwhale_api/playlists/views.py

@@ -23,7 +23,7 @@ class PlaylistViewSet(
     serializer_class = serializers.PlaylistSerializer
     queryset = (
         models.Playlist.objects.all()
-        .select_related("user")
+        .select_related("user__actor")
         .annotate(tracks_count=Count("playlist_tracks"))
         .with_covers()
         .with_duration()

api/funkwhale_api/radios/filters.py

@@ -178,9 +178,9 @@ class TagFilter(RadioFilter):
             "autocomplete_fields": {
                 "remoteValues": "results",
                 "name": "name",
-                "value": "slug",
+                "value": "name",
             },
-            "autocomplete_qs": "query={query}",
+            "autocomplete_qs": "q={query}&ordering=length",
             "label": "Tags",
             "placeholder": "Select tags",
         }
@@ -189,4 +189,8 @@ class TagFilter(RadioFilter):
     label = "Tag"
 
     def get_query(self, candidates, names, **kwargs):
-        return Q(tags__slug__in=names)
+        return (
+            Q(tagged_items__tag__name__in=names)
+            | Q(artist__tagged_items__tag__name__in=names)
+            | Q(album__tagged_items__tag__name__in=names)
+        )

api/funkwhale_api/radios/radios.py

@@ -2,18 +2,20 @@ import random
 
 from django.core.exceptions import ValidationError
 from django.db import connection
+from django.db.models import Q
 from rest_framework import serializers
-from taggit.models import Tag
 
 from funkwhale_api.moderation import filters as moderation_filters
 from funkwhale_api.music.models import Artist, Track
-from funkwhale_api.users.models import User
+from funkwhale_api.tags.models import Tag
 
 from . import filters, models
 from .registries import registry
 
 
 class SimpleRadio(object):
+    related_object_field = None
+
     def clean(self, instance):
         return
 
@@ -146,6 +148,8 @@ class CustomRadio(SessionRadio):
 class RelatedObjectRadio(SessionRadio):
     """Abstract radio related to an object (tag, artist, user...)"""
 
+    related_object_field = serializers.IntegerField(required=True)
+
     def clean(self, instance):
         super().clean(instance)
         if not instance.related_object:
@@ -162,10 +166,22 @@ class RelatedObjectRadio(SessionRadio):
 @registry.register(name="tag")
 class TagRadio(RelatedObjectRadio):
     model = Tag
+    related_object_field = serializers.CharField(required=True)
+
+    def get_related_object(self, name):
+        return self.model.objects.get(name=name)
 
     def get_queryset(self, **kwargs):
         qs = super().get_queryset(**kwargs)
-        return qs.filter(tags__in=[self.session.related_object])
+        query = (
+            Q(tagged_items__tag=self.session.related_object)
+            | Q(artist__tagged_items__tag=self.session.related_object)
+            | Q(album__tagged_items__tag=self.session.related_object)
+        )
+        return qs.filter(query)
+
+    def get_related_object_id_repr(self, obj):
+        return obj.name
 
 
 def weighted_choice(choices):
@@ -246,9 +262,7 @@ class ArtistRadio(RelatedObjectRadio):
 
 
 @registry.register(name="less-listened")
-class LessListenedRadio(RelatedObjectRadio):
-    model = User
-
+class LessListenedRadio(SessionRadio):
     def clean(self, instance):
         instance.related_object = instance.user
         super().clean(instance)

api/funkwhale_api/radios/serializers.py

@@ -54,6 +54,9 @@ class RadioSessionTrackSerializer(serializers.ModelSerializer):
 
 
 class RadioSessionSerializer(serializers.ModelSerializer):
+
+    related_object_id = serializers.CharField(required=False, allow_null=True)
+
     class Meta:
         model = models.RadioSession
         fields = (
@@ -66,7 +69,17 @@ class RadioSessionSerializer(serializers.ModelSerializer):
         )
 
     def validate(self, data):
-        registry[data["radio_type"]]().validate_session(data, **self.context)
+        radio_conf = registry[data["radio_type"]]()
+        if radio_conf.related_object_field:
+            try:
+                data[
+                    "related_object_id"
+                ] = radio_conf.related_object_field.to_internal_value(
+                    data["related_object_id"]
+                )
+            except KeyError:
+                raise serializers.ValidationError("Radio requires a related object")
+        radio_conf.validate_session(data, **self.context)
         return data
 
     def create(self, validated_data):
@@ -77,3 +90,11 @@ class RadioSessionSerializer(serializers.ModelSerializer):
                 validated_data["related_object_id"]
             )
         return super().create(validated_data)
+
+    def to_representation(self, instance):
+        repr = super().to_representation(instance)
+        radio_conf = registry[repr["radio_type"]]()
+        handler = getattr(radio_conf, "get_related_object_id_repr", None)
+        if handler and instance.related_object:
+            repr["related_object_id"] = handler(instance.related_object)
+        return repr

api/funkwhale_api/radios/urls.py

@@ -1,8 +1,8 @@
-from rest_framework import routers
+from funkwhale_api.common import routers
 
 from . import views
 
-router = routers.SimpleRouter()
+router = routers.OptionalSlashRouter()
 router.register(r"sessions", views.RadioSessionViewSet, "sessions")
 router.register(r"radios", views.RadioViewSet, "radios")
 router.register(r"tracks", views.RadioSessionTrackViewSet, "tracks")

api/funkwhale_api/subsonic/renderers.py

@@ -53,5 +53,8 @@ def dict_to_xml_tree(root_tag, d, parent=None):
             for obj in value:
                 root.append(dict_to_xml_tree(key, obj, parent=root))
         else:
-            root.set(key, str(value))
+            if key == "value":
+                root.text = str(value)
+            else:
+                root.set(key, str(value))
     return root

api/funkwhale_api/subsonic/serializers.py

@@ -5,6 +5,7 @@ from rest_framework import serializers
 
 from funkwhale_api.history import models as history_models
 from funkwhale_api.music import models as music_models
+from funkwhale_api.music import utils as music_utils
 
 
 def get_artist_data(artist_values):
@@ -71,7 +72,14 @@ def get_track_data(album, track, upload):
         "artist": album.artist.name,
         "track": track.position or 1,
         "discNumber": track.disc_number or 1,
-        "contentType": upload.mimetype,
+        # Ugly fallback to mp3 but some subsonic clients fail if the value is empty or null, and we don't always
+        # have the info on legacy uploads
+        "contentType": upload.mimetype
+        or (
+            music_utils.get_type_from_ext(upload.extension)
+            if upload.extension
+            else "audio/mpeg"
+        ),
         "suffix": upload.extension or "",
         "duration": upload.duration or 0,
         "created": track.creation_date,
@@ -228,7 +236,11 @@ def get_music_directory_data(artist):
 
 
 def get_folders(user):
-    return []
+    return [
+        # Dummy folder ID to match what is returned in the getMusicFolders endpoint
+        # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/624
+        {"id": 1, "name": "Music"}
+    ]
 
 
 def get_user_detail_data(user):
@@ -263,3 +275,11 @@ class ScrobbleSerializer(serializers.Serializer):
         return history_models.Listening.objects.create(
             user=self.context["user"], track=data["id"]
         )
+
+
+def get_genre_data(tag):
+    return {
+        "songCount": getattr(tag, "_tracks_count", 0),
+        "albumCount": getattr(tag, "_albums_count", 0),
+        "value": tag.name,
+    }

api/funkwhale_api/subsonic/views.py

@@ -2,6 +2,8 @@ import datetime
 import functools
 
 from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
+from django.db.models import Count, Q
 from django.utils import timezone
 from rest_framework import exceptions
 from rest_framework import permissions as rest_permissions
@@ -18,6 +20,7 @@ from funkwhale_api.music import models as music_models
 from funkwhale_api.music import utils
 from funkwhale_api.music import views as music_views
 from funkwhale_api.playlists import models as playlists_models
+from funkwhale_api.tags import models as tags_models
 from funkwhale_api.users import models as users_models
 
 from . import authentication, filters, negotiation, serializers
@@ -330,6 +333,48 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         }
         return response.Response(data)
 
+    @action(
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_songs_by_genre",
+        url_path="getSongsByGenre",
+    )
+    def get_songs_by_genre(self, request, *args, **kwargs):
+        data = request.GET or request.POST
+        actor = utils.get_actor_from_request(request)
+        queryset = music_models.Track.objects.all().exclude(
+            moderation_filters.get_filtered_content_query(
+                moderation_filters.USER_FILTER_CONFIG["TRACK"], request.user
+            )
+        )
+        queryset = queryset.playable_by(actor)
+        try:
+            size = int(
+                data["count"]
+            )  # yep. Some endpoints have size, other have count…
+        except (TypeError, KeyError, ValueError):
+            size = 50
+
+        genre = data.get("genre")
+        queryset = (
+            queryset.playable_by(actor)
+            .filter(
+                Q(tagged_items__tag__name=genre)
+                | Q(artist__tagged_items__tag__name=genre)
+                | Q(album__artist__tagged_items__tag__name=genre)
+                | Q(album__tagged_items__tag__name=genre)
+            )
+            .prefetch_related("uploads")
+            .distinct()
+            .order_by("-creation_date")[:size]
+        )
+        data = {
+            "songsByGenre": {
+                "song": serializers.GetSongSerializer(queryset, many=True).data
+            }
+        }
+        return response.Response(data)
+
     @action(
         detail=False,
         methods=["get", "post"],
@@ -362,6 +407,26 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         queryset = filterset.qs
         actor = utils.get_actor_from_request(request)
         queryset = queryset.playable_by(actor)
+        type = data.get("type", "alphabeticalByArtist")
+
+        if type == "alphabeticalByArtist":
+            queryset = queryset.order_by("artist__name")
+        elif type == "random":
+            queryset = queryset.order_by("?")
+        elif type == "alphabeticalByName" or not type:
+            queryset = queryset.order_by("artist__title")
+        elif type == "recent" or not type:
+            queryset = queryset.exclude(release_date__in=["", None]).order_by(
+                "-release_date"
+            )
+        elif type == "newest" or not type:
+            queryset = queryset.order_by("-creation_date")
+        elif type == "byGenre" and data.get("genre"):
+            genre = data.get("genre")
+            queryset = queryset.filter(
+                Q(tagged_items__tag__name=genre)
+                | Q(artist__tagged_items__tag__name=genre)
+            )
 
         try:
             offset = int(data["offset"])
@@ -669,3 +734,29 @@ class SubsonicViewSet(viewsets.GenericViewSet):
             listening = serializer.save()
             record.send(listening)
         return response.Response({})
+
+    @action(
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_genres",
+        url_path="getGenres",
+    )
+    def get_genres(self, request, *args, **kwargs):
+        album_ct = ContentType.objects.get_for_model(music_models.Album)
+        track_ct = ContentType.objects.get_for_model(music_models.Track)
+        queryset = (
+            tags_models.Tag.objects.annotate(
+                _albums_count=Count(
+                    "tagged_items", filter=Q(tagged_items__content_type=album_ct)
+                ),
+                _tracks_count=Count(
+                    "tagged_items", filter=Q(tagged_items__content_type=track_ct)
+                ),
+            )
+            .exclude(_tracks_count=0, _albums_count=0)
+            .order_by("name")
+        )
+        data = {
+            "genres": {"genre": [serializers.get_genre_data(tag) for tag in queryset]}
+        }
+        return response.Response(data)

api/funkwhale_api/tags/admin.py

@@ -0,0 +1,18 @@
+from funkwhale_api.common import admin
+
+from . import models
+
+
+@admin.register(models.Tag)
+class TagAdmin(admin.ModelAdmin):
+    list_display = ["name", "creation_date"]
+    search_fields = ["name"]
+    list_select_related = True
+
+
+@admin.register(models.TaggedItem)
+class TaggedItemAdmin(admin.ModelAdmin):
+    list_display = ["object_id", "content_type", "tag", "creation_date"]
+    search_fields = ["tag__name"]
+    list_filter = ["content_type"]
+    list_select_related = True

api/funkwhale_api/tags/factories.py

@@ -0,0 +1,33 @@
+import factory
+
+from funkwhale_api.factories import registry, NoUpdateOnCreate
+
+from . import models
+
+
+@registry.register
+class TagFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    name = factory.Faker("music_hashtag")
+
+    class Meta:
+        model = "tags.Tag"
+
+
+@registry.register
+class TaggedItemFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    tag = factory.SubFactory(TagFactory)
+    content_object = None
+
+    class Meta:
+        model = "tags.TaggedItem"
+
+
+class TaggableFactory(factory.django.DjangoModelFactory):
+    @factory.post_generation
+    def set_tags(self, create, extracted, **kwargs):
+        if not create:
+            # Simple build, do nothing.
+            return
+
+        if extracted:
+            models.set_tags(self, *extracted)

api/funkwhale_api/tags/filters.py

@@ -0,0 +1,21 @@
+import django_filters
+from django_filters import rest_framework as filters
+
+from funkwhale_api.common import fields
+
+from . import models
+
+
+class TagFilter(filters.FilterSet):
+    q = fields.SearchFilter(search_fields=["name"])
+    ordering = django_filters.OrderingFilter(
+        fields=(
+            ("name", "name"),
+            ("creation_date", "creation_date"),
+            ("__size", "length"),
+        )
+    )
+
+    class Meta:
+        model = models.Tag
+        fields = {"q": ["exact"], "name": ["exact", "startswith"]}

api/funkwhale_api/tags/migrations/0001_initial.py

@@ -0,0 +1,85 @@
+# Generated by Django 2.2.3 on 2019-07-05 08:22
+
+import django.contrib.postgres.fields.citext
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("common", "0003_cit_extension"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Tag",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "name",
+                    django.contrib.postgres.fields.citext.CICharField(
+                        max_length=100, unique=True
+                    ),
+                ),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="TaggedItem",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                (
+                    "object_id",
+                    models.IntegerField(db_index=True, verbose_name="Object id"),
+                ),
+                (
+                    "content_type",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="tags_taggeditem_tagged_items",
+                        to="contenttypes.ContentType",
+                        verbose_name="Content type",
+                    ),
+                ),
+                (
+                    "tag",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="tags_taggeditem_items",
+                        to="tags.Tag",
+                    ),
+                ),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name="taggeditem", unique_together={("tag", "content_type", "object_id")}
+        ),
+    ]