Merge branch 'release/0.18'

.env.dev

@@ -12,3 +12,9 @@ MUSIC_DIRECTORY_PATH=/music
 BROWSABLE_API_ENABLED=True
 FORWARDED_PROTO=http
 LDAP_ENABLED=False
+
+# Uncomment this if you're using traefik/https
+# FORCE_HTTPS_URLS=True
+
+# Customize to your needs
+POSTGRES_VERSION=11

.gitignore

@@ -93,4 +93,6 @@ po/*.po
 docs/swagger
 _build
 front/src/translations.json
+front/src/translations/*.json
 front/locales/en_US/LC_MESSAGES/app.po
+*.prof

.gitlab-ci.yml

@@ -1,7 +1,8 @@
 variables:
   IMAGE_NAME: funkwhale/funkwhale
   IMAGE: $IMAGE_NAME:$CI_COMMIT_REF_NAME
-  IMAGE_LATEST: $IMAGE_NAME:latest
+  ALL_IN_ONE_IMAGE_NAME: funkwhale/all-in-one
+  ALL_IN_ONE_IMAGE: $ALL_IN_ONE_IMAGE_NAME:$CI_COMMIT_REF_NAME
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
   PYTHONDONTWRITEBYTECODE: "true"
   REVIEW_DOMAIN: preview.funkwhale.audio
@@ -131,16 +132,15 @@ flake8:
 
 test_api:
   services:
-    - postgres:9.4
+    - postgres:11
     - redis:3
   stage: test
-  image: funkwhale/funkwhale:latest
+  image: funkwhale/funkwhale:develop
   cache:
     key: "$CI_PROJECT_ID__pip_cache"
     paths:
       - "$PIP_CACHE_DIR"
   variables:
-    DJANGO_ALLOWED_HOSTS: "localhost"
     DATABASE_URL: "postgresql://postgres@postgres/postgres"
     FUNKWHALE_URL: "https://funkwhale.ci"
     DJANGO_SETTINGS_MODULE: config.settings.local
@@ -148,11 +148,10 @@ test_api:
     - branches
   before_script:
     - cd api
-    - apt-get update
+    - sed -i '/Pillow/d' requirements/base.txt
-    - grep "^[^#;]" requirements.apt | grep -Fv "python3-dev" | xargs apt-get install -y --no-install-recommends
+    - pip3 install -r requirements/base.txt
-    - pip install -r requirements/base.txt
+    - pip3 install -r requirements/local.txt
-    - pip install -r requirements/local.txt
+    - pip3 install -r requirements/test.txt
-    - pip install -r requirements/test.txt
   script:
     - pytest --cov=funkwhale_api tests/
   tags:
@@ -166,7 +165,7 @@ test_front:
   only:
     - branches
   script:
-    - yarn install
+    - yarn install --check-files
     - yarn test:unit
   cache:
     key: "funkwhale__front_dependencies"
@@ -194,11 +193,6 @@ build_front:
     # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169
     - yarn build | tee /dev/stderr | (! grep -i 'ERROR in')
     - chmod -R 755 dist
-  cache:
-    key: "funkwhale__front_dependencies"
-    paths:
-      - front/node_modules
-      - front/yarn.lock
   artifacts:
     name: "front_${CI_COMMIT_REF_NAME}"
     paths:
@@ -207,6 +201,7 @@ build_front:
     - tags@funkwhale/funkwhale
     - master@funkwhale/funkwhale
     - develop@funkwhale/funkwhale
+
   tags:
     - docker
 
@@ -236,9 +231,11 @@ pages:
 
 docker_release:
   stage: deploy
+  image: bash
   before_script:
     - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
     - cp -r front/dist api/frontend
+    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
     - cd api
   script:
     - docker build -t $IMAGE .
@@ -249,15 +246,42 @@ docker_release:
   tags:
     - docker-build
 
+docker_all_in_one_release:
+  stage: deploy
+  image: bash
+  variables:
+    ALL_IN_ONE_REF: master
+    ALL_IN_ONE_ARTIFACT_URL: https://github.com/thetarkus/docker-funkwhale/archive/$ALL_IN_ONE_REF.zip
+    BUILD_PATH: all_in_one
+  before_script:
+    - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
+    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
+  script:
+    - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
+    - unzip -o all_in_one.zip -d tmpdir
+    - mv tmpdir/docker-funkwhale-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
+    - cp -r api $BUILD_PATH/src/api
+    - cp -r front $BUILD_PATH/src/front
+    - cd $BUILD_PATH
+    - ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
+    - docker build -t $ALL_IN_ONE_IMAGE .
+    - docker push $ALL_IN_ONE_IMAGE
+  only:
+    - develop@funkwhale/funkwhale
+    - tags@funkwhale/funkwhale
+  tags:
+    - docker-build
+
 build_api:
   # Simply publish a zip containing api/ directory
   stage: deploy
-  image: busybox
+  image: bash
   artifacts:
     name: "api_${CI_COMMIT_REF_NAME}"
     paths:
       - api
   script:
+    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
     - chmod -R 750 api
     - echo Done!
   only:

.gitlab/issue_templates/Bug.md

@@ -34,6 +34,12 @@ Describe the expected behaviour.
 
 ## Context
 
+<!--
+The version of your instance can be found on the footer : Source code (x.y)
+-->
+
+**Funkwhale version(s) affected**: x.y
+
 <!--
 If relevant, share additional context here like:
 

CHANGELOG

@@ -10,6 +10,297 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.
 
 .. towncrier
 
+0.18 "Naomi" (2019-01-22)
+-------------------------
+
+This release is dedicated to Naomi, an early contributor and beta tester of Funkwhale.
+Her positivity, love and support have been incredibly helpful and helped shape the project
+as you can enjoy it today. Thank you so much Naomi <3
+
+Upgrade instructions are available at
+https://docs.funkwhale.audio/index.html, ensure you also execute the intructions
+marked with ``[manual action required]`` and ``[manual action suggested]``.
+
+See ``Full changelog`` below for an exhaustive list of changes!
+
+Audio transcoding is back!
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+After removal of our first, buggy transcoding implementation, we're proud to announce
+that this feature is back. It is enabled by default, and can be configured/disabled
+in your instance settings!
+
+This feature works in the browser, with federated/non-federated tracks and using Subsonic clients.
+Transcoded tracks are generated on the fly, and cached for a configurable amount of time,
+to reduce the load on the server.
+
+
+Licensing and copyright information
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Funkwhale is now able to parse copyright and license data from file and store
+this information. Apart from displaying it on each track detail page,
+no additional behaviour is currently implemented to use this new data, but this
+will change in future releases.
+
+License and copyright data is also broadcasted over federation.
+
+License matching is done on the content of the ``License`` tag in the files,
+with a fallback on the ``Copyright`` tag.
+
+Funkwhale will successfully extract licensing data for the following licenses:
+
+- Creative Commons 0 (Public Domain)
+- Creative Commons 1.0 (All declinations)
+- Creative Commons 2.0 (All declinations)
+- Creative Commons 2.5 (All declinations and countries)
+- Creative Commons 3.0 (All declinations and countries)
+- Creative Commons 4.0 (All declinations)
+
+Support for other licenses such as Art Libre or WTFPL will be added in future releases.
+
+
+Instance-level moderation tools
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This release includes a first set of moderation tools that will give more control
+to admins about the way their instance federate with other instance and accounts on the network.
+Using these tools, it's now possible to:
+
+- Browse known accounts and domains, and associated data (storage size, software version, etc.)
+- Purge data belonging to given accounts and domains
+- Block or partially restrict interactions with any account or domain
+
+All those features are usable using a brand new "moderation" permission, meaning
+you can appoints one or nultiple moderators to help with this task.
+
+I'd like to thank all Mastodon contributors, because some of the these tools are heavily
+inspired from what's being done in Mastodon. Thank you so much!
+
+
+Iframe widget to embed public tracks and albums [manual action required]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Funkwhale now support embedding a lightweight audio player on external websites
+for album and tracks that are available in public libraries. Important pages,
+such as artist, album and track pages also include OpenGraph tags that will
+enable previews on compatible apps (like sharing a Funkwhale track link on Mastodon
+or Twitter).
+
+To achieve that, we had to tweak the way Funkwhale front-end is served. You'll have
+to modify your nginx configuration when upgrading to keep your instance working.
+
+**On docker setups**, edit your ``/srv/funkwhale/nginx/funkwhale.template`` and replace
+the ``location /api/`` and `location /` blocks by the following snippets::
+
+    location / {
+        include /etc/nginx/funkwhale_proxy.conf;
+        # this is needed if you have file import via upload enabled
+        client_max_body_size ${NGINX_MAX_BODY_SIZE};
+        proxy_pass   http://funkwhale-api/;
+    }
+
+    location /front/ {
+        alias /frontend/;
+    }
+
+The change of configuration will be picked when restarting your nginx container.
+
+**On non-docker setups**, edit your ``/etc/nginx/sites-available/funkwhale.conf`` file,
+and replace the ``location /api/`` and `location /` blocks by the following snippets::
+
+
+    location / {
+        include /etc/nginx/funkwhale_proxy.conf;
+        # this is needed if you have file import via upload enabled
+        client_max_body_size ${NGINX_MAX_BODY_SIZE};
+        proxy_pass   http://funkwhale-api/;
+    }
+
+    location /front/ {
+        alias ${FUNKWHALE_FRONTEND_PATH}/;
+    }
+
+Replace ``${FUNKWHALE_FRONTEND_PATH}`` by the corresponding variable from your .env file,
+which should be ``/srv/funkwhale/front/dist`` by default, then reload your nginx process with
+``sudo systemctl reload nginx``.
+
+
+Alternative docker deployment method
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Thanks to the awesome done by @thetarkus at https://github.com/thetarkus/docker-funkwhale,
+we're now able to provide an alternative and easier Docker deployment method!
+
+In contrast with our current, multi-container offer, this method integrates
+all Funkwhale processes and services (database, redis, etc.) into a single, easier to deploy container.
+
+Both method will coexist in parallel, as each one has pros and cons. You can learn more
+about this exciting new deployment option by visiting https://docs.funkwhale.audio/installation/docker.html!
+
+Automatically load .env file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+On non-docker deployments, earlier versions required you to source
+the config/.env file before launching any Funkwhale command, with ``export $(cat config/.env | grep -v ^# | xargs)``
+This led to more complex and error prode deployment / setup.
+
+This is not the case anymore, and Funkwhale will automatically load this file if it's available.
+
+
+Delete pre 0.17 federated tracks [manual action suggested]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you were using Funkwhale before the 0.17 release and federated with other instances,
+it's possible that you still have some unplayable federated files in the database.
+
+To purge the database of those entries, you can run the following command:
+
+On docker setups::
+
+    docker-compose run --rm api python manage.py script delete_pre_017_federated_uploads --no-input
+
+On non-docker setups::
+
+    python manage.py script delete_pre_017_federated_uploads --no-input
+
+
+Enable gzip compression [manual action suggested]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Gzip compression will be enabled on new instances by default
+and will reduce the amount of bandwidth consumed by your instance.
+
+If you with to benefit from gzip compression on your instance,
+edit your reverse proxy virtualhost file (located at ``/etc/nginx/sites-available/funkwhale.conf``) and add the following snippet
+in the server block, then reload your nginx server::
+
+    server {
+        # ... exiting configuration
+
+        # compression settings
+        gzip on;
+        gzip_comp_level    5;
+        gzip_min_length    256;
+        gzip_proxied       any;
+        gzip_vary          on;
+
+        gzip_types
+            application/atom+xml
+            application/javascript
+            application/json
+            application/ld+json
+            application/activity+json
+            application/manifest+json
+            application/rss+xml
+            application/vnd.geo+json
+            application/vnd.ms-fontobject
+            application/x-font-ttf
+            application/x-web-app-manifest+json
+            application/xhtml+xml
+            application/xml
+            font/opentype
+            image/bmp
+            image/svg+xml
+            image/x-icon
+            text/cache-manifest
+            text/css
+            text/plain
+            text/vcard
+            text/vnd.rim.location.xloc
+            text/vtt
+            text/x-component
+            text/x-cross-domain-policy;
+        # end of compression settings
+    }
+
+Full changelog
+^^^^^^^^^^^^^^
+
+Features:
+
+- Allow embedding of albums and tracks available in public libraries via an <iframe> (#578)
+- Audio transcoding is back! (#272)
+- First set of instance level moderation tools (#580, !521)
+- Store licensing and copyright information from file metadata, if available (#308)
+
+
+Enhancements:
+
+- Add UI elements for multi-disc albums (#631)
+- Added alternative funkwhale/all-in-one docker image (#614)
+- Broadcast library updates (name, description, visibility) over federation
+- Based Docker image on alpine to have a smaller (and faster to build) image
+- Improved front-end performance by stripping unused dependencies, reducing bundle size
+  and enabling gzip compression
+- Improved accessibility by using main/section/nav tags and aria-labels in most critical places (#612)
+- The progress bar in the player now display loading state / buffer loading (#586)
+- Added "type: funkwhale" and "funkwhale-version" in Subsonic responses (#573)
+- Documented keyboard shortcuts, list is now available by pressing "h" or in the footer (#611)
+- Documented which Subsonic endpoints are implemented (#575)
+- Hide invitation code field during signup when it's not required (#410)
+- Importer will now pick embedded images in files with OTHER type if no COVER_FRONT is present
+- Improved keyboard accessibility on player, queue and various controls (#576)
+- Improved performance when listing playable tracks, albums and artists
+- Increased default upload limit from 30 to 100MB (#654)
+- Load env file in config/.env automatically to avoid sourcing it by hand (#626)
+- More resilient date parsing during audio import, will not crash anymore on
+  invalid dates (#622)
+- Now start radios immediatly, skipping any existing tracks in queue (#585)
+- Officially support connecting to a password protected redis server, with
+  the redis://:password@localhost:6379/0 scheme (#640)
+- Performance improvement when fetching favorites, down to a single, small http request
+- Removed "Activity" page, since all the data is available on the "Browse" page (#600)
+- Removed the need to specify the DJANGO_ALLOWED_HOSTS variable
+- Restructured the footer, added useful links and removed unused content
+- Show short entries first in search results to improve UX
+- Store disc number and order tracks by disc number / position) (#507)
+- Strip EXIF metadata from uploaded avatars to avoid leaking private data (#374)
+- Support blind key rotation in HTTP Signatures (#658)
+- Support setting a server URL in settings.json (#650)
+- Updated default docker postgres version from 9.4 to 11 (#656)
+- Updated lots of dependencies (especially django 2.0->2.1), and removed unused dependencies (#657)
+- Improved test suite speed by reducing / disabling expensive operations (#648)
+
+
+Bugfixes:
+
+- Fixed parsing of embedded file cover for ogg files tagged with MusicBrainz (#469)
+- Upgraded core dependencies to fix websocket/messaging issues and possible memory leaks (#643)
+- Fix ".None" extension when downloading Flac file (#473)
+- Fixed None extension when downloading an in-place imported file (#621)
+- Added a script to prune pre 0.17 federated tracks (#564)
+- Advertise public libraries properly in ActivityPub representations (#553)
+- Allow opus file upload (#598)
+- Do not display "view on MusicBrainz" button if we miss the mbid (#422)
+- Do not try to create unaccent extension if it's already present (#663)
+- Ensure admin links in sidebar are displayed for users with relavant permissions, and only them (#597)
+- Fix broken websocket connexion under Chrome browser (#589)
+- Fix play button not starting playback with empty queue (#632)
+- Fixed a styling inconsistency on about page when instance description was missing (#659)
+- Fixed a UI discrepency in playlist tracks count (#647)
+- Fixed greyed tracks in radio builder and detail page (#637)
+- Fixed inconsistencies in subsonic error responses (#616)
+- Fixed incorrect icon for "next track" in player control (#613)
+- Fixed malformed search string when redirecting to LyricsWiki (#608)
+- Fixed missing track count on various library cards (#581)
+- Fixed skipped track when appending multiple tracks to the queue under certain conditions (#209)
+- Fixed wrong album/track count on artist page (#599)
+- Hide unplayable/emtpy playlists in "Browse playlist" pages (#424)
+- Initial UI render using correct language from browser (#644)
+- Invalid URI for reverse proxy websocket with apache (#617)
+- Properly encode Wikipedia and lyrics search urls (#470)
+- Refresh profile after user settings update to avoid cache issues (#606)
+- Use role=button instead of empty links for player controls (#610)
+
+
+Documentation:
+
+- Deploy documentation from the master branch instead of the develop branch to avoid inconsistencies (#642)
+- Document how to find and use library id when importing files in CLI (#562)
+- Fix documentation typos (#645)
+
+
 0.17 (2018-10-07)
 -----------------
 
@@ -120,7 +411,7 @@ Then, add the following block at the end of your docker-compose.yml file::
         - .env
       environment:
         # Override those variables in your .env file if needed
-        - "NGINX_MAX_BODY_SIZE=${NGINX_MAX_BODY_SIZE-30M}"
+        - "NGINX_MAX_BODY_SIZE=${NGINX_MAX_BODY_SIZE-100M}"
       volumes:
         - "./nginx/funkwhale.template:/etc/nginx/conf.d/funkwhale.template:ro"
         - "./nginx/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro"

CONTRIBUTING.rst

@@ -35,18 +35,20 @@ Setup front-end only development environment
     cd funkwhale
     cd front
 
-2. Install [nodejs](https://nodejs.org/en/download/package-manager/) and [yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)
+2. Install `nodejs <https://nodejs.org/en/download/package-manager/>`_ and `yarn <https://yarnpkg.com/lang/en/docs/install/#debian-stable>`_
+
 3. Install the dependencies::
 
     yarn install
 
 4. Launch the development server::
 
-    # this will serve the front-end on http://localhost:8000
+    # this will serve the front-end on http://localhost:8000/front/
     VUE_PORT=8000 yarn serve
 
 5. Make the front-end talk with an existing server (like https://demo.funkwhale.audio),
    by clicking on the corresponding link in the footer
+
 6. Start hacking!
 
 Setup your development environment
@@ -307,7 +309,7 @@ A typical fragment looks like that:
     Fixed broken audio player on Chrome 42 for ogg files (#567)
 
 If the work fixes one or more issues, the issue number should be included at the
-end of the fragment (``(#567)`` is the issue number in the previous example.
+end of the fragment (``(#567)`` is the issue number in the previous example).
 
 If your work is not related to a specific issue, use the merge request
 identifier instead, like this:
@@ -389,7 +391,7 @@ This is regular pytest, so you can use any arguments/options that pytest usually
     # Stop on first failure
     docker-compose -f dev.yml run --rm api pytest -x
     # Run a specific test file
-    docker-compose -f dev.yml run --rm api pytest tests/test_acoustid.py
+    docker-compose -f dev.yml run --rm api pytest tests/music/test_models.py
 
 Writing tests
 ^^^^^^^^^^^^^
@@ -507,7 +509,7 @@ useful when testing components that depend on each other:
         # here, we ensure no email was sent
         mocked_notify.assert_not_called()
 
-Views: you can find some readable views tests in :file:`tests/users/test_views.py`
+Views: you can find some readable views tests in file: ``api/tests/users/test_views.py``
 
 .. note::
 

api/Dockerfile

@@ -1,27 +1,47 @@
-FROM python:3.6
+FROM alpine:3.8
 
-ENV PYTHONUNBUFFERED 1
+RUN \
+    echo 'installing dependencies' && \
+    apk add                \
+    bash             \
+    git             \
+    gettext            \
+    musl-dev           \
+    gcc \
+    postgresql-dev \
+    python3-dev        \
+    py3-psycopg2       \
+    py3-pillow         \
+    libldap            \
+    ffmpeg             \
+    libpq              \
+    libmagic           \
+    libffi-dev         \
+    zlib-dev           \
+    openldap-dev && \
+    \
+    \
+    ln -s /usr/bin/python3 /usr/bin/python
 
-# Requirements have to be pulled and installed here, otherwise caching won't work
+RUN mkdir /requirements
-RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
-COPY ./requirements.apt /requirements.apt
-RUN apt-get update; \
-    grep "^[^#;]" requirements.apt | \
-    grep -Fv "python3-dev" | \
-    xargs apt-get install -y --no-install-recommends; \
-    rm -rf /usr/share/doc/* /usr/share/locale/*
-RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
 COPY ./requirements/base.txt /requirements/base.txt
-RUN pip install -r /requirements/base.txt
+RUN \
-COPY ./requirements/production.txt /requirements/production.txt
+    echo 'fixing requirements file for alpine' && \
-RUN pip install -r /requirements/production.txt
+    sed -i '/Pillow/d' /requirements/base.txt && \
+    \
+    \
+    echo 'installing pip requirements' && \
+    pip3 install --no-cache-dir --upgrade pip && \
+    pip3 install --no-cache-dir setuptools wheel && \
+    pip3 install --no-cache-dir -r /requirements/base.txt
 
-COPY . /app
+ARG install_dev_deps=0
-
+COPY ./requirements/*.txt /requirements/
-# Since youtube-dl code is updated fairly often, we split it here
+RUN \
-RUN pip install --upgrade youtube-dl
+    if [ "$install_dev_deps" = "1" ] ; then echo "Installing dev dependencies" && pip3 install --no-cache-dir -r /requirements/local.txt -r /requirements/test.txt ; else echo "Skipping dev deps installation" ; fi
-
-WORKDIR /app
 
 ENTRYPOINT ["./compose/django/entrypoint.sh"]
 CMD ["./compose/django/daphne.sh"]
+
+COPY . /app
+WORKDIR /app

api/compose/django/daphne.sh

@@ -1,3 +1,3 @@
 #!/bin/bash -eux
 python /app/manage.py collectstatic --noinput
-/usr/local/bin/daphne -b 0.0.0.0 -p 5000 config.asgi:application --proxy-headers
+daphne -b 0.0.0.0 -p 5000 config.asgi:application --proxy-headers

api/compose/django/dev-entrypoint.sh

@@ -1,3 +1,3 @@
-#!/bin/bash
+#!/bin/sh
 set -e
 exec "$@"

api/compose/django/entrypoint.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 set -e
 # This entrypoint is used to play nicely with the current cookiecutter configuration.
 # Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple

api/config/api_urls.py

@@ -10,7 +10,7 @@ from funkwhale_api.playlists import views as playlists_views
 from funkwhale_api.subsonic.views import SubsonicViewSet
 
 router = routers.SimpleRouter()
-router.register(r"settings", GlobalPreferencesViewSet, base_name="settings")
+router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
 router.register(r"tags", views.TagViewSet, "tags")
 router.register(r"tracks", views.TrackViewSet, "tracks")
@@ -19,6 +19,7 @@ router.register(r"libraries", views.LibraryViewSet, "libraries")
 router.register(r"listen", views.ListenViewSet, "listen")
 router.register(r"artists", views.ArtistViewSet, "artists")
 router.register(r"albums", views.AlbumViewSet, "albums")
+router.register(r"licenses", views.LicenseViewSet, "licenses")
 router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
 router.register(
     r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks"
@@ -26,10 +27,11 @@ router.register(
 v1_patterns = router.urls
 
 subsonic_router = routers.SimpleRouter(trailing_slash=False)
-subsonic_router.register(r"subsonic/rest", SubsonicViewSet, base_name="subsonic")
+subsonic_router.register(r"subsonic/rest", SubsonicViewSet, basename="subsonic")
 
 
 v1_patterns += [
+    url(r"^oembed/$", views.OembedView.as_view(), name="oembed"),
     url(
         r"^instance/",
         include(("funkwhale_api.instance.urls", "instance"), namespace="instance"),

api/config/settings/common.py

@@ -13,7 +13,7 @@ from __future__ import absolute_import, unicode_literals
 import datetime
 import logging
 
-from urllib.parse import urlparse, urlsplit
+from urllib.parse import urlsplit
 
 import environ
 from celery.schedules import crontab
@@ -69,12 +69,23 @@ else:
         FUNKWHALE_HOSTNAME = _parsed.netloc
         FUNKWHALE_PROTOCOL = _parsed.scheme
 
+FUNKWHALE_PROTOCOL = FUNKWHALE_PROTOCOL.lower()
+FUNKWHALE_HOSTNAME = FUNKWHALE_HOSTNAME.lower()
 FUNKWHALE_URL = "{}://{}".format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME)
-
+FUNKWHALE_SPA_HTML_ROOT = env(
+    "FUNKWHALE_SPA_HTML_ROOT", default=FUNKWHALE_URL + "/front/"
+)
+FUNKWHALE_SPA_HTML_CACHE_DURATION = env.int(
+    "FUNKWHALE_SPA_HTML_CACHE_DURATION", default=60 * 15
+)
+FUNKWHALE_EMBED_URL = env(
+    "FUNKWHALE_EMBED_URL", default=FUNKWHALE_SPA_HTML_ROOT + "embed.html"
+)
+APP_NAME = "Funkwhale"
 
 # XXX: deprecated, see #186
 FEDERATION_ENABLED = env.bool("FEDERATION_ENABLED", default=True)
-FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME)
+FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME).lower()
 # XXX: deprecated, see #186
 FEDERATION_COLLECTION_PAGE_SIZE = env.int("FEDERATION_COLLECTION_PAGE_SIZE", default=50)
 # XXX: deprecated, see #186
@@ -83,7 +94,7 @@ FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
 )
 # XXX: deprecated, see #186
 FEDERATION_ACTOR_FETCH_DELAY = env.int("FEDERATION_ACTOR_FETCH_DELAY", default=60 * 12)
-ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS")
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNAME]
 
 # APP CONFIGURATION
 # ------------------------------------------------------------------------------
@@ -145,10 +156,10 @@ LOCAL_APPS = (
     "funkwhale_api.requests",
     "funkwhale_api.favorites",
     "funkwhale_api.federation",
+    "funkwhale_api.moderation",
     "funkwhale_api.radios",
     "funkwhale_api.history",
     "funkwhale_api.playlists",
-    "funkwhale_api.providers.acoustid",
     "funkwhale_api.subsonic",
 )
 
@@ -159,7 +170,7 @@ INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
 # MIDDLEWARE CONFIGURATION
 # ------------------------------------------------------------------------------
 MIDDLEWARE = (
-    # Make sure djangosecure.middleware.SecurityMiddleware is listed first
+    "funkwhale_api.common.middleware.SPAFallbackMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "corsheaders.middleware.CorsMiddleware",
     "django.middleware.common.CommonMiddleware",
@@ -305,8 +316,7 @@ FILE_UPLOAD_PERMISSIONS = 0o644
 # URL Configuration
 # ------------------------------------------------------------------------------
 ROOT_URLCONF = "config.urls"
-# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
+SPA_URLCONF = "config.spa_urls"
-WSGI_APPLICATION = "config.wsgi.application"
 ASGI_APPLICATION = "config.routing.application"
 
 # This ensures that Django will be able to detect a secure connection
@@ -315,7 +325,7 @@ SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
 # AUTHENTICATION CONFIGURATION
 # ------------------------------------------------------------------------------
 AUTHENTICATION_BACKENDS = (
-    "django.contrib.auth.backends.ModelBackend",
+    "funkwhale_api.users.auth_backends.ModelBackend",
     "allauth.account.auth_backends.AuthenticationBackend",
 )
 SESSION_COOKIE_HTTPONLY = False
@@ -400,15 +410,20 @@ if AUTH_LDAP_ENABLED:
 AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify"
 
 CACHE_DEFAULT = "redis://127.0.0.1:6379/0"
-CACHES = {"default": env.cache_url("CACHE_URL", default=CACHE_DEFAULT)}
+CACHES = {
+    "default": env.cache_url("CACHE_URL", default=CACHE_DEFAULT),
+    "local": {
+        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+        "LOCATION": "local-cache",
+    },
+}
 
 CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
 
-cache_url = urlparse(CACHES["default"]["LOCATION"])
 CHANNEL_LAYERS = {
     "default": {
         "BACKEND": "channels_redis.core.RedisChannelLayer",
-        "CONFIG": {"hosts": [(cache_url.hostname, cache_url.port)]},
+        "CONFIG": {"hosts": [CACHES["default"]["LOCATION"]]},
     }
 }
 
@@ -435,7 +450,12 @@ CELERY_BEAT_SCHEDULE = {
         "task": "federation.clean_music_cache",
         "schedule": crontab(hour="*/2"),
         "options": {"expires": 60 * 2},
-    }
+    },
+    "music.clean_transcoding_cache": {
+        "task": "music.clean_transcoding_cache",
+        "schedule": crontab(hour="*"),
+        "options": {"expires": 60 * 2},
+    },
 }
 
 JWT_AUTH = {
@@ -516,6 +536,7 @@ MUSICBRAINZ_HOSTNAME = env("MUSICBRAINZ_HOSTNAME", default="musicbrainz.org")
 # Custom Admin URL, use {% url 'admin:index' %}
 ADMIN_URL = env("DJANGO_ADMIN_URL", default="^api/admin/")
 CSRF_USE_SESSIONS = True
+SESSION_ENGINE = "django.contrib.sessions.backends.cache"
 
 # Playlist settings
 # XXX: deprecated, see #186
@@ -570,3 +591,9 @@ VERSATILEIMAGEFIELD_RENDITION_KEY_SETS = {
     ]
 }
 VERSATILEIMAGEFIELD_SETTINGS = {"create_images_on_demand": False}
+RSA_KEY_SIZE = 2048
+# for performance gain in tests, since we don't need to actually create the
+# thumbnails
+CREATE_IMAGE_THUMBNAILS = env.bool("CREATE_IMAGE_THUMBNAILS", default=True)
+# we rotate actor keys at most every two days by default
+ACTOR_KEY_ROTATION_DELAY = env.int("ACTOR_KEY_ROTATION_DELAY", default=3600 * 48)

api/config/settings/local.py

@@ -14,6 +14,7 @@ from .common import *  # noqa
 # DEBUG
 # ------------------------------------------------------------------------------
 DEBUG = env.bool("DJANGO_DEBUG", default=True)
+FORCE_HTTPS_URLS = env.bool("FORCE_HTTPS_URLS", default=False)
 TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG
 
 # SECRET CONFIGURATION
@@ -31,7 +32,6 @@ EMAIL_PORT = 1025
 
 # django-debug-toolbar
 # ------------------------------------------------------------------------------
-MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)
 
 # INTERNAL_IPS = ('127.0.0.1', '10.0.2.2',)
 
@@ -39,20 +39,24 @@ DEBUG_TOOLBAR_CONFIG = {
     "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
     "SHOW_TEMPLATE_CONTEXT": True,
     "SHOW_TOOLBAR_CALLBACK": lambda request: True,
-    "JQUERY_URL": "",
+    "JQUERY_URL": "/staticfiles/admin/js/vendor/jquery/jquery.js",
 }
 
 # django-extensions
 # ------------------------------------------------------------------------------
 # INSTALLED_APPS += ('django_extensions', )
-INSTALLED_APPS += ("debug_toolbar",)
+
+# Debug toolbar is slow, we disable it for tests
+DEBUG_TOOLBAR_ENABLED = env.bool("DEBUG_TOOLBAR_ENABLED", default=DEBUG)
+if DEBUG_TOOLBAR_ENABLED:
+    MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)
+    INSTALLED_APPS += ("debug_toolbar",)
 
 # TESTING
 # ------------------------------------------------------------------------------
 TEST_RUNNER = "django.test.runner.DiscoverRunner"
 
 # CELERY
-# In development, all tasks will be executed locally by blocking until the task returns
 CELERY_TASK_ALWAYS_EAGER = False
 # END CELERY
 
@@ -72,3 +76,10 @@ LOGGING = {
     },
 }
 CSRF_TRUSTED_ORIGINS = [o for o in ALLOWED_HOSTS]
+
+
+if env.bool("WEAK_PASSWORDS", default=False):
+    # Faster during tests
+    PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",)
+
+MIDDLEWARE = ("funkwhale_api.common.middleware.DevHttpsMiddleware",) + MIDDLEWARE

api/config/spa_urls.py

@@ -0,0 +1,18 @@
+from django import urls
+
+from funkwhale_api.music import spa_views
+
+
+urlpatterns = [
+    urls.re_path(
+        r"^library/tracks/(?P<pk>\d+)/?$", spa_views.library_track, name="library_track"
+    ),
+    urls.re_path(
+        r"^library/albums/(?P<pk>\d+)/?$", spa_views.library_album, name="library_album"
+    ),
+    urls.re_path(
+        r"^library/artists/(?P<pk>\d+)/?$",
+        spa_views.library_artist,
+        name="library_artist",
+    ),
+]

api/config/wsgi.py

@@ -1,39 +0,0 @@
-"""
-WSGI config for funkwhale_api project.
-
-This module contains the WSGI application used by Django's development server
-and any production WSGI deployments. It should expose a module-level variable
-named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
-this application via the ``WSGI_APPLICATION`` setting.
-
-Usually you will have the standard Django WSGI application here, but it also
-might make sense to replace the whole Django WSGI application with a custom one
-that later delegates to the Django one. For example, you could introduce WSGI
-middleware here, or combine a Django application with an application of another
-framework.
-
-"""
-import os
-
-from django.core.wsgi import get_wsgi_application
-from whitenoise.django import DjangoWhiteNoise
-
-# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks
-# if running multiple sites in the same mod_wsgi process. To fix this, use
-# mod_wsgi daemon mode with each site in its own daemon process, or use
-# os.environ["DJANGO_SETTINGS_MODULE"] = "config.settings.production"
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
-
-# This application object is used by any WSGI server configured to use this
-# file. This includes Django's development server, if the WSGI_APPLICATION
-# setting points here.
-application = get_wsgi_application()
-
-# Use Whitenoise to serve static files
-# See: https://whitenoise.readthedocs.org/
-application = DjangoWhiteNoise(application)
-
-
-# Apply WSGI middleware here.
-# from helloworld.wsgi import HelloWorldApplication
-# application = HelloWorldApplication(application)

api/docker/Dockerfile.test

@@ -1,26 +0,0 @@
-FROM python:3.6
-
-ENV PYTHONUNBUFFERED 1
-
-# Requirements have to be pulled and installed here, otherwise caching won't work
-RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
-COPY ./requirements.apt /requirements.apt
-RUN apt-get update; \
-    grep "^[^#;]" requirements.apt | \
-    grep -Fv "python3-dev" | \
-    xargs apt-get install -y --no-install-recommends; \
-    rm -rf /usr/share/doc/* /usr/share/locale/*
-
-RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
-
-RUN mkdir /requirements
-COPY ./requirements/base.txt /requirements/base.txt
-RUN pip install -r /requirements/base.txt
-COPY ./requirements/local.txt /requirements/local.txt
-RUN pip install -r /requirements/local.txt
-COPY ./requirements/test.txt /requirements/test.txt
-RUN pip install -r /requirements/test.txt
-
-COPY . /app
-WORKDIR /app
-ENTRYPOINT ["compose/django/dev-entrypoint.sh"]

api/funkwhale_api/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = "0.17"
+__version__ = "0.18"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

api/funkwhale_api/common/channels.py

@@ -5,7 +5,7 @@ from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
 from django.core.serializers.json import DjangoJSONEncoder
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 channel_layer = get_channel_layer()
 group_add = async_to_sync(channel_layer.group_add)
 

api/funkwhale_api/common/decorators.py

@@ -0,0 +1,14 @@
+from rest_framework import response
+from rest_framework import decorators
+
+
+def action_route(serializer_class):
+    @decorators.action(methods=["post"], detail=False)
+    def action(self, request, *args, **kwargs):
+        queryset = self.get_queryset()
+        serializer = serializer_class(request.data, queryset=queryset)
+        serializer.is_valid(raise_exception=True)
+        result = serializer.save()
+        return response.Response(result, status=200)
+
+    return action

api/funkwhale_api/common/middleware.py

@@ -0,0 +1,159 @@
+import html
+import requests
+
+from django import http
+from django.conf import settings
+from django.core.cache import caches
+from django import urls
+
+from . import preferences
+from . import utils
+
+EXCLUDED_PATHS = ["/api", "/federation", "/.well-known"]
+
+
+def should_fallback_to_spa(path):
+    if path == "/":
+        return True
+    return not any([path.startswith(m) for m in EXCLUDED_PATHS])
+
+
+def serve_spa(request):
+    html = get_spa_html(settings.FUNKWHALE_SPA_HTML_ROOT)
+    head, tail = html.split("</head>", 1)
+    if not preferences.get("common__api_authentication_required"):
+        try:
+            request_tags = get_request_head_tags(request) or []
+        except urls.exceptions.Resolver404:
+            # we don't have any custom tags for this route
+            request_tags = []
+    else:
+        # API is not open, we don't expose any custom data
+        request_tags = []
+    default_tags = get_default_head_tags(request.path)
+    unique_attributes = ["name", "property"]
+
+    final_tags = request_tags
+    skip = []
+
+    for t in final_tags:
+        for attr in unique_attributes:
+            if attr in t:
+                skip.append(t[attr])
+    for t in default_tags:
+        existing = False
+        for attr in unique_attributes:
+            if t.get(attr) in skip:
+                existing = True
+                break
+        if not existing:
+            final_tags.append(t)
+
+    # let's inject our meta tags in the HTML
+    head += "\n" + "\n".join(render_tags(final_tags)) + "\n</head>"
+
+    return http.HttpResponse(head + tail)
+
+
+def get_spa_html(spa_url):
+    cache_key = "spa-html:{}".format(spa_url)
+    cached = caches["local"].get(cache_key)
+    if cached:
+        return cached
+
+    response = requests.get(
+        utils.join_url(spa_url, "index.html"),
+        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+    )
+    response.raise_for_status()
+    content = response.text
+    caches["local"].set(cache_key, content, settings.FUNKWHALE_SPA_HTML_CACHE_DURATION)
+    return content
+
+
+def get_default_head_tags(path):
+    instance_name = preferences.get("instance__name")
+    short_description = preferences.get("instance__short_description")
+    app_name = settings.APP_NAME
+
+    parts = [instance_name, app_name]
+
+    return [
+        {"tag": "meta", "property": "og:type", "content": "website"},
+        {
+            "tag": "meta",
+            "property": "og:site_name",
+            "content": " - ".join([p for p in parts if p]),
+        },
+        {"tag": "meta", "property": "og:description", "content": short_description},
+        {
+            "tag": "meta",
+            "property": "og:image",
+            "content": utils.join_url(settings.FUNKWHALE_URL, "/front/favicon.png"),
+        },
+        {
+            "tag": "meta",
+            "property": "og:url",
+            "content": utils.join_url(settings.FUNKWHALE_URL, path),
+        },
+    ]
+
+
+def render_tags(tags):
+    """
+    Given a dict like {'tag': 'meta', 'hello': 'world'}
+    return a html ready tag like
+    <meta hello="world" />
+    """
+    for tag in tags:
+
+        yield "<{tag} {attrs} />".format(
+            tag=tag.pop("tag"),
+            attrs=" ".join(
+                [
+                    '{}="{}"'.format(a, html.escape(str(v)))
+                    for a, v in sorted(tag.items())
+                    if v
+                ]
+            ),
+        )
+
+
+def get_request_head_tags(request):
+    match = urls.resolve(request.path, urlconf=settings.SPA_URLCONF)
+    return match.func(request, *match.args, **match.kwargs)
+
+
+class SPAFallbackMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+
+        if response.status_code == 404 and should_fallback_to_spa(request.path):
+            return serve_spa(request)
+
+        return response
+
+
+class DevHttpsMiddleware:
+    """
+    In development, it's sometimes difficult to have django use HTTPS
+    when we have django behind nginx behind traefix.
+
+    We thus use a simple setting (in dev ONLY) to control that.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if settings.FORCE_HTTPS_URLS:
+            setattr(request.__class__, "scheme", "https")
+            setattr(
+                request,
+                "get_host",
+                lambda: request.__class__.get_host(request).replace(":80", ":443"),
+            )
+        return self.get_response(request)

api/funkwhale_api/common/migrations/0001_initial.py

@@ -3,8 +3,20 @@ from django.db import migrations
 from django.contrib.postgres.operations import UnaccentExtension
 
 
+class CustomUnaccentExtension(UnaccentExtension):
+    def database_forwards(self, app_label, schema_editor, from_state, to_state):
+        check_sql = "SELECT 1 FROM pg_extension WHERE extname = 'unaccent'"
+        with schema_editor.connection.cursor() as cursor:
+            cursor.execute(check_sql)
+            result = cursor.fetchall()
+
+        if result:
+            return
+        return super().database_forwards(app_label, schema_editor, from_state, to_state)
+
+
 class Migration(migrations.Migration):
 
     dependencies = []
 
-    operations = [UnaccentExtension()]
+    operations = [CustomUnaccentExtension()]

api/funkwhale_api/common/scripts/__init__.py

@@ -2,6 +2,7 @@ from . import create_actors
 from . import create_image_variations
 from . import django_permissions_to_user_permissions
 from . import migrate_to_user_libraries
+from . import delete_pre_017_federated_uploads
 from . import test
 
 
@@ -10,5 +11,6 @@ __all__ = [
     "create_image_variations",
     "django_permissions_to_user_permissions",
     "migrate_to_user_libraries",
+    "delete_pre_017_federated_uploads",
     "test",
 ]

api/funkwhale_api/common/scripts/delete_pre_017_federated_uploads.py

@@ -0,0 +1,14 @@
+"""
+Compute different sizes of image used for Album covers and User avatars
+"""
+
+from funkwhale_api.music.models import Upload
+
+
+def main(command, **kwargs):
+    queryset = Upload.objects.filter(
+        source__startswith="http", source__contains="/federation/music/file/"
+    ).exclude(source__contains="youtube")
+    total = queryset.count()
+    command.stdout.write("{} uploads found".format(total))
+    queryset.delete()

api/funkwhale_api/common/scripts/django_permissions_to_user_permissions.py

@@ -10,7 +10,6 @@ from funkwhale_api.users import models
 mapping = {
     "dynamic_preferences.change_globalpreferencemodel": "settings",
     "music.add_importbatch": "library",
-    "federation.change_library": "federation",
 }
 
 

api/funkwhale_api/common/serializers.py

@@ -1,8 +1,12 @@
 import collections
+import io
+import PIL
+import os
 
 from rest_framework import serializers
 
 from django.core.exceptions import ObjectDoesNotExist
+from django.core.files.uploadedfile import SimpleUploadedFile
 from django.utils.encoding import smart_text
 from django.utils.translation import ugettext_lazy as _
 
@@ -119,7 +123,7 @@ class ActionSerializer(serializers.Serializer):
         if type(value) in [list, tuple]:
             return self.queryset.filter(
                 **{"{}__in".format(self.pk_field): value}
-            ).order_by("id")
+            ).order_by(self.pk_field)
 
         raise serializers.ValidationError(
             "{} is not a valid value for objects. You must provide either a "
@@ -159,3 +163,56 @@ class ActionSerializer(serializers.Serializer):
             "result": result,
         }
         return payload
+
+
+def track_fields_for_update(*fields):
+    """
+    Apply this decorator to serializer to call function when specific values
+    are updated on an object:
+
+    .. code-block:: python
+
+        @track_fields_for_update('privacy_level')
+        class LibrarySerializer(serializers.ModelSerializer):
+            def on_updated_privacy_level(self, obj, old_value, new_value):
+                print('Do someting')
+    """
+
+    def decorator(serializer_class):
+        original_update = serializer_class.update
+
+        def new_update(self, obj, validated_data):
+            tracked_fields_before = {f: getattr(obj, f) for f in fields}
+            obj = original_update(self, obj, validated_data)
+            tracked_fields_after = {f: getattr(obj, f) for f in fields}
+
+            if tracked_fields_before != tracked_fields_after:
+                self.on_updated_fields(obj, tracked_fields_before, tracked_fields_after)
+            return obj
+
+        serializer_class.update = new_update
+        return serializer_class
+
+    return decorator
+
+
+class StripExifImageField(serializers.ImageField):
+    def to_internal_value(self, data):
+        file_obj = super().to_internal_value(data)
+
+        image = PIL.Image.open(file_obj)
+        data = list(image.getdata())
+        image_without_exif = PIL.Image.new(image.mode, image.size)
+        image_without_exif.putdata(data)
+
+        with io.BytesIO() as output:
+            image_without_exif.save(
+                output,
+                format=PIL.Image.EXTENSION[os.path.splitext(file_obj.name)[-1]],
+                quality=100,
+            )
+            content = output.getvalue()
+
+        return SimpleUploadedFile(
+            file_obj.name, content, content_type=file_obj.content_type
+        )

api/funkwhale_api/common/utils.py

@@ -3,10 +3,13 @@ from django.utils.deconstruct import deconstructible
 import os
 import shutil
 import uuid
+import xml.etree.ElementTree as ET
 
 from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
 
-from django.db import transaction
+from django.conf import settings
+from django import urls
+from django.db import models, transaction
 
 
 def rename_file(instance, field_name, new_name, allow_missing_file=False):
@@ -107,3 +110,40 @@ def chunk_queryset(source_qs, chunk_size):
 
         if nb_items < chunk_size:
             return
+
+
+def join_url(start, end):
+    if start.endswith("/") and end.startswith("/"):
+        return start + end[1:]
+
+    if not start.endswith("/") and not end.startswith("/"):
+        return start + "/" + end
+
+    return start + end
+
+
+def spa_reverse(name, args=[], kwargs={}):
+    return urls.reverse(name, urlconf=settings.SPA_URLCONF, args=args, kwargs=kwargs)
+
+
+def spa_resolve(path):
+    return urls.resolve(path, urlconf=settings.SPA_URLCONF)
+
+
+def parse_meta(html):
+    # dirty but this is only for testing so we don't really care,
+    # we convert the html string to xml so it can be parsed as xml
+    html = '<?xml version="1.0"?>' + html
+    tree = ET.fromstring(html)
+
+    meta = [elem for elem in tree.iter() if elem.tag in ["meta", "link"]]
+
+    return [dict([("tag", elem.tag)] + list(elem.items())) for elem in meta]
+
+
+def order_for_search(qs, field):
+    """
+    When searching, it's often more useful to have short results first,
+    this function will order the given qs based on the length of the given field
+    """
+    return qs.annotate(__size=models.functions.Length(field)).order_by("__size")

api/funkwhale_api/common/validators.py

@@ -1,6 +1,7 @@
 import mimetypes
 from os.path import splitext
 
+from django.core import validators
 from django.core.exceptions import ValidationError
 from django.core.files.images import get_image_dimensions
 from django.template.defaultfilters import filesizeformat
@@ -150,3 +151,17 @@ class FileValidator(object):
             }
 
             raise ValidationError(message)
+
+
+class DomainValidator(validators.URLValidator):
+    message = "Enter a valid domain name."
+
+    def __call__(self, value):
+        """
+        This is a bit hackish but since we don't have any built-in domain validator,
+        we use the url one, and prepend http:// in front of it.
+
+        If it fails, we know the domain is not valid.
+        """
+        super().__call__("http://{}".format(value))
+        return value

api/funkwhale_api/downloader/__init__.py

@@ -1,3 +0,0 @@
-from .downloader import download
-
-__all__ = ["download"]

api/funkwhale_api/downloader/downloader.py

@@ -1,19 +0,0 @@
-import os
-
-import youtube_dl
-from django.conf import settings
-
-
-def download(
-    url, target_directory=settings.MEDIA_ROOT, name="%(id)s.%(ext)s", bitrate=192
-):
-    target_path = os.path.join(target_directory, name)
-    ydl_opts = {
-        "quiet": True,
-        "outtmpl": target_path,
-        "postprocessors": [{"key": "FFmpegExtractAudio", "preferredcodec": "vorbis"}],
-    }
-    _downloader = youtube_dl.YoutubeDL(ydl_opts)
-    info = _downloader.extract_info(url)
-    info["audio_file_path"] = target_path % {"id": info["id"], "ext": "ogg"}
-    return info

api/funkwhale_api/factories.py

@@ -28,3 +28,14 @@ def ManyToManyFromList(field_name):
             field.add(*extracted)
 
     return inner
+
+
+class NoUpdateOnCreate:
+    """
+    Factory boy calls save after the initial create. In most case, this
+    is not needed, so we disable this behaviour
+    """
+
+    @classmethod
+    def _after_postgeneration(cls, instance, create, results=None):
+        return

api/funkwhale_api/favorites/factories.py

@@ -1,12 +1,12 @@
 import factory
 
-from funkwhale_api.factories import registry
+from funkwhale_api.factories import registry, NoUpdateOnCreate
 from funkwhale_api.music.factories import TrackFactory
 from funkwhale_api.users.factories import UserFactory
 
 
 @registry.register
-class TrackFavorite(factory.django.DjangoModelFactory):
+class TrackFavorite(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     track = factory.SubFactory(TrackFactory)
     user = factory.SubFactory(UserFactory)
 

api/funkwhale_api/favorites/views.py

@@ -1,5 +1,5 @@
 from rest_framework import mixins, status, viewsets
-from rest_framework.decorators import list_route
+from rest_framework.decorators import action
 from rest_framework.permissions import IsAuthenticatedOrReadOnly
 from rest_framework.response import Response
 
@@ -20,7 +20,7 @@ class TrackFavoriteViewSet(
     viewsets.GenericViewSet,
 ):
 
-    filter_class = filters.TrackFavoriteFilter
+    filterset_class = filters.TrackFavoriteFilter
     serializer_class = serializers.UserTrackFavoriteSerializer
     queryset = models.TrackFavorite.objects.all().select_related("user")
     permission_classes = [
@@ -51,7 +51,7 @@ class TrackFavoriteViewSet(
         queryset = queryset.filter(
             fields.privacy_level_query(self.request.user, "user__privacy_level")
         )
-        tracks = Track.objects.annotate_playable_by_actor(
+        tracks = Track.objects.with_playable_uploads(
             music_utils.get_actor_from_request(self.request)
         ).select_related("artist", "album__artist")
         queryset = queryset.prefetch_related(Prefetch("track", queryset=tracks))
@@ -62,7 +62,7 @@ class TrackFavoriteViewSet(
         favorite = models.TrackFavorite.add(track=track, user=self.request.user)
         return favorite
 
-    @list_route(methods=["delete", "post"])
+    @action(methods=["delete", "post"], detail=False)
     def remove(self, request, *args, **kwargs):
         try:
             pk = int(request.data["track"])
@@ -71,3 +71,19 @@ class TrackFavoriteViewSet(
             return Response({}, status=400)
         favorite.delete()
         return Response([], status=status.HTTP_204_NO_CONTENT)
+
+    @action(methods=["get"], detail=False)
+    def all(self, request, *args, **kwargs):
+        """
+        Return all the favorites of the current user, with only limited data
+        to have a performant endpoint and avoid lots of queries just to display
+        favorites status in the UI
+        """
+        if not request.user.is_authenticated:
+            return Response({"results": [], "count": 0}, status=200)
+
+        favorites = list(
+            request.user.track_favorites.values("id", "track").order_by("id")
+        )
+        payload = {"results": favorites, "count": len(favorites)}
+        return Response(payload, status=200)

api/funkwhale_api/federation/activity.py

@@ -1,6 +1,8 @@
 import uuid
 import logging
 
+from django.core.cache import cache
+from django.conf import settings
 from django.db import transaction, IntegrityError
 from django.db.models import Q
 
@@ -42,28 +44,68 @@ ACTIVITY_TYPES = [
     "View",
 ]
 
-
+FUNKWHALE_OBJECT_TYPES = [
-OBJECT_TYPES = [
+    ("Domain", "Domain"),
-    "Article",
+    ("Artist", "Artist"),
-    "Audio",
+    ("Album", "Album"),
-    "Collection",
+    ("Track", "Track"),
-    "Document",
+    ("Library", "Library"),
-    "Event",
+]
-    "Image",
+OBJECT_TYPES = (
-    "Note",
+    [
-    "OrderedCollection",
+        "Application",
-    "Page",
+        "Article",
-    "Place",
+        "Audio",
-    "Profile",
+        "Collection",
-    "Relationship",
+        "Document",
-    "Tombstone",
+        "Event",
-    "Video",
+        "Group",
-] + ACTIVITY_TYPES
+        "Image",
+        "Note",
+        "Object",
+        "OrderedCollection",
+        "Organization",
+        "Page",
+        "Person",
+        "Place",
+        "Profile",
+        "Relationship",
+        "Service",
+        "Tombstone",
+        "Video",
+    ]
+    + ACTIVITY_TYPES
+    + FUNKWHALE_OBJECT_TYPES
+)
 
 
 BROADCAST_TO_USER_ACTIVITIES = ["Follow", "Accept"]
 
 
+def should_reject(id, actor_id=None, payload={}):
+    from funkwhale_api.moderation import models as moderation_models
+
+    policies = moderation_models.InstancePolicy.objects.active()
+
+    media_types = ["Audio", "Artist", "Album", "Track", "Library", "Image"]
+    relevant_values = [
+        recursive_gettattr(payload, "type", permissive=True),
+        recursive_gettattr(payload, "object.type", permissive=True),
+        recursive_gettattr(payload, "target.type", permissive=True),
+    ]
+    # if one of the payload types match our internal media types, then
+    # we apply policies that reject media
+    if set(media_types) & set(relevant_values):
+        policy_type = Q(block_all=True) | Q(reject_media=True)
+    else:
+        policy_type = Q(block_all=True)
+
+    query = policies.matching_url_query(id) & policy_type
+    if actor_id:
+        query |= policies.matching_url_query(actor_id) & policy_type
+    return policies.filter(query).exists()
+
+
 @transaction.atomic
 def receive(activity, on_behalf_of):
     from . import models
@@ -76,6 +118,16 @@ def receive(activity, on_behalf_of):
         data=activity, context={"actor": on_behalf_of, "local_recipients": True}
     )
     serializer.is_valid(raise_exception=True)
+    if should_reject(
+        id=serializer.validated_data["id"],
+        actor_id=serializer.validated_data["actor"].fid,
+        payload=activity,
+    ):
+        logger.info(
+            "[federation] Discarding activity due to instance policies %s",
+            serializer.validated_data.get("id"),
+        )
+        return
     try:
         copy = serializer.save()
     except IntegrityError:
@@ -186,6 +238,21 @@ class InboxRouter(Router):
                 return
 
 
+ACTOR_KEY_ROTATION_LOCK_CACHE_KEY = "federation:actor-key-rotation-lock:{}"
+
+
+def should_rotate_actor_key(actor_id):
+    lock = cache.get(ACTOR_KEY_ROTATION_LOCK_CACHE_KEY.format(actor_id))
+    return lock is None
+
+
+def schedule_key_rotation(actor_id, delay):
+    from . import tasks
+
+    cache.set(ACTOR_KEY_ROTATION_LOCK_CACHE_KEY.format(actor_id), True, timeout=delay)
+    tasks.rotate_actor_key.apply_async(kwargs={"actor_id": actor_id}, countdown=delay)
+
+
 class OutboxRouter(Router):
     @transaction.atomic
     def dispatch(self, routing, context):
@@ -206,6 +273,15 @@ class OutboxRouter(Router):
                 # a route can yield zero, one or more activity payloads
                 if e:
                     activities_data.append(e)
+            deletions = [
+                a["actor"].id
+                for a in activities_data
+                if a["payload"]["type"] == "Delete"
+            ]
+            for actor_id in deletions:
+                # we way need to triggers a blind key rotation
+                if should_rotate_actor_key(actor_id):
+                    schedule_key_rotation(actor_id, settings.ACTOR_KEY_ROTATION_DELAY)
             inbox_items_by_activity_uuid = {}
             deliveries_by_activity_uuid = {}
             prepared_activities = []
@@ -267,7 +343,7 @@ class OutboxRouter(Router):
             return activities
 
 
-def recursive_gettattr(obj, key):
+def recursive_gettattr(obj, key, permissive=False):
     """
     Given a dictionary such as {'user': {'name': 'Bob'}} and
     a dotted string such as user.name, returns 'Bob'.
@@ -276,7 +352,12 @@ def recursive_gettattr(obj, key):
     """
     v = obj
     for k in key.split("."):
-        v = v.get(k)
+        try:
+            v = v.get(k)
+        except (TypeError, AttributeError):
+            if not permissive:
+                raise
+            return
         if v is None:
             return
 
@@ -386,15 +467,3 @@ def get_actors_from_audience(urls):
     if not final_query:
         return models.Actor.objects.none()
     return models.Actor.objects.filter(final_query)
-
-
-def get_inbox_urls(actor_queryset):
-    """
-    Given an actor queryset, returns a deduplicated set containing
-    all inbox or shared inbox urls where we should deliver our payloads for
-    those actors
-    """
-    values = actor_queryset.values("inbox_url", "shared_inbox_url")
-
-    urls = set([actor["shared_inbox_url"] or actor["inbox_url"] for actor in values])
-    return sorted(urls)

api/funkwhale_api/federation/actors.py

@@ -25,17 +25,18 @@ def get_actor_data(actor_url):
         raise ValueError("Invalid actor payload: {}".format(response.text))
 
 
-def get_actor(fid):
+def get_actor(fid, skip_cache=False):
-    try:
+    if not skip_cache:
-        actor = models.Actor.objects.get(fid=fid)
+        try:
-    except models.Actor.DoesNotExist:
+            actor = models.Actor.objects.get(fid=fid)
-        actor = None
+        except models.Actor.DoesNotExist:
-    fetch_delta = datetime.timedelta(
+            actor = None
-        minutes=preferences.get("federation__actor_fetch_delay")
+        fetch_delta = datetime.timedelta(
-    )
+            minutes=preferences.get("federation__actor_fetch_delay")
-    if actor and actor.last_fetch_date > timezone.now() - fetch_delta:
+        )
-        # cache is hot, we can return as is
+        if actor and actor.last_fetch_date > timezone.now() - fetch_delta:
-        return actor
+            # cache is hot, we can return as is
+            return actor
     data = get_actor_data(fid)
     serializer = serializers.ActorSerializer(data=data)
     serializer.is_valid(raise_exception=True)

api/funkwhale_api/federation/admin.py

@@ -24,6 +24,12 @@ def redeliver_activities(modeladmin, request, queryset):
 redeliver_activities.short_description = "Redeliver"
 
 
+@admin.register(models.Domain)
+class DomainAdmin(admin.ModelAdmin):
+    list_display = ["name", "creation_date"]
+    search_fields = ["name"]
+
+
 @admin.register(models.Activity)
 class ActivityAdmin(admin.ModelAdmin):
     list_display = ["type", "fid", "url", "actor", "creation_date"]

api/funkwhale_api/federation/api_views.py

@@ -13,6 +13,7 @@ from funkwhale_api.music import models as music_models
 
 from . import activity
 from . import api_serializers
+from . import exceptions
 from . import filters
 from . import models
 from . import routes
@@ -42,7 +43,7 @@ class LibraryFollowViewSet(
     )
     serializer_class = api_serializers.LibraryFollowSerializer
     permission_classes = [permissions.IsAuthenticated]
-    filter_class = filters.LibraryFollowFilter
+    filterset_class = filters.LibraryFollowFilter
     ordering_fields = ("creation_date",)
 
     def get_queryset(self):
@@ -65,7 +66,7 @@ class LibraryFollowViewSet(
         context["actor"] = self.request.user.actor
         return context
 
-    @decorators.detail_route(methods=["post"])
+    @decorators.action(methods=["post"], detail=True)
     def accept(self, request, *args, **kwargs):
         try:
             follow = self.queryset.get(
@@ -76,7 +77,7 @@ class LibraryFollowViewSet(
         update_follow(follow, approved=True)
         return response.Response(status=204)
 
-    @decorators.detail_route(methods=["post"])
+    @decorators.action(methods=["post"], detail=True)
     def reject(self, request, *args, **kwargs):
         try:
             follow = self.queryset.get(
@@ -104,7 +105,7 @@ class LibraryViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
         qs = super().get_queryset()
         return qs.viewable_by(actor=self.request.user.actor)
 
-    @decorators.detail_route(methods=["post"])
+    @decorators.action(methods=["post"], detail=True)
     def scan(self, request, *args, **kwargs):
         library = self.get_object()
         if library.actor.get_user():
@@ -121,18 +122,23 @@ class LibraryViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
             )
         return response.Response({"status": "skipped"}, 200)
 
-    @decorators.list_route(methods=["post"])
+    @decorators.action(methods=["post"], detail=False)
     def fetch(self, request, *args, **kwargs):
         try:
             fid = request.data["fid"]
         except KeyError:
             return response.Response({"fid": ["This field is required"]})
         try:
-            library = utils.retrieve(
+            library = utils.retrieve_ap_object(
                 fid,
                 queryset=self.queryset,
                 serializer_class=serializers.LibrarySerializer,
             )
+        except exceptions.BlockedActorOrDomain:
+            return response.Response(
+                {"detail": "This domain/account is blocked on your instance."},
+                status=400,
+            )
         except requests.exceptions.RequestException as e:
             return response.Response(
                 {"detail": "Error while fetching the library: {}".format(str(e))},
@@ -162,14 +168,14 @@ class InboxItemViewSet(
     )
     serializer_class = api_serializers.InboxItemSerializer
     permission_classes = [permissions.IsAuthenticated]
-    filter_class = filters.InboxItemFilter
+    filterset_class = filters.InboxItemFilter
     ordering_fields = ("activity__creation_date",)
 
     def get_queryset(self):
         qs = super().get_queryset()
         return qs.filter(actor=self.request.user.actor)
 
-    @decorators.list_route(methods=["post"])
+    @decorators.action(methods=["post"], detail=False)
     def action(self, request, *args, **kwargs):
         queryset = self.get_queryset()
         serializer = api_serializers.InboxItemActionSerializer(

api/funkwhale_api/federation/authentication.py

@@ -1,8 +1,14 @@
 import cryptography
-from django.contrib.auth.models import AnonymousUser
+import logging
-from rest_framework import authentication, exceptions
 
-from . import actors, keys, signing, utils
+from django.contrib.auth.models import AnonymousUser
+from rest_framework import authentication, exceptions as rest_exceptions
+
+from funkwhale_api.moderation import models as moderation_models
+from . import actors, exceptions, keys, signing, utils
+
+
+logger = logging.getLogger(__name__)
 
 
 class SignatureAuthentication(authentication.BaseAuthentication):
@@ -14,20 +20,42 @@ class SignatureAuthentication(authentication.BaseAuthentication):
         except KeyError:
             return
         except ValueError as e:
-            raise exceptions.AuthenticationFailed(str(e))
+            raise rest_exceptions.AuthenticationFailed(str(e))
 
         try:
-            actor = actors.get_actor(key_id.split("#")[0])
+            actor_url = key_id.split("#")[0]
+        except (TypeError, IndexError, AttributeError):
+            raise rest_exceptions.AuthenticationFailed("Invalid key id")
+
+        policies = (
+            moderation_models.InstancePolicy.objects.active()
+            .filter(block_all=True)
+            .matching_url(actor_url)
+        )
+        if policies.exists():
+            raise exceptions.BlockedActorOrDomain()
+
+        try:
+            actor = actors.get_actor(actor_url)
         except Exception as e:
-            raise exceptions.AuthenticationFailed(str(e))
+            logger.info(
+                "Discarding HTTP request from blocked actor/domain %s", actor_url
+            )
+            raise rest_exceptions.AuthenticationFailed(str(e))
 
         if not actor.public_key:
-            raise exceptions.AuthenticationFailed("No public key found")
+            raise rest_exceptions.AuthenticationFailed("No public key found")
 
         try:
             signing.verify_django(request, actor.public_key.encode("utf-8"))
         except cryptography.exceptions.InvalidSignature:
-            raise exceptions.AuthenticationFailed("Invalid signature")
+            # in case of invalid signature, we refetch the actor object
+            # to load a potentially new public key. This process is called
+            # Blind key rotation, and is described at
+            # https://blog.dereferenced.org/the-case-for-blind-key-rotation
+            # if signature verification fails after that, then we return a 403 error
+            actor = actors.get_actor(actor_url, skip_cache=True)
+            signing.verify_django(request, actor.public_key.encode("utf-8"))
 
         return actor
 

api/funkwhale_api/federation/exceptions.py

@@ -1,6 +1,13 @@
+from rest_framework import exceptions
+
+
 class MalformedPayload(ValueError):
     pass
 
 
 class MissingSignature(KeyError):
     pass
+
+
+class BlockedActorOrDomain(exceptions.AuthenticationFailed):
+    pass

api/funkwhale_api/federation/factories.py

@@ -7,7 +7,7 @@ from django.conf import settings
 from django.utils import timezone
 from django.utils.http import http_date
 
-from funkwhale_api.factories import registry
+from funkwhale_api.factories import registry, NoUpdateOnCreate
 from funkwhale_api.users import factories as user_factories
 
 from . import keys, models
@@ -67,24 +67,40 @@ def create_user(actor):
 
 
 @registry.register
-class ActorFactory(factory.DjangoModelFactory):
+class DomainFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    name = factory.Faker("domain_name")
+
+    class Meta:
+        model = "federation.Domain"
+        django_get_or_create = ("name",)
+
+
+@registry.register
+class ActorFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
     public_key = None
     private_key = None
     preferred_username = factory.Faker("user_name")
     summary = factory.Faker("paragraph")
-    domain = factory.Faker("domain_name")
+    domain = factory.SubFactory(DomainFactory)
     fid = factory.LazyAttribute(
-        lambda o: "https://{}/users/{}".format(o.domain, o.preferred_username)
+        lambda o: "https://{}/users/{}".format(o.domain.name, o.preferred_username)
     )
     followers_url = factory.LazyAttribute(
-        lambda o: "https://{}/users/{}followers".format(o.domain, o.preferred_username)
+        lambda o: "https://{}/users/{}followers".format(
+            o.domain.name, o.preferred_username
+        )
     )
     inbox_url = factory.LazyAttribute(
-        lambda o: "https://{}/users/{}/inbox".format(o.domain, o.preferred_username)
+        lambda o: "https://{}/users/{}/inbox".format(
+            o.domain.name, o.preferred_username
+        )
     )
     outbox_url = factory.LazyAttribute(
-        lambda o: "https://{}/users/{}/outbox".format(o.domain, o.preferred_username)
+        lambda o: "https://{}/users/{}/outbox".format(
+            o.domain.name, o.preferred_username
+        )
     )
+    keys = factory.LazyFunction(keys.get_key_pair)
 
     class Meta:
         model = models.Actor
@@ -95,7 +111,9 @@ class ActorFactory(factory.DjangoModelFactory):
             return
         from funkwhale_api.users.factories import UserFactory
 
-        self.domain = settings.FEDERATION_HOSTNAME
+        self.domain = models.Domain.objects.get_or_create(
+            name=settings.FEDERATION_HOSTNAME
+        )[0]
         self.save(update_fields=["domain"])
         if not create:
             if extracted and hasattr(extracted, "pk"):
@@ -108,19 +126,9 @@ class ActorFactory(factory.DjangoModelFactory):
         else:
             self.user = UserFactory(actor=self, **kwargs)
 
-    @factory.post_generation
-    def keys(self, create, extracted, **kwargs):
-        if not create:
-            # Simple build, do nothing.
-            return
-        if not extracted:
-            private, public = keys.get_key_pair()
-            self.private_key = private.decode("utf-8")
-            self.public_key = public.decode("utf-8")
-
 
 @registry.register
-class FollowFactory(factory.DjangoModelFactory):
+class FollowFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
     target = factory.SubFactory(ActorFactory)
     actor = factory.SubFactory(ActorFactory)
 
@@ -132,28 +140,23 @@ class FollowFactory(factory.DjangoModelFactory):
 
 
 @registry.register
-class MusicLibraryFactory(factory.django.DjangoModelFactory):
+class MusicLibraryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     actor = factory.SubFactory(ActorFactory)
     privacy_level = "me"
     name = factory.Faker("sentence")
     description = factory.Faker("sentence")
     uploads_count = 0
     fid = factory.Faker("federation_url")
+    followers_url = factory.LazyAttribute(
+        lambda o: o.fid + "/followers" if o.fid else None
+    )
 
     class Meta:
         model = "music.Library"
 
-    @factory.post_generation
-    def followers_url(self, create, extracted, **kwargs):
-        if not create:
-            # Simple build, do nothing.
-            return
-
-        self.followers_url = extracted or self.fid + "/followers"
-
 
 @registry.register
-class LibraryScan(factory.django.DjangoModelFactory):
+class LibraryScan(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     library = factory.SubFactory(MusicLibraryFactory)
     actor = factory.SubFactory(ActorFactory)
     total_files = factory.LazyAttribute(lambda o: o.library.uploads_count)
@@ -163,7 +166,7 @@ class LibraryScan(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class ActivityFactory(factory.django.DjangoModelFactory):
+class ActivityFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     actor = factory.SubFactory(ActorFactory)
     url = factory.Faker("federation_url")
     payload = factory.LazyFunction(lambda: {"type": "Create"})
@@ -173,7 +176,7 @@ class ActivityFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class InboxItemFactory(factory.django.DjangoModelFactory):
+class InboxItemFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     actor = factory.SubFactory(ActorFactory, local=True)
     activity = factory.SubFactory(ActivityFactory)
     type = "to"
@@ -183,7 +186,7 @@ class InboxItemFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class DeliveryFactory(factory.django.DjangoModelFactory):
+class DeliveryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     activity = factory.SubFactory(ActivityFactory)
     inbox_url = factory.Faker("url")
 
@@ -192,7 +195,7 @@ class DeliveryFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class LibraryFollowFactory(factory.DjangoModelFactory):
+class LibraryFollowFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
     target = factory.SubFactory(MusicLibraryFactory)
     actor = factory.SubFactory(ActorFactory)
 

api/funkwhale_api/federation/fields.py

@@ -0,0 +1,18 @@
+from rest_framework import serializers
+
+from . import models
+
+
+class ActorRelatedField(serializers.EmailField):
+    def to_representation(self, value):
+        return value.full_username
+
+    def to_internal_value(self, value):
+        value = super().to_internal_value(value)
+        username, domain = value.split("@")
+        try:
+            return models.Actor.objects.get(
+                preferred_username=username, domain_id=domain
+            )
+        except models.Actor.DoesNotExist:
+            raise serializers.ValidationError("Invalid actor name")

api/funkwhale_api/federation/keys.py

@@ -1,6 +1,8 @@
 import re
 import urllib.parse
 
+from django.conf import settings
+
 from cryptography.hazmat.backends import default_backend as crypto_default_backend
 from cryptography.hazmat.primitives import serialization as crypto_serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
@@ -8,7 +10,8 @@ from cryptography.hazmat.primitives.asymmetric import rsa
 KEY_ID_REGEX = re.compile(r"keyId=\"(?P<id>.*)\"")
 
 
-def get_key_pair(size=2048):
+def get_key_pair(size=None):
+    size = size or settings.RSA_KEY_SIZE
     key = rsa.generate_private_key(
         backend=crypto_default_backend(), public_exponent=65537, key_size=size
     )

api/funkwhale_api/federation/migrations/0013_auto_20181226_1935.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.0.9 on 2018-12-26 19:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0012_auto_20180920_1803")]
+
+    operations = [
+        migrations.AlterField(
+            model_name="actor",
+            name="private_key",
+            field=models.TextField(blank=True, max_length=5000, null=True),
+        ),
+        migrations.AlterField(
+            model_name="actor",
+            name="public_key",
+            field=models.TextField(blank=True, max_length=5000, null=True),
+        ),
+    ]

api/funkwhale_api/federation/migrations/0014_auto_20181205_0958.py

@@ -0,0 +1,46 @@
+# Generated by Django 2.0.9 on 2018-12-05 09:58
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0013_auto_20181226_1935")]
+
+    operations = [
+        migrations.CreateModel(
+            name="Domain",
+            fields=[
+                (
+                    "name",
+                    models.CharField(max_length=255, primary_key=True, serialize=False),
+                ),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+            ],
+        ),
+        migrations.AlterField(
+            model_name="actor",
+            name="domain",
+            field=models.CharField(max_length=1000, null=True),
+        ),
+        migrations.RenameField("actor", "domain", "old_domain"),
+        migrations.AddField(
+            model_name="actor",
+            name="domain",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="actors",
+                to="federation.Domain",
+            ),
+        ),
+        migrations.AlterUniqueTogether(name="actor", unique_together=set()),
+        migrations.AlterUniqueTogether(
+            name="actor", unique_together={("domain", "preferred_username")}
+        ),
+    ]

api/funkwhale_api/federation/migrations/0015_populate_domains.py

@@ -0,0 +1,56 @@
+# Generated by Django 2.0.9 on 2018-11-14 08:55
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+def populate_domains(apps, schema_editor):
+    Domain = apps.get_model("federation", "Domain")
+    Actor = apps.get_model("federation", "Actor")
+
+    domains = set(
+        [v.lower() for v in Actor.objects.values_list("old_domain", flat=True)]
+    )
+    for domain in sorted(domains):
+        print("Populating domain {}...".format(domain))
+        first_actor = (
+            Actor.objects.order_by("creation_date")
+            .exclude(creation_date=None)
+            .filter(old_domain__iexact=domain)
+            .first()
+        )
+
+        if first_actor:
+            first_seen = first_actor.creation_date
+        else:
+            first_seen = django.utils.timezone.now()
+
+        Domain.objects.update_or_create(
+            name=domain, defaults={"creation_date": first_seen}
+        )
+
+    for domain in Domain.objects.all():
+        Actor.objects.filter(old_domain__iexact=domain.name).update(domain=domain)
+
+
+def skip(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0014_auto_20181205_0958")]
+
+    operations = [
+        migrations.RunPython(populate_domains, skip),
+        migrations.AlterField(
+            model_name="actor",
+            name="domain",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="actors",
+                to="federation.Domain",
+            ),
+        ),
+    ]

api/funkwhale_api/federation/migrations/0016_auto_20181227_1605.py

@@ -0,0 +1,25 @@
+# Generated by Django 2.0.9 on 2018-12-27 16:05
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0015_populate_domains")]
+
+    operations = [
+        migrations.AddField(
+            model_name="domain",
+            name="nodeinfo",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                default=funkwhale_api.federation.models.empty_dict, max_length=50000
+            ),
+        ),
+        migrations.AddField(
+            model_name="domain",
+            name="nodeinfo_fetch_date",
+            field=models.DateTimeField(blank=True, default=None, null=True),
+        ),
+    ]

api/funkwhale_api/federation/models.py

@@ -13,6 +13,7 @@ from django.urls import reverse
 
 from funkwhale_api.common import session
 from funkwhale_api.common import utils as common_utils
+from funkwhale_api.common import validators as common_validators
 from funkwhale_api.music import utils as music_utils
 
 from . import utils as federation_utils
@@ -61,6 +62,83 @@ class ActorQuerySet(models.QuerySet):
 
         return qs
 
+    def with_uploads_count(self):
+        return self.annotate(
+            uploads_count=models.Count("libraries__uploads", distinct=True)
+        )
+
+
+class DomainQuerySet(models.QuerySet):
+    def external(self):
+        return self.exclude(pk=settings.FEDERATION_HOSTNAME)
+
+    def with_actors_count(self):
+        return self.annotate(actors_count=models.Count("actors", distinct=True))
+
+    def with_outbox_activities_count(self):
+        return self.annotate(
+            outbox_activities_count=models.Count(
+                "actors__outbox_activities", distinct=True
+            )
+        )
+
+
+class Domain(models.Model):
+    name = models.CharField(
+        primary_key=True,
+        max_length=255,
+        validators=[common_validators.DomainValidator()],
+    )
+    creation_date = models.DateTimeField(default=timezone.now)
+    nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True)
+    nodeinfo = JSONField(default=empty_dict, max_length=50000, blank=True)
+
+    objects = DomainQuerySet.as_manager()
+
+    def __str__(self):
+        return self.name
+
+    def save(self, **kwargs):
+        lowercase_fields = ["name"]
+        for field in lowercase_fields:
+            v = getattr(self, field, None)
+            if v:
+                setattr(self, field, v.lower())
+
+        super().save(**kwargs)
+
+    def get_stats(self):
+        from funkwhale_api.music import models as music_models
+
+        data = Domain.objects.filter(pk=self.pk).aggregate(
+            actors=models.Count("actors", distinct=True),
+            outbox_activities=models.Count("actors__outbox_activities", distinct=True),
+            libraries=models.Count("actors__libraries", distinct=True),
+            received_library_follows=models.Count(
+                "actors__libraries__received_follows", distinct=True
+            ),
+            emitted_library_follows=models.Count(
+                "actors__library_follows", distinct=True
+            ),
+        )
+        data["artists"] = music_models.Artist.objects.filter(
+            from_activity__actor__domain_id=self.pk
+        ).count()
+        data["albums"] = music_models.Album.objects.filter(
+            from_activity__actor__domain_id=self.pk
+        ).count()
+        data["tracks"] = music_models.Track.objects.filter(
+            from_activity__actor__domain_id=self.pk
+        ).count()
+
+        uploads = music_models.Upload.objects.filter(library__actor__domain_id=self.pk)
+        data["uploads"] = uploads.count()
+        data["media_total_size"] = uploads.aggregate(v=models.Sum("size"))["v"] or 0
+        data["media_downloaded_size"] = (
+            uploads.with_file().aggregate(v=models.Sum("size"))["v"] or 0
+        )
+        return data
+
 
 class Actor(models.Model):
     ap_type = "Actor"
@@ -74,7 +152,7 @@ class Actor(models.Model):
     shared_inbox_url = models.URLField(max_length=500, null=True, blank=True)
     type = models.CharField(choices=TYPE_CHOICES, default="Person", max_length=25)
     name = models.CharField(max_length=200, null=True, blank=True)
-    domain = models.CharField(max_length=1000)
+    domain = models.ForeignKey(Domain, on_delete=models.CASCADE, related_name="actors")
     summary = models.CharField(max_length=500, null=True, blank=True)
     preferred_username = models.CharField(max_length=200, null=True, blank=True)
     public_key = models.TextField(max_length=5000, null=True, blank=True)
@@ -105,41 +183,14 @@ class Actor(models.Model):
 
     @property
     def full_username(self):
-        return "{}@{}".format(self.preferred_username, self.domain)
+        return "{}@{}".format(self.preferred_username, self.domain_id)
 
     def __str__(self):
-        return "{}@{}".format(self.preferred_username, self.domain)
+        return "{}@{}".format(self.preferred_username, self.domain_id)
-
-    def save(self, **kwargs):
-        lowercase_fields = ["domain"]
-        for field in lowercase_fields:
-            v = getattr(self, field, None)
-            if v:
-                setattr(self, field, v.lower())
-
-        super().save(**kwargs)
 
     @property
     def is_local(self):
-        return self.domain == settings.FEDERATION_HOSTNAME
+        return self.domain_id == settings.FEDERATION_HOSTNAME
-
-    @property
-    def is_system(self):
-        from . import actors
-
-        return all(
-            [
-                settings.FEDERATION_HOSTNAME == self.domain,
-                self.preferred_username in actors.SYSTEM_ACTORS,
-            ]
-        )
-
-    @property
-    def system_conf(self):
-        from . import actors
-
-        if self.is_system:
-            return actors.SYSTEM_ACTORS[self.preferred_username]
 
     def get_approved_followers(self):
         follows = self.received_follows.filter(approved=True)
@@ -163,6 +214,44 @@ class Actor(models.Model):
         data["total"] = sum(data.values())
         return data
 
+    def get_stats(self):
+        from funkwhale_api.music import models as music_models
+
+        data = Actor.objects.filter(pk=self.pk).aggregate(
+            outbox_activities=models.Count("outbox_activities", distinct=True),
+            libraries=models.Count("libraries", distinct=True),
+            received_library_follows=models.Count(
+                "libraries__received_follows", distinct=True
+            ),
+            emitted_library_follows=models.Count("library_follows", distinct=True),
+        )
+        data["artists"] = music_models.Artist.objects.filter(
+            from_activity__actor=self.pk
+        ).count()
+        data["albums"] = music_models.Album.objects.filter(
+            from_activity__actor=self.pk
+        ).count()
+        data["tracks"] = music_models.Track.objects.filter(
+            from_activity__actor=self.pk
+        ).count()
+
+        uploads = music_models.Upload.objects.filter(library__actor=self.pk)
+        data["uploads"] = uploads.count()
+        data["media_total_size"] = uploads.aggregate(v=models.Sum("size"))["v"] or 0
+        data["media_downloaded_size"] = (
+            uploads.with_file().aggregate(v=models.Sum("size"))["v"] or 0
+        )
+        return data
+
+    @property
+    def keys(self):
+        return self.private_key, self.public_key
+
+    @keys.setter
+    def keys(self, v):
+        self.private_key = v[0].decode("utf-8")
+        self.public_key = v[1].decode("utf-8")
+
 
 class InboxItem(models.Model):
     """

api/funkwhale_api/federation/routes.py

@@ -82,7 +82,7 @@ def inbox_undo_follow(payload, context):
     serializer = serializers.UndoFollowSerializer(data=payload, context=context)
     if not serializer.is_valid(raise_exception=context.get("raise_exception", False)):
         logger.debug(
-            "Discarding invalid follow undo from {}: %s",
+            "Discarding invalid follow undo from %s: %s",
             context["actor"].fid,
             serializer.errors,
         )
@@ -195,6 +195,45 @@ def outbox_delete_library(context):
     }
 
 
+@outbox.register({"type": "Update", "object.type": "Library"})
+def outbox_update_library(context):
+    library = context["library"]
+    serializer = serializers.ActivitySerializer(
+        {"type": "Update", "object": serializers.LibrarySerializer(library).data}
+    )
+
+    yield {
+        "type": "Update",
+        "actor": library.actor,
+        "payload": with_recipients(
+            serializer.data, to=[{"type": "followers", "target": library}]
+        ),
+    }
+
+
+@inbox.register({"type": "Update", "object.type": "Library"})
+def inbox_update_library(payload, context):
+    actor = context["actor"]
+    library_id = payload["object"].get("id")
+    if not library_id:
+        logger.debug("Discarding deletion of empty library")
+        return
+
+    if not actor.libraries.filter(fid=library_id).exists():
+        logger.debug("Discarding deletion of unkwnown library %s", library_id)
+        return
+
+    serializer = serializers.LibrarySerializer(data=payload["object"])
+    if serializer.is_valid():
+        serializer.save()
+    else:
+        logger.debug(
+            "Discarding update of library %s because of payload errors: %s",
+            library_id,
+            serializer.errors,
+        )
+
+
 @inbox.register({"type": "Delete", "object.type": "Audio"})
 def inbox_delete_audio(payload, context):
     actor = context["actor"]

api/funkwhale_api/federation/serializers.py

@@ -114,7 +114,7 @@ class ActorSerializer(serializers.Serializer):
         if maf is not None:
             kwargs["manually_approves_followers"] = maf
         domain = urllib.parse.urlparse(kwargs["fid"]).netloc
-        kwargs["domain"] = domain
+        kwargs["domain"] = models.Domain.objects.get_or_create(pk=domain)[0]
         for endpoint, url in self.initial_data.get("endpoints", {}).items():
             if endpoint == "sharedInbox":
                 kwargs["shared_inbox_url"] = url
@@ -560,14 +560,14 @@ class LibrarySerializer(PaginatedCollectionSerializer):
         r = super().to_representation(conf)
         r["audience"] = (
             "https://www.w3.org/ns/activitystreams#Public"
-            if library.privacy_level == "public"
+            if library.privacy_level == "everyone"
             else ""
         )
         r["followers"] = library.followers_url
         return r
 
     def create(self, validated_data):
-        actor = utils.retrieve(
+        actor = utils.retrieve_ap_object(
             validated_data["actor"],
             queryset=models.Actor,
             serializer_class=ActorSerializer,
@@ -729,8 +729,11 @@ class AlbumSerializer(MusicEntitySerializer):
 
 class TrackSerializer(MusicEntitySerializer):
     position = serializers.IntegerField(min_value=0, allow_null=True, required=False)
+    disc = serializers.IntegerField(min_value=1, allow_null=True, required=False)
     artists = serializers.ListField(child=ArtistSerializer(), min_length=1)
     album = AlbumSerializer()
+    license = serializers.URLField(allow_null=True, required=False)
+    copyright = serializers.CharField(allow_null=True, required=False)
 
     def to_representation(self, instance):
         d = {
@@ -740,6 +743,11 @@ class TrackSerializer(MusicEntitySerializer):
             "published": instance.creation_date.isoformat(),
             "musicbrainzId": str(instance.mbid) if instance.mbid else None,
             "position": instance.position,
+            "disc": instance.disc_number,
+            "license": instance.local_license["identifiers"][0]
+            if instance.local_license
+            else None,
+            "copyright": instance.copyright if instance.copyright else None,
             "artists": [
                 ArtistSerializer(
                     instance.artist, context={"include_ap_context": False}
@@ -880,3 +888,12 @@ class CollectionSerializer(serializers.Serializer):
         if self.context.get("include_ap_context", True):
             d["@context"] = AP_CONTEXT
         return d
+
+
+class NodeInfoLinkSerializer(serializers.Serializer):
+    href = serializers.URLField()
+    rel = serializers.URLField()
+
+
+class NodeInfoSerializer(serializers.Serializer):
+    links = serializers.ListField(child=NodeInfoLinkSerializer(), min_length=1)

api/funkwhale_api/federation/signing.py

@@ -85,7 +85,7 @@ def verify_django(django_request, public_key):
 def get_auth(private_key, private_key_id):
     return requests_http_signature.HTTPSignatureAuth(
         use_auth_header=False,
-        headers=["(request-target)", "user-agent", "host", "date", "content-type"],
+        headers=["(request-target)", "user-agent", "host", "date"],
         algorithm="rsa-sha256",
         key=private_key.encode("utf-8"),
         key_id=private_key_id,

api/funkwhale_api/federation/tasks.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import os
+import requests
 
 from django.conf import settings
 from django.db.models import Q, F
@@ -13,7 +14,9 @@ from funkwhale_api.common import session
 from funkwhale_api.music import models as music_models
 from funkwhale_api.taskapp import celery
 
+from . import keys
 from . import models, signing
+from . import serializers
 from . import routes
 
 logger = logging.getLogger(__name__)
@@ -147,3 +150,92 @@ def deliver_to_remote(delivery):
         delivery.attempts = F("attempts") + 1
         delivery.is_delivered = True
         delivery.save(update_fields=["last_attempt_date", "attempts", "is_delivered"])
+
+
+def fetch_nodeinfo(domain_name):
+    s = session.get_session()
+    wellknown_url = "https://{}/.well-known/nodeinfo".format(domain_name)
+    response = s.get(
+        url=wellknown_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL
+    )
+    response.raise_for_status()
+    serializer = serializers.NodeInfoSerializer(data=response.json())
+    serializer.is_valid(raise_exception=True)
+    nodeinfo_url = None
+    for link in serializer.validated_data["links"]:
+        if link["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0":
+            nodeinfo_url = link["href"]
+            break
+
+    response = s.get(
+        url=nodeinfo_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL
+    )
+    response.raise_for_status()
+    return response.json()
+
+
+@celery.app.task(name="federation.update_domain_nodeinfo")
+@celery.require_instance(
+    models.Domain.objects.external(), "domain", id_kwarg_name="domain_name"
+)
+def update_domain_nodeinfo(domain):
+    now = timezone.now()
+    try:
+        nodeinfo = {"status": "ok", "payload": fetch_nodeinfo(domain.name)}
+    except (requests.RequestException, serializers.serializers.ValidationError) as e:
+        nodeinfo = {"status": "error", "error": str(e)}
+    domain.nodeinfo_fetch_date = now
+    domain.nodeinfo = nodeinfo
+    domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date"])
+
+
+def delete_qs(qs):
+    label = qs.model._meta.label
+    result = qs.delete()
+    related = sum(result[1].values())
+
+    logger.info(
+        "Purged %s %s objects (and %s related entities)", result[0], label, related
+    )
+
+
+def handle_purge_actors(ids, only=[]):
+    """
+    Empty only means we purge everything
+    Otherwise, we purge only the requested bits: media
+    """
+    # purge follows (received emitted)
+    if not only:
+        delete_qs(models.LibraryFollow.objects.filter(target__actor_id__in=ids))
+        delete_qs(models.Follow.objects.filter(actor_id__in=ids))
+
+    # purge audio content
+    if not only or "media" in only:
+        delete_qs(models.LibraryFollow.objects.filter(actor_id__in=ids))
+        delete_qs(models.Follow.objects.filter(target_id__in=ids))
+        delete_qs(music_models.Upload.objects.filter(library__actor_id__in=ids))
+        delete_qs(music_models.Library.objects.filter(actor_id__in=ids))
+
+    # purge remaining activities / deliveries
+    if not only:
+        delete_qs(models.InboxItem.objects.filter(actor_id__in=ids))
+        delete_qs(models.Activity.objects.filter(actor_id__in=ids))
+
+
+@celery.app.task(name="federation.purge_actors")
+def purge_actors(ids=[], domains=[], only=[]):
+    actors = models.Actor.objects.filter(
+        Q(id__in=ids) | Q(domain_id__in=domains)
+    ).order_by("id")
+    found_ids = list(actors.values_list("id", flat=True))
+    logger.info("Starting purging %s accounts", len(found_ids))
+    handle_purge_actors(ids=found_ids, only=only)
+
+
+@celery.app.task(name="federation.rotate_actor_key")
+@celery.require_instance(models.Actor.objects.local(), "actor")
+def rotate_actor_key(actor):
+    pair = keys.get_key_pair()
+    actor.private_key = pair[0].decode()
+    actor.public_key = pair[1].decode()
+    actor.save(update_fields=["private_key", "public_key"])

api/funkwhale_api/federation/utils.py

@@ -3,7 +3,9 @@ import re
 from django.conf import settings
 
 from funkwhale_api.common import session
+from funkwhale_api.moderation import models as moderation_models
 
+from . import exceptions
 from . import signing
 
 
@@ -58,7 +60,14 @@ def slugify_username(username):
     return re.sub(r"[-\s]+", "_", value)
 
 
-def retrieve(fid, actor=None, serializer_class=None, queryset=None):
+def retrieve_ap_object(
+    fid, actor=None, serializer_class=None, queryset=None, apply_instance_policies=True
+):
+    from . import activity
+
+    policies = moderation_models.InstancePolicy.objects.active().filter(block_all=True)
+    if apply_instance_policies and policies.matching_url(fid):
+        raise exceptions.BlockedActorOrDomain()
     if queryset:
         try:
             # queryset can also be a Model class
@@ -83,6 +92,16 @@ def retrieve(fid, actor=None, serializer_class=None, queryset=None):
     )
     response.raise_for_status()
     data = response.json()
+
+    # we match against moderation policies here again, because the FID of the returned
+    # object may not be the same as the URL used to access it
+    try:
+        id = data["id"]
+    except KeyError:
+        pass
+    else:
+        if apply_instance_policies and activity.should_reject(id=id, payload=data):
+            raise exceptions.BlockedActorOrDomain()
     if not serializer_class:
         return data
     serializer = serializer_class(data=data)

api/funkwhale_api/federation/views.py

@@ -3,7 +3,7 @@ from django.core import paginator
 from django.http import HttpResponse
 from django.urls import reverse
 from rest_framework import exceptions, mixins, response, viewsets
-from rest_framework.decorators import detail_route, list_route
+from rest_framework.decorators import action
 
 from funkwhale_api.common import preferences
 from funkwhale_api.music import models as music_models
@@ -23,7 +23,7 @@ class SharedViewSet(FederationMixin, viewsets.GenericViewSet):
     authentication_classes = [authentication.SignatureAuthentication]
     renderer_classes = [renderers.ActivityPubRenderer]
 
-    @list_route(methods=["post"])
+    @action(methods=["post"], detail=False)
     def inbox(self, request, *args, **kwargs):
         if request.method.lower() == "post" and request.actor is None:
             raise exceptions.AuthenticationFailed(
@@ -42,7 +42,7 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
     queryset = models.Actor.objects.local().select_related("user")
     serializer_class = serializers.ActorSerializer
 
-    @detail_route(methods=["get", "post"])
+    @action(methods=["get", "post"], detail=True)
     def inbox(self, request, *args, **kwargs):
         if request.method.lower() == "post" and request.actor is None:
             raise exceptions.AuthenticationFailed(
@@ -52,17 +52,17 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
             activity.receive(activity=request.data, on_behalf_of=request.actor)
         return response.Response({}, status=200)
 
-    @detail_route(methods=["get", "post"])
+    @action(methods=["get", "post"], detail=True)
     def outbox(self, request, *args, **kwargs):
         return response.Response({}, status=200)
 
-    @detail_route(methods=["get"])
+    @action(methods=["get"], detail=True)
     def followers(self, request, *args, **kwargs):
         self.get_object()
         # XXX to implement
         return response.Response({})
 
-    @detail_route(methods=["get"])
+    @action(methods=["get"], detail=True)
     def following(self, request, *args, **kwargs):
         self.get_object()
         # XXX to implement
@@ -74,7 +74,7 @@ class WellKnownViewSet(viewsets.GenericViewSet):
     permission_classes = []
     renderer_classes = [renderers.JSONRenderer, renderers.WebfingerRenderer]
 
-    @list_route(methods=["get"])
+    @action(methods=["get"], detail=False)
     def nodeinfo(self, request, *args, **kwargs):
         if not preferences.get("instance__nodeinfo_enabled"):
             return HttpResponse(status=404)
@@ -88,7 +88,7 @@ class WellKnownViewSet(viewsets.GenericViewSet):
         }
         return response.Response(data)
 
-    @list_route(methods=["get"])
+    @action(methods=["get"], detail=False)
     def webfinger(self, request, *args, **kwargs):
         if not preferences.get("federation__enabled"):
             return HttpResponse(status=405)
@@ -180,7 +180,7 @@ class MusicLibraryViewSet(
 
         return response.Response(data)
 
-    @detail_route(methods=["get"])
+    @action(methods=["get"], detail=True)
     def followers(self, request, *args, **kwargs):
         self.get_object()
         # XXX Implement this

api/funkwhale_api/history/factories.py

@@ -1,12 +1,12 @@
 import factory
 
-from funkwhale_api.factories import registry
+from funkwhale_api.factories import registry, NoUpdateOnCreate
 from funkwhale_api.music import factories
 from funkwhale_api.users.factories import UserFactory
 
 
 @registry.register
-class ListeningFactory(factory.django.DjangoModelFactory):
+class ListeningFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     user = factory.SubFactory(UserFactory)
     track = factory.SubFactory(factories.TrackFactory)
 

api/funkwhale_api/history/views.py

@@ -41,7 +41,7 @@ class ListeningViewSet(
         queryset = queryset.filter(
             fields.privacy_level_query(self.request.user, "user__privacy_level")
         )
-        tracks = Track.objects.annotate_playable_by_actor(
+        tracks = Track.objects.with_playable_uploads(
             music_utils.get_actor_from_request(self.request)
         ).select_related("artist", "album__artist")
         return queryset.prefetch_related(Prefetch("track", queryset=tracks))

api/funkwhale_api/instance/dynamic_preferences_registry.py

@@ -58,15 +58,6 @@ class RavenDSN(types.StringPreference):
     field_kwargs = {"required": False}
 
 
-@global_preferences_registry.register
-class RavenEnabled(types.BooleanPreference):
-    show_in_api = True
-    section = raven
-    name = "front_enabled"
-    default = False
-    verbose_name = "Report front-end errors with Raven"
-
-
 @global_preferences_registry.register
 class InstanceNodeinfoEnabled(types.BooleanPreference):
     show_in_api = False

api/funkwhale_api/instance/nodeinfo.py

@@ -17,7 +17,7 @@ def get():
         "protocols": ["activitypub"],
         "services": {"inbound": [], "outbound": []},
         "openRegistrations": preferences.get("users__registration_enabled"),
-        "usage": {"users": {"total": 0}},
+        "usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}},
         "metadata": {
             "private": preferences.get("instance__nodeinfo_private"),
             "shortDescription": preferences.get("instance__short_description"),
@@ -28,7 +28,7 @@ def get():
                 "federationNeedsApproval": preferences.get(
                     "federation__music_needs_approval"
                 ),
-                "anonymousCanListen": preferences.get(
+                "anonymousCanListen": not preferences.get(
                     "common__api_authentication_required"
                 ),
             },
@@ -37,7 +37,11 @@ def get():
     if share_stats:
         getter = memo(lambda: stats.get(), max_age=600)
         statistics = getter()
-        data["usage"]["users"]["total"] = statistics["users"]
+        data["usage"]["users"]["total"] = statistics["users"]["total"]
+        data["usage"]["users"]["activeHalfyear"] = statistics["users"][
+            "active_halfyear"
+        ]
+        data["usage"]["users"]["activeMonth"] = statistics["users"]["active_month"]
         data["metadata"]["library"]["tracks"] = {"total": statistics["tracks"]}
         data["metadata"]["library"]["artists"] = {"total": statistics["artists"]}
         data["metadata"]["library"]["albums"] = {"total": statistics["albums"]}

api/funkwhale_api/instance/stats.py

@@ -1,4 +1,7 @@
+import datetime
+
 from django.db.models import Sum
+from django.utils import timezone
 
 from funkwhale_api.favorites.models import TrackFavorite
 from funkwhale_api.history.models import Listening
@@ -19,6 +22,15 @@ def get():
 
 
 def get_users():
+    qs = User.objects.filter(is_active=True)
+    now = timezone.now()
+    active_month = now - datetime.timedelta(days=30)
+    active_halfyear = now - datetime.timedelta(days=30 * 6)
+    return {
+        "total": qs.count(),
+        "active_month": qs.filter(last_activity__gte=active_month).count(),
+        "active_halfyear": qs.filter(last_activity__gte=active_halfyear).count(),
+    }
     return User.objects.count()
 
 

api/funkwhale_api/manage/filters.py

@@ -1,6 +1,10 @@
 from django_filters import rest_framework as filters
 
 from funkwhale_api.common import fields
+from funkwhale_api.common import search
+
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.moderation import models as moderation_models
 from funkwhale_api.music import models as music_models
 from funkwhale_api.users import models as users_models
 
@@ -20,6 +24,41 @@ class ManageUploadFilterSet(filters.FilterSet):
         fields = ["q", "track__album", "track__artist", "track"]
 
 
+class ManageDomainFilterSet(filters.FilterSet):
+    q = fields.SearchFilter(search_fields=["name"])
+
+    class Meta:
+        model = federation_models.Domain
+        fields = ["name"]
+
+
+class ManageActorFilterSet(filters.FilterSet):
+    q = fields.SmartSearchFilter(
+        config=search.SearchConfig(
+            search_fields={
+                "name": {"to": "name"},
+                "username": {"to": "preferred_username"},
+                "email": {"to": "user__email"},
+                "bio": {"to": "summary"},
+                "type": {"to": "type"},
+            },
+            filter_fields={
+                "domain": {"to": "domain__name__iexact"},
+                "username": {"to": "preferred_username__iexact"},
+                "email": {"to": "user__email__iexact"},
+            },
+        )
+    )
+    local = filters.BooleanFilter(field_name="_", method="filter_local")
+
+    class Meta:
+        model = federation_models.Actor
+        fields = ["q", "domain", "type", "manually_approves_followers", "local"]
+
+    def filter_local(self, queryset, name, value):
+        return queryset.local(value)
+
+
 class ManageUserFilterSet(filters.FilterSet):
     q = fields.SearchFilter(search_fields=["username", "email", "name"])
 
@@ -31,10 +70,9 @@ class ManageUserFilterSet(filters.FilterSet):
             "privacy_level",
             "is_staff",
             "is_superuser",
-            "permission_upload",
             "permission_library",
             "permission_settings",
-            "permission_federation",
+            "permission_moderation",
         ]
 
 
@@ -50,3 +88,24 @@ class ManageInvitationFilterSet(filters.FilterSet):
         if value is None:
             return queryset
         return queryset.open(value)
+
+
+class ManageInstancePolicyFilterSet(filters.FilterSet):
+    q = fields.SearchFilter(
+        search_fields=[
+            "summary",
+            "target_domain__name",
+            "target_actor__username",
+            "target_actor__domain__name",
+        ]
+    )
+
+    class Meta:
+        model = moderation_models.InstancePolicy
+        fields = [
+            "q",
+            "block_all",
+            "silence_activity",
+            "silence_notifications",
+            "reject_media",
+        ]

api/funkwhale_api/manage/serializers.py

@@ -3,6 +3,11 @@ from django.db import transaction
 from rest_framework import serializers
 
 from funkwhale_api.common import serializers as common_serializers
+from funkwhale_api.common import utils as common_utils
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import fields as federation_fields
+from funkwhale_api.federation import tasks as federation_tasks
+from funkwhale_api.moderation import models as moderation_models
 from funkwhale_api.music import models as music_models
 from funkwhale_api.users import models as users_models
 
@@ -115,6 +120,7 @@ class ManageUserSerializer(serializers.ModelSerializer):
             "permissions",
             "privacy_level",
             "upload_quota",
+            "full_username",
         )
         read_only_fields = [
             "id",
@@ -168,3 +174,168 @@ class ManageInvitationActionSerializer(common_serializers.ActionSerializer):
     @transaction.atomic
     def handle_delete(self, objects):
         return objects.delete()
+
+
+class ManageDomainSerializer(serializers.ModelSerializer):
+    actors_count = serializers.SerializerMethodField()
+    outbox_activities_count = serializers.SerializerMethodField()
+
+    class Meta:
+        model = federation_models.Domain
+        fields = [
+            "name",
+            "creation_date",
+            "actors_count",
+            "outbox_activities_count",
+            "nodeinfo",
+            "nodeinfo_fetch_date",
+            "instance_policy",
+        ]
+        read_only_fields = [
+            "creation_date",
+            "instance_policy",
+            "nodeinfo",
+            "nodeinfo_fetch_date",
+        ]
+
+    def get_actors_count(self, o):
+        return getattr(o, "actors_count", 0)
+
+    def get_outbox_activities_count(self, o):
+        return getattr(o, "outbox_activities_count", 0)
+
+
+class ManageDomainActionSerializer(common_serializers.ActionSerializer):
+    actions = [common_serializers.Action("purge", allow_all=False)]
+    filterset_class = filters.ManageDomainFilterSet
+    pk_field = "name"
+
+    @transaction.atomic
+    def handle_purge(self, objects):
+        ids = objects.values_list("pk", flat=True)
+        common_utils.on_commit(federation_tasks.purge_actors.delay, domains=list(ids))
+
+
+class ManageActorSerializer(serializers.ModelSerializer):
+    uploads_count = serializers.SerializerMethodField()
+    user = ManageUserSerializer()
+
+    class Meta:
+        model = federation_models.Actor
+        fields = [
+            "id",
+            "url",
+            "fid",
+            "preferred_username",
+            "full_username",
+            "domain",
+            "name",
+            "summary",
+            "type",
+            "creation_date",
+            "last_fetch_date",
+            "inbox_url",
+            "outbox_url",
+            "shared_inbox_url",
+            "manually_approves_followers",
+            "uploads_count",
+            "user",
+            "instance_policy",
+        ]
+        read_only_fields = ["creation_date", "instance_policy"]
+
+    def get_uploads_count(self, o):
+        return getattr(o, "uploads_count", 0)
+
+
+class ManageActorActionSerializer(common_serializers.ActionSerializer):
+    actions = [common_serializers.Action("purge", allow_all=False)]
+    filterset_class = filters.ManageActorFilterSet
+
+    @transaction.atomic
+    def handle_purge(self, objects):
+        ids = objects.values_list("id", flat=True)
+        common_utils.on_commit(federation_tasks.purge_actors.delay, ids=list(ids))
+
+
+class TargetSerializer(serializers.Serializer):
+    type = serializers.ChoiceField(choices=["domain", "actor"])
+    id = serializers.CharField()
+
+    def to_representation(self, value):
+        if value["type"] == "domain":
+            return {"type": "domain", "id": value["obj"].name}
+        if value["type"] == "actor":
+            return {"type": "actor", "id": value["obj"].full_username}
+
+    def to_internal_value(self, value):
+        if value["type"] == "domain":
+            field = serializers.PrimaryKeyRelatedField(
+                queryset=federation_models.Domain.objects.external()
+            )
+        if value["type"] == "actor":
+            field = federation_fields.ActorRelatedField()
+        value["obj"] = field.to_internal_value(value["id"])
+        return value
+
+
+class ManageInstancePolicySerializer(serializers.ModelSerializer):
+    target = TargetSerializer()
+    actor = federation_fields.ActorRelatedField(read_only=True)
+
+    class Meta:
+        model = moderation_models.InstancePolicy
+        fields = [
+            "id",
+            "uuid",
+            "target",
+            "creation_date",
+            "actor",
+            "summary",
+            "is_active",
+            "block_all",
+            "silence_activity",
+            "silence_notifications",
+            "reject_media",
+        ]
+
+        read_only_fields = ["uuid", "id", "creation_date", "actor", "target"]
+
+    def validate(self, data):
+        try:
+            target = data.pop("target")
+        except KeyError:
+            # partial update
+            return data
+        if target["type"] == "domain":
+            data["target_domain"] = target["obj"]
+        if target["type"] == "actor":
+            data["target_actor"] = target["obj"]
+
+        return data
+
+    @transaction.atomic
+    def save(self, *args, **kwargs):
+        instance = super().save(*args, **kwargs)
+        need_purge = self.instance.is_active and (
+            self.instance.block_all or self.instance.reject_media
+        )
+        if need_purge:
+            only = []
+            if self.instance.reject_media:
+                only.append("media")
+            target = instance.target
+            if target["type"] == "domain":
+                common_utils.on_commit(
+                    federation_tasks.purge_actors.delay,
+                    domains=[target["obj"].pk],
+                    only=only,
+                )
+            if target["type"] == "actor":
+                common_utils.on_commit(
+                    federation_tasks.purge_actors.delay,
+                    ids=[target["obj"].pk],
+                    only=only,
+                )
+
+        return instance

api/funkwhale_api/manage/urls.py

@@ -3,13 +3,33 @@ from rest_framework import routers
 
 from . import views
 
+federation_router = routers.SimpleRouter()
+federation_router.register(r"domains", views.ManageDomainViewSet, "domains")
+
 library_router = routers.SimpleRouter()
 library_router.register(r"uploads", views.ManageUploadViewSet, "uploads")
+
+moderation_router = routers.SimpleRouter()
+moderation_router.register(
+    r"instance-policies", views.ManageInstancePolicyViewSet, "instance-policies"
+)
+
 users_router = routers.SimpleRouter()
 users_router.register(r"users", views.ManageUserViewSet, "users")
 users_router.register(r"invitations", views.ManageInvitationViewSet, "invitations")
 
+other_router = routers.SimpleRouter()
+other_router.register(r"accounts", views.ManageActorViewSet, "accounts")
+
 urlpatterns = [
+    url(
+        r"^federation/",
+        include((federation_router.urls, "federation"), namespace="federation"),
+    ),
     url(r"^library/", include((library_router.urls, "instance"), namespace="library")),
+    url(
+        r"^moderation/",
+        include((moderation_router.urls, "moderation"), namespace="moderation"),
+    ),
     url(r"^users/", include((users_router.urls, "instance"), namespace="users")),
-]
+] + other_router.urls

api/funkwhale_api/manage/views.py

@@ -1,8 +1,12 @@
 from rest_framework import mixins, response, viewsets
-from rest_framework.decorators import list_route
+from rest_framework import decorators as rest_decorators
+from django.shortcuts import get_object_or_404
 
-from funkwhale_api.common import preferences
+from funkwhale_api.common import preferences, decorators
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import tasks as federation_tasks
 from funkwhale_api.music import models as music_models
+from funkwhale_api.moderation import models as moderation_models
 from funkwhale_api.users import models as users_models
 from funkwhale_api.users.permissions import HasUserPermission
 
@@ -18,7 +22,7 @@ class ManageUploadViewSet(
         .order_by("-id")
     )
     serializer_class = serializers.ManageUploadSerializer
-    filter_class = filters.ManageUploadFilterSet
+    filterset_class = filters.ManageUploadFilterSet
     permission_classes = (HasUserPermission,)
     required_permissions = ["library"]
     ordering_fields = [
@@ -31,7 +35,7 @@ class ManageUploadViewSet(
         "duration",
     ]
 
-    @list_route(methods=["post"])
+    @rest_decorators.action(methods=["post"], detail=False)
     def action(self, request, *args, **kwargs):
         queryset = self.get_queryset()
         serializer = serializers.ManageUploadActionSerializer(
@@ -50,7 +54,7 @@ class ManageUserViewSet(
 ):
     queryset = users_models.User.objects.all().order_by("-id")
     serializer_class = serializers.ManageUserSerializer
-    filter_class = filters.ManageUserFilterSet
+    filterset_class = filters.ManageUserFilterSet
     permission_classes = (HasUserPermission,)
     required_permissions = ["settings"]
     ordering_fields = ["date_joined", "last_activity", "username"]
@@ -75,7 +79,7 @@ class ManageInvitationViewSet(
         .select_related("owner")
     )
     serializer_class = serializers.ManageInvitationSerializer
-    filter_class = filters.ManageInvitationFilterSet
+    filterset_class = filters.ManageInvitationFilterSet
     permission_classes = (HasUserPermission,)
     required_permissions = ["settings"]
     ordering_fields = ["creation_date", "expiration_date"]
@@ -83,7 +87,7 @@ class ManageInvitationViewSet(
     def perform_create(self, serializer):
         serializer.save(owner=self.request.user)
 
-    @list_route(methods=["post"])
+    @rest_decorators.action(methods=["post"], detail=False)
     def action(self, request, *args, **kwargs):
         queryset = self.get_queryset()
         serializer = serializers.ManageInvitationActionSerializer(
@@ -92,3 +96,112 @@ class ManageInvitationViewSet(
         serializer.is_valid(raise_exception=True)
         result = serializer.save()
         return response.Response(result, status=200)
+
+
+class ManageDomainViewSet(
+    mixins.CreateModelMixin,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_value_regex = r"[a-zA-Z0-9\-\.]+"
+    queryset = (
+        federation_models.Domain.objects.external()
+        .with_actors_count()
+        .with_outbox_activities_count()
+        .prefetch_related("instance_policy")
+        .order_by("name")
+    )
+    serializer_class = serializers.ManageDomainSerializer
+    filterset_class = filters.ManageDomainFilterSet
+    permission_classes = (HasUserPermission,)
+    required_permissions = ["moderation"]
+    ordering_fields = [
+        "name",
+        "creation_date",
+        "nodeinfo_fetch_date",
+        "actors_count",
+        "outbox_activities_count",
+        "instance_policy",
+    ]
+
+    @rest_decorators.action(methods=["get"], detail=True)
+    def nodeinfo(self, request, *args, **kwargs):
+        domain = self.get_object()
+        federation_tasks.update_domain_nodeinfo(domain_name=domain.name)
+        domain.refresh_from_db()
+        return response.Response(domain.nodeinfo, status=200)
+
+    @rest_decorators.action(methods=["get"], detail=True)
+    def stats(self, request, *args, **kwargs):
+        domain = self.get_object()
+        return response.Response(domain.get_stats(), status=200)
+
+    action = decorators.action_route(serializers.ManageDomainActionSerializer)
+
+
+class ManageActorViewSet(
+    mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+):
+    lookup_value_regex = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
+    queryset = (
+        federation_models.Actor.objects.all()
+        .with_uploads_count()
+        .order_by("-creation_date")
+        .select_related("user")
+        .prefetch_related("instance_policy")
+    )
+    serializer_class = serializers.ManageActorSerializer
+    filterset_class = filters.ManageActorFilterSet
+    permission_classes = (HasUserPermission,)
+    required_permissions = ["moderation"]
+    ordering_fields = [
+        "name",
+        "preferred_username",
+        "domain",
+        "fid",
+        "creation_date",
+        "last_fetch_date",
+        "uploads_count",
+        "outbox_activities_count",
+        "instance_policy",
+    ]
+
+    def get_object(self):
+        queryset = self.filter_queryset(self.get_queryset())
+        username, domain = self.kwargs["pk"].split("@")
+        filter_kwargs = {"domain_id": domain, "preferred_username": username}
+        obj = get_object_or_404(queryset, **filter_kwargs)
+        self.check_object_permissions(self.request, obj)
+
+        return obj
+
+    @rest_decorators.action(methods=["get"], detail=True)
+    def stats(self, request, *args, **kwargs):
+        domain = self.get_object()
+        return response.Response(domain.get_stats(), status=200)
+
+    action = decorators.action_route(serializers.ManageActorActionSerializer)
+
+
+class ManageInstancePolicyViewSet(
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    mixins.CreateModelMixin,
+    mixins.UpdateModelMixin,
+    viewsets.GenericViewSet,
+):
+    queryset = (
+        moderation_models.InstancePolicy.objects.all()
+        .order_by("-creation_date")
+        .select_related()
+    )
+    serializer_class = serializers.ManageInstancePolicySerializer
+    filterset_class = filters.ManageInstancePolicyFilterSet
+    permission_classes = (HasUserPermission,)
+    required_permissions = ["moderation"]
+    ordering_fields = ["id", "creation_date"]
+
+    def perform_create(self, serializer):
+        serializer.save(actor=self.request.user.actor)

api/funkwhale_api/moderation/admin.py

@@ -0,0 +1,30 @@
+from funkwhale_api.common import admin
+
+from . import models
+
+
+@admin.register(models.InstancePolicy)
+class InstancePolicyAdmin(admin.ModelAdmin):
+    list_display = [
+        "actor",
+        "target_domain",
+        "target_actor",
+        "creation_date",
+        "block_all",
+        "reject_media",
+        "silence_activity",
+        "silence_notifications",
+    ]
+    list_filter = [
+        "block_all",
+        "reject_media",
+        "silence_activity",
+        "silence_notifications",
+    ]
+    search_fields = [
+        "actor__fid",
+        "target_domain__name",
+        "target_domain__actor__fid",
+        "summary",
+    ]
+    list_select_related = True

api/funkwhale_api/moderation/factories.py

@@ -0,0 +1,23 @@
+import factory
+
+from funkwhale_api.factories import registry, NoUpdateOnCreate
+from funkwhale_api.federation import factories as federation_factories
+
+
+@registry.register
+class InstancePolicyFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
+    summary = factory.Faker("paragraph")
+    actor = factory.SubFactory(federation_factories.ActorFactory)
+    block_all = True
+    is_active = True
+
+    class Meta:
+        model = "moderation.InstancePolicy"
+
+    class Params:
+        for_domain = factory.Trait(
+            target_domain=factory.SubFactory(federation_factories.DomainFactory)
+        )
+        for_actor = factory.Trait(
+            target_actor=factory.SubFactory(federation_factories.ActorFactory)
+        )

api/funkwhale_api/moderation/migrations/0001_initial.py

@@ -0,0 +1,35 @@
+# Generated by Django 2.0.9 on 2019-01-07 06:06
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('federation', '0016_auto_20181227_1605'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='InstancePolicy',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('is_active', models.BooleanField(default=True)),
+                ('summary', models.TextField(blank=True, max_length=10000, null=True)),
+                ('block_all', models.BooleanField(default=False)),
+                ('silence_activity', models.BooleanField(default=False)),
+                ('silence_notifications', models.BooleanField(default=False)),
+                ('reject_media', models.BooleanField(default=False)),
+                ('actor', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='created_instance_policies', to='federation.Actor')),
+                ('target_actor', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='instance_policy', to='federation.Actor')),
+                ('target_domain', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='instance_policy', to='federation.Domain')),
+            ],
+        ),
+    ]

api/funkwhale_api/moderation/models.py

@@ -0,0 +1,75 @@
+import urllib.parse
+import uuid
+
+from django.db import models
+from django.utils import timezone
+
+
+class InstancePolicyQuerySet(models.QuerySet):
+    def active(self):
+        return self.filter(is_active=True)
+
+    def matching_url(self, *urls):
+        if not urls:
+            return self.none()
+        query = None
+        for url in urls:
+            new_query = self.matching_url_query(url)
+            if query:
+                query = query | new_query
+            else:
+                query = new_query
+        return self.filter(query)
+
+    def matching_url_query(self, url):
+        parsed = urllib.parse.urlparse(url)
+        return models.Q(target_domain_id=parsed.hostname) | models.Q(
+            target_actor__fid=url
+        )
+
+
+class InstancePolicy(models.Model):
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    actor = models.ForeignKey(
+        "federation.Actor",
+        related_name="created_instance_policies",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+    )
+    target_domain = models.OneToOneField(
+        "federation.Domain",
+        related_name="instance_policy",
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+    )
+    target_actor = models.OneToOneField(
+        "federation.Actor",
+        related_name="instance_policy",
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+    )
+    creation_date = models.DateTimeField(default=timezone.now)
+
+    is_active = models.BooleanField(default=True)
+    # a summary explaining why the policy is in place
+    summary = models.TextField(max_length=10000, null=True, blank=True)
+    # either block everything (simpler, but less granularity)
+    block_all = models.BooleanField(default=False)
+    # or pick individual restrictions below
+    # do not show in timelines/notifications, except for actual followers
+    silence_activity = models.BooleanField(default=False)
+    silence_notifications = models.BooleanField(default=False)
+    # do not download any media from the target
+    reject_media = models.BooleanField(default=False)
+
+    objects = InstancePolicyQuerySet.as_manager()
+
+    @property
+    def target(self):
+        if self.target_actor:
+            return {"type": "actor", "obj": self.target_actor}
+        if self.target_domain_id:
+            return {"type": "domain", "obj": self.target_domain}

api/funkwhale_api/music/admin.py

@@ -78,6 +78,28 @@ class UploadAdmin(admin.ModelAdmin):
     list_filter = ["mimetype", "import_status", "library__privacy_level"]
 
 
+@admin.register(models.UploadVersion)
+class UploadVersionAdmin(admin.ModelAdmin):
+    list_display = [
+        "upload",
+        "audio_file",
+        "mimetype",
+        "size",
+        "bitrate",
+        "creation_date",
+        "accessed_date",
+    ]
+    list_select_related = ["upload"]
+    search_fields = [
+        "upload__source",
+        "upload__acoustid_track_id",
+        "upload__track__title",
+        "upload__track__album__title",
+        "upload__track__artist__name",
+    ]
+    list_filter = ["mimetype"]
+
+
 def launch_scan(modeladmin, request, queryset):
     for library in queryset:
         library.schedule_scan(actor=request.user.actor, force=True)

api/funkwhale_api/music/dynamic_preferences_registry.py

@@ -0,0 +1,34 @@
+from dynamic_preferences import types
+from dynamic_preferences.registries import global_preferences_registry
+
+music = types.Section("music")
+
+
+@global_preferences_registry.register
+class MaxTracks(types.BooleanPreference):
+    show_in_api = True
+    section = music
+    name = "transcoding_enabled"
+    verbose_name = "Transcoding enabled"
+    help_text = (
+        "Enable transcoding of audio files in formats requested by the client. "
+        "This is especially useful for devices that do not support formats "
+        "such as Flac or Ogg, but the transcoding process will increase the "
+        "load on the server."
+    )
+    default = True
+
+
+@global_preferences_registry.register
+class MusicCacheDuration(types.IntPreference):
+    show_in_api = True
+    section = music
+    name = "transcoding_cache_duration"
+    default = 60 * 24 * 7
+    verbose_name = "Transcoding cache duration"
+    help_text = (
+        "How much minutes do you want to keep a copy of transcoded tracks "
+        "on the server? Transcoded files that were not listened in this interval "
+        "will be erased and retranscoded on the next listening."
+    )
+    field_kwargs = {"required": False}

api/funkwhale_api/music/factories.py

@@ -2,10 +2,11 @@ import os
 
 import factory
 
-from funkwhale_api.factories import ManyToManyFromList, registry
+from funkwhale_api.factories import ManyToManyFromList, registry, NoUpdateOnCreate
-from funkwhale_api.federation import factories as federation_factories
-from funkwhale_api.users import factories as users_factories
 
+from funkwhale_api.federation import factories as federation_factories
+from funkwhale_api.music import licenses
+from funkwhale_api.users import factories as users_factories
 
 SAMPLES_PATH = os.path.join(
     os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
@@ -30,8 +31,31 @@ def playable_factory(field):
     return inner
 
 
+def deduce_from_conf(field):
+    @factory.lazy_attribute
+    def inner(self):
+        return licenses.LICENSES_BY_ID[self.code][field]
+
+    return inner
+
+
 @registry.register
-class ArtistFactory(factory.django.DjangoModelFactory):
+class LicenseFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    code = "cc-by-4.0"
+    url = deduce_from_conf("url")
+    commercial = deduce_from_conf("commercial")
+    redistribute = deduce_from_conf("redistribute")
+    copyleft = deduce_from_conf("copyleft")
+    attribution = deduce_from_conf("attribution")
+    derivative = deduce_from_conf("derivative")
+
+    class Meta:
+        model = "music.License"
+        django_get_or_create = ("code",)
+
+
+@registry.register
+class ArtistFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     name = factory.Faker("name")
     mbid = factory.Faker("uuid4")
     fid = factory.Faker("federation_url")
@@ -42,7 +66,7 @@ class ArtistFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class AlbumFactory(factory.django.DjangoModelFactory):
+class AlbumFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     title = factory.Faker("sentence", nb_words=3)
     mbid = factory.Faker("uuid4")
     release_date = factory.Faker("date_object")
@@ -57,7 +81,7 @@ class AlbumFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class TrackFactory(factory.django.DjangoModelFactory):
+class TrackFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     fid = factory.Faker("federation_url")
     title = factory.Faker("sentence", nb_words=3)
     mbid = factory.Faker("uuid4")
@@ -70,9 +94,18 @@ class TrackFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = "music.Track"
 
+    @factory.post_generation
+    def license(self, created, extracted, **kwargs):
+        if not created:
+            return
+
+        if extracted:
+            self.license = LicenseFactory(code=extracted)
+            self.save()
+
 
 @registry.register
-class UploadFactory(factory.django.DjangoModelFactory):
+class UploadFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     fid = factory.Faker("federation_url")
     track = factory.SubFactory(TrackFactory)
     library = factory.SubFactory(federation_factories.MusicLibraryFactory)
@@ -89,14 +122,26 @@ class UploadFactory(factory.django.DjangoModelFactory):
         model = "music.Upload"
 
     class Params:
-        in_place = factory.Trait(audio_file=None)
+        in_place = factory.Trait(audio_file=None, mimetype=None)
         playable = factory.Trait(
             import_status="finished", library__privacy_level="everyone"
         )
 
 
 @registry.register
-class WorkFactory(factory.django.DjangoModelFactory):
+class UploadVersionFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    upload = factory.SubFactory(UploadFactory, bitrate=200000)
+    bitrate = factory.SelfAttribute("upload.bitrate")
+    mimetype = "audio/mpeg"
+    audio_file = factory.django.FileField()
+    size = 2000000
+
+    class Meta:
+        model = "music.UploadVersion"
+
+
+@registry.register
+class WorkFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     mbid = factory.Faker("uuid4")
     language = "eng"
     nature = "song"
@@ -107,7 +152,7 @@ class WorkFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class LyricsFactory(factory.django.DjangoModelFactory):
+class LyricsFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     work = factory.SubFactory(WorkFactory)
     url = factory.Faker("url")
     content = factory.Faker("paragraphs", nb=4)
@@ -117,7 +162,7 @@ class LyricsFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class TagFactory(factory.django.DjangoModelFactory):
+class TagFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     name = factory.SelfAttribute("slug")
     slug = factory.Faker("slug")
 
@@ -128,7 +173,7 @@ class TagFactory(factory.django.DjangoModelFactory):
 # XXX To remove
 
 
-class ImportBatchFactory(factory.django.DjangoModelFactory):
+class ImportBatchFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     submitted_by = factory.SubFactory(users_factories.UserFactory)
 
     class Meta:
@@ -136,7 +181,7 @@ class ImportBatchFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class ImportJobFactory(factory.django.DjangoModelFactory):
+class ImportJobFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     batch = factory.SubFactory(ImportBatchFactory)
     source = factory.Faker("url")
     mbid = factory.Faker("uuid4")

api/funkwhale_api/music/filters.py

@@ -9,7 +9,7 @@ from . import utils
 
 class ArtistFilter(filters.FilterSet):
     q = fields.SearchFilter(search_fields=["name"])
-    playable = filters.BooleanFilter(name="_", method="filter_playable")
+    playable = filters.BooleanFilter(field_name="_", method="filter_playable")
 
     class Meta:
         model = models.Artist
@@ -25,7 +25,7 @@ class ArtistFilter(filters.FilterSet):
 
 class TrackFilter(filters.FilterSet):
     q = fields.SearchFilter(search_fields=["title", "album__title", "artist__name"])
-    playable = filters.BooleanFilter(name="_", method="filter_playable")
+    playable = filters.BooleanFilter(field_name="_", method="filter_playable")
 
     class Meta:
         model = models.Track
@@ -34,6 +34,7 @@ class TrackFilter(filters.FilterSet):
             "playable": ["exact"],
             "artist": ["exact"],
             "album": ["exact"],
+            "license": ["exact"],
         }
 
     def filter_playable(self, queryset, name, value):
@@ -47,7 +48,7 @@ class UploadFilter(filters.FilterSet):
     track_artist = filters.UUIDFilter("track__artist__uuid")
     album_artist = filters.UUIDFilter("track__album__artist__uuid")
     library = filters.UUIDFilter("library__uuid")
-    playable = filters.BooleanFilter(name="_", method="filter_playable")
+    playable = filters.BooleanFilter(field_name="_", method="filter_playable")
     q = fields.SmartSearchFilter(
         config=search.SearchConfig(
             search_fields={
@@ -85,7 +86,7 @@ class UploadFilter(filters.FilterSet):
 
 
 class AlbumFilter(filters.FilterSet):
-    playable = filters.BooleanFilter(name="_", method="filter_playable")
+    playable = filters.BooleanFilter(field_name="_", method="filter_playable")
     q = fields.SearchFilter(search_fields=["title", "artist__name" "source"])
 
     class Meta:

api/funkwhale_api/music/licenses.py

@@ -0,0 +1,363 @@
+import logging
+import re
+
+from django.db import transaction
+
+from . import models
+
+logger = logging.getLogger(__name__)
+
+MODEL_FIELDS = [
+    "redistribute",
+    "derivative",
+    "attribution",
+    "copyleft",
+    "commercial",
+    "url",
+]
+
+
+@transaction.atomic
+def load(data):
+    """
+    Load/update database objects with our hardcoded data
+    """
+    existing = models.License.objects.all()
+    existing_by_code = {e.code: e for e in existing}
+    to_create = []
+
+    for row in data:
+        try:
+            license = existing_by_code[row["code"]]
+        except KeyError:
+            logger.info("Loading new license: {}".format(row["code"]))
+            to_create.append(
+                models.License(code=row["code"], **{f: row[f] for f in MODEL_FIELDS})
+            )
+        else:
+            logger.info("Updating license: {}".format(row["code"]))
+            stored = [getattr(license, f) for f in MODEL_FIELDS]
+            wanted = [row[f] for f in MODEL_FIELDS]
+            if wanted == stored:
+                continue
+            # the object in database needs an update
+            for f in MODEL_FIELDS:
+                setattr(license, f, row[f])
+
+            license.save()
+
+    models.License.objects.bulk_create(to_create)
+    return sorted(models.License.objects.all(), key=lambda o: o.code)
+
+
+_cache = None
+
+
+def match(*values):
+    """
+    Given a string, extracted from music file tags, return corresponding License
+    instance, if found
+    """
+    global _cache
+    for value in values:
+        if not value:
+            continue
+
+        # we are looking for the first url in our value
+        # This regex is not perfect, but it's good enough for now
+        urls = re.findall(
+            r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+",
+            value,
+        )
+        if not urls:
+            logger.debug('Impossible to guess license from string "{}"'.format(value))
+            continue
+        url = urls[0]
+        if _cache:
+            existing = _cache
+        else:
+            existing = load(LICENSES)
+            _cache = existing
+        for license in existing:
+            if license.conf is None:
+                continue
+            for i in license.conf["identifiers"]:
+                if match_urls(url, i):
+                    return license
+
+
+def match_urls(*urls):
+    """
+    We want to ensure the two url match but don't care for protocol
+    or trailing slashes
+    """
+    urls = [u.rstrip("/") for u in urls]
+    urls = [u.lstrip("http://") for u in urls]
+    urls = [u.lstrip("https://") for u in urls]
+    return len(set(urls)) == 1
+
+
+def get_cc_license(version, perks, country=None, country_name=None):
+    if len(perks) == 0:
+        raise ValueError("No perks!")
+
+    url_template = "//creativecommons.org/licenses/{type}/{version}/"
+
+    code_parts = []
+    name_parts = []
+    perks_data = [
+        ("by", "Attribution"),
+        ("nc", "NonCommercial"),
+        ("sa", "ShareAlike"),
+        ("nd", "NoDerivatives"),
+    ]
+    for perk, name in perks_data:
+        if perk in perks:
+            code_parts.append(perk)
+            name_parts.append(name)
+    url = url_template.format(version=version, type="-".join(code_parts))
+    code_parts.append(version)
+    name = "Creative commons - {perks} {version}".format(
+        perks="-".join(name_parts), version=version
+    )
+    if country:
+        code_parts.append(country)
+        name += " {}".format(country_name)
+        url += country + "/"
+    data = {
+        "name": name,
+        "code": "cc-{}".format("-".join(code_parts)),
+        "redistribute": True,
+        "commercial": "nc" not in perks,
+        "derivative": "nd" not in perks,
+        "copyleft": "sa" in perks,
+        "attribution": "by" in perks,
+        "url": "https:" + url,
+        "identifiers": ["http:" + url],
+    }
+
+    return data
+
+
+COUNTRIES = {
+    "ar": "Argentina",
+    "au": "Australia",
+    "at": "Austria",
+    "be": "Belgium",
+    "br": "Brazil",
+    "bg": "Bulgaria",
+    "ca": "Canada",
+    "cl": "Chile",
+    "cn": "China Mainland",
+    "co": "Colombia",
+    "cr": "Costa Rica",
+    "hr": "Croatia",
+    "cz": "Czech Republic",
+    "dk": "Denmark",
+    "ec": "Ecuador",
+    "eg": "Egypt",
+    "ee": "Estonia",
+    "fi": "Finland",
+    "fr": "France",
+    "de": "Germany",
+    "gr": "Greece",
+    "gt": "Guatemala",
+    "hk": "Hong Kong",
+    "hu": "Hungary",
+    "igo": "IGO",
+    "in": "India",
+    "ie": "Ireland",
+    "il": "Israel",
+    "it": "Italy",
+    "jp": "Japan",
+    "lu": "Luxembourg",
+    "mk": "Macedonia",
+    "my": "Malaysia",
+    "mt": "Malta",
+    "mx": "Mexico",
+    "nl": "Netherlands",
+    "nz": "New Zealand",
+    "no": "Norway",
+    "pe": "Peru",
+    "ph": "Philippines",
+    "pl": "Poland",
+    "pt": "Portugal",
+    "pr": "Puerto Rico",
+    "ro": "Romania",
+    "rs": "Serbia",
+    "sg": "Singapore",
+    "si": "Slovenia",
+    "za": "South Africa",
+    "kr": "South Korea",
+    "es": "Spain",
+    "se": "Sweden",
+    "ch": "Switzerland",
+    "tw": "Taiwan",
+    "th": "Thailand",
+    "uk": "UK: England & Wales",
+    "scotland": "UK: Scotland",
+    "ug": "Uganda",
+    "us": "United States",
+    "ve": "Venezuela",
+    "vn": "Vietnam",
+}
+CC_30_COUNTRIES = [
+    "at",
+    "au",
+    "br",
+    "ch",
+    "cl",
+    "cn",
+    "cr",
+    "cz",
+    "de",
+    "ec",
+    "ee",
+    "eg",
+    "es",
+    "fr",
+    "gr",
+    "gt",
+    "hk",
+    "hr",
+    "ie",
+    "igo",
+    "it",
+    "lu",
+    "nl",
+    "no",
+    "nz",
+    "ph",
+    "pl",
+    "pr",
+    "pt",
+    "ro",
+    "rs",
+    "sg",
+    "th",
+    "tw",
+    "ug",
+    "us",
+    "ve",
+    "vn",
+    "za",
+]
+
+CC_25_COUNTRIES = [
+    "ar",
+    "bg",
+    "ca",
+    "co",
+    "dk",
+    "hu",
+    "il",
+    "in",
+    "mk",
+    "mt",
+    "mx",
+    "my",
+    "pe",
+    "scotland",
+]
+
+LICENSES = [
+    # a non-exhaustive list: http://musique-libre.org/doc/le-tableau-des-licences-libres-et-ouvertes-de-dogmazic/
+    {
+        "code": "cc0-1.0",
+        "name": "CC0 - Public domain",
+        "redistribute": True,
+        "derivative": True,
+        "commercial": True,
+        "attribution": False,
+        "copyleft": False,
+        "url": "https://creativecommons.org/publicdomain/zero/1.0/",
+        "identifiers": [
+            # note the http here.
+            # This is the kind of URL that is embedded in music files metadata
+            "http://creativecommons.org/publicdomain/zero/1.0/"
+        ],
+    },
+    # Creative commons version 4.0
+    get_cc_license(version="4.0", perks=["by"]),
+    get_cc_license(version="4.0", perks=["by", "sa"]),
+    get_cc_license(version="4.0", perks=["by", "nc"]),
+    get_cc_license(version="4.0", perks=["by", "nc", "sa"]),
+    get_cc_license(version="4.0", perks=["by", "nc", "nd"]),
+    get_cc_license(version="4.0", perks=["by", "nd"]),
+    # Creative commons version 3.0
+    get_cc_license(version="3.0", perks=["by"]),
+    get_cc_license(version="3.0", perks=["by", "sa"]),
+    get_cc_license(version="3.0", perks=["by", "nc"]),
+    get_cc_license(version="3.0", perks=["by", "nc", "sa"]),
+    get_cc_license(version="3.0", perks=["by", "nc", "nd"]),
+    get_cc_license(version="3.0", perks=["by", "nd"]),
+    # Creative commons version 2.5
+    get_cc_license(version="2.5", perks=["by"]),
+    get_cc_license(version="2.5", perks=["by", "sa"]),
+    get_cc_license(version="2.5", perks=["by", "nc"]),
+    get_cc_license(version="2.5", perks=["by", "nc", "sa"]),
+    get_cc_license(version="2.5", perks=["by", "nc", "nd"]),
+    get_cc_license(version="2.5", perks=["by", "nd"]),
+    # Creative commons version 2.0
+    get_cc_license(version="2.0", perks=["by"]),
+    get_cc_license(version="2.0", perks=["by", "sa"]),
+    get_cc_license(version="2.0", perks=["by", "nc"]),
+    get_cc_license(version="2.0", perks=["by", "nc", "sa"]),
+    get_cc_license(version="2.0", perks=["by", "nc", "nd"]),
+    get_cc_license(version="2.0", perks=["by", "nd"]),
+    # Creative commons version 1.0
+    get_cc_license(version="1.0", perks=["by"]),
+    get_cc_license(version="1.0", perks=["by", "sa"]),
+    get_cc_license(version="1.0", perks=["by", "nc"]),
+    get_cc_license(version="1.0", perks=["by", "nc", "sa"]),
+    get_cc_license(version="1.0", perks=["by", "nc", "nd"]),
+    get_cc_license(version="1.0", perks=["by", "nd"]),
+]
+
+# generate ported (by country) CC licenses:
+
+for country in CC_30_COUNTRIES:
+    name = COUNTRIES[country]
+    LICENSES += [
+        get_cc_license(version="3.0", perks=["by"], country=country, country_name=name),
+        get_cc_license(
+            version="3.0", perks=["by", "sa"], country=country, country_name=name
+        ),
+        get_cc_license(
+            version="3.0", perks=["by", "nc"], country=country, country_name=name
+        ),
+        get_cc_license(
+            version="3.0", perks=["by", "nc", "sa"], country=country, country_name=name
+        ),
+        get_cc_license(
+            version="3.0", perks=["by", "nc", "nd"], country=country, country_name=name
+        ),
+        get_cc_license(
+            version="3.0", perks=["by", "nd"], country=country, country_name=name
+        ),
+    ]
+
+
+for country in CC_25_COUNTRIES:
+    name = COUNTRIES[country]
+    LICENSES += [
+        get_cc_license(version="2.5", perks=["by"], country=country, country_name=name),
+        get_cc_license(
+            version="2.5", perks=["by", "sa"], country=country, country_name=name
+        ),
+        get_cc_license(
+            version="2.5", perks=["by", "nc"], country=country, country_name=name
+        ),
+        get_cc_license(
+            version="2.5", perks=["by", "nc", "sa"], country=country, country_name=name
+        ),
+        get_cc_license(
+            version="2.5", perks=["by", "nc", "nd"], country=country, country_name=name
+        ),
+        get_cc_license(
+            version="2.5", perks=["by", "nd"], country=country, country_name=name
+        ),
+    ]
+
+LICENSES = sorted(LICENSES, key=lambda l: l["code"])
+LICENSES_BY_ID = {l["code"]: l for l in LICENSES}

api/funkwhale_api/music/management/commands/check_licenses.py

@@ -0,0 +1,34 @@
+from django.core.management.base import BaseCommand, CommandError
+import requests.exceptions
+
+from funkwhale_api.music import licenses
+
+
+class Command(BaseCommand):
+    help = "Check that specified licenses URLs are actually reachable"
+
+    def handle(self, *args, **options):
+        errored = []
+        objs = licenses.LICENSES
+        total = len(objs)
+        for i, data in enumerate(objs):
+            self.stderr.write("{}/{} Checking {}...".format(i + 1, total, data["code"]))
+            response = requests.get(data["url"])
+            try:
+                response.raise_for_status()
+            except requests.exceptions.RequestException:
+                self.stderr.write("!!! Error while fetching {}!".format(data["code"]))
+                errored.append((data, response))
+
+        if errored:
+            self.stdout.write("{} licenses were not reachable!".format(len(errored)))
+            for row, response in errored:
+                self.stdout.write(
+                    "- {}: error {} at url {}".format(
+                        row["code"], response.status_code, row["url"]
+                    )
+                )
+
+            raise CommandError()
+        else:
+            self.stdout.write("All licenses are valid and reachable :)")

api/funkwhale_api/music/metadata.py

@@ -1,8 +1,16 @@
+import base64
 import datetime
-import mutagen
+import logging
 import pendulum
+
+import mutagen._util
+import mutagen.oggtheora
+import mutagen.oggvorbis
+import mutagen.flac
+
 from django import forms
 
+logger = logging.getLogger(__name__)
 NODEFAULT = object()
 
 
@@ -14,14 +22,26 @@ class UnsupportedTag(KeyError):
     pass
 
 
+class ParseError(ValueError):
+    pass
+
+
 def get_id3_tag(f, k):
     if k == "pictures":
         return f.tags.getall("APIC")
     # First we try to grab the standard key
-    try:
+    possible_attributes = [("text", True), ("url", False)]
-        return f.tags[k].text[0]
+    for attr, select_first in possible_attributes:
-    except KeyError:
+        try:
-        pass
+            v = getattr(f.tags[k], attr)
+            if select_first:
+                v = v[0]
+            return v
+        except KeyError:
+            break
+        except AttributeError:
+            continue
+
     # then we fallback on parsing non standard tags
     all_tags = f.tags.getall("TXXX")
     try:
@@ -68,6 +88,31 @@ def clean_flac_pictures(apic):
     return pictures
 
 
+def clean_ogg_pictures(metadata_block_picture):
+    pictures = []
+    for b64_data in [metadata_block_picture]:
+
+        try:
+            data = base64.b64decode(b64_data)
+        except (TypeError, ValueError):
+            continue
+
+        try:
+            picture = mutagen.flac.Picture(data)
+        except mutagen.flac.FLACError:
+            continue
+
+        pictures.append(
+            {
+                "mimetype": picture.mime,
+                "content": picture.data,
+                "description": "",
+                "type": picture.type.real,
+            }
+        )
+    return pictures
+
+
 def get_mp3_recording_id(f, k):
     try:
         return [t for t in f.tags.getall("UFID") if "musicbrainz.org" in t.owner][
@@ -77,7 +122,7 @@ def get_mp3_recording_id(f, k):
         raise TagNotFound(k)
 
 
-def convert_track_number(v):
+def convert_position(v):
     try:
         return int(v)
     except ValueError:
@@ -103,8 +148,22 @@ class FirstUUIDField(forms.UUIDField):
 
 
 def get_date(value):
-    parsed = pendulum.parse(str(value))
+    ADDITIONAL_FORMATS = ["%Y-%d-%m %H:%M"]  # deezer date format
-    return datetime.date(parsed.year, parsed.month, parsed.day)
+    try:
+        parsed = pendulum.parse(str(value))
+        return datetime.date(parsed.year, parsed.month, parsed.day)
+    except pendulum.exceptions.ParserError:
+        pass
+
+    for date_format in ADDITIONAL_FORMATS:
+        try:
+            parsed = datetime.datetime.strptime(value, date_format)
+        except ValueError:
+            continue
+        else:
+            return datetime.date(parsed.year, parsed.month, parsed.day)
+
+    raise ParseError("{} cannot be parsed as a date".format(value))
 
 
 def split_and_return_first(separator):
@@ -127,8 +186,9 @@ CONF = {
         "fields": {
             "track_number": {
                 "field": "TRACKNUMBER",
-                "to_application": convert_track_number,
+                "to_application": convert_position,
             },
+            "disc_number": {"field": "DISCNUMBER", "to_application": convert_position},
             "title": {},
             "artist": {},
             "album_artist": {
@@ -141,6 +201,8 @@ CONF = {
             "musicbrainz_artistid": {},
             "musicbrainz_albumartistid": {},
             "musicbrainz_recordingid": {"field": "musicbrainz_trackid"},
+            "license": {},
+            "copyright": {},
         },
     },
     "OggVorbis": {
@@ -148,8 +210,9 @@ CONF = {
         "fields": {
             "track_number": {
                 "field": "TRACKNUMBER",
-                "to_application": convert_track_number,
+                "to_application": convert_position,
             },
+            "disc_number": {"field": "DISCNUMBER", "to_application": convert_position},
             "title": {},
             "artist": {},
             "album_artist": {
@@ -162,6 +225,12 @@ CONF = {
             "musicbrainz_artistid": {},
             "musicbrainz_albumartistid": {},
             "musicbrainz_recordingid": {"field": "musicbrainz_trackid"},
+            "license": {},
+            "copyright": {},
+            "pictures": {
+                "field": "metadata_block_picture",
+                "to_application": clean_ogg_pictures,
+            },
         },
     },
     "OggTheora": {
@@ -169,8 +238,9 @@ CONF = {
         "fields": {
             "track_number": {
                 "field": "TRACKNUMBER",
-                "to_application": convert_track_number,
+                "to_application": convert_position,
             },
+            "disc_number": {"field": "DISCNUMBER", "to_application": convert_position},
             "title": {},
             "artist": {},
             "album_artist": {"field": "albumartist"},
@@ -180,13 +250,16 @@ CONF = {
             "musicbrainz_artistid": {"field": "MusicBrainz Artist Id"},
             "musicbrainz_albumartistid": {"field": "MusicBrainz Album Artist Id"},
             "musicbrainz_recordingid": {"field": "MusicBrainz Track Id"},
+            "license": {},
+            "copyright": {},
         },
     },
     "MP3": {
         "getter": get_id3_tag,
         "clean_pictures": clean_id3_pictures,
         "fields": {
-            "track_number": {"field": "TRCK", "to_application": convert_track_number},
+            "track_number": {"field": "TRCK", "to_application": convert_position},
+            "disc_number": {"field": "TPOS", "to_application": convert_position},
             "title": {"field": "TIT2"},
             "artist": {"field": "TPE1"},
             "album_artist": {"field": "TPE2"},
@@ -200,6 +273,8 @@ CONF = {
                 "getter": get_mp3_recording_id,
             },
             "pictures": {},
+            "license": {"field": "WCOP"},
+            "copyright": {"field": "TCOP"},
         },
     },
     "FLAC": {
@@ -208,8 +283,9 @@ CONF = {
         "fields": {
             "track_number": {
                 "field": "tracknumber",
-                "to_application": convert_track_number,
+                "to_application": convert_position,
             },
+            "disc_number": {"field": "discnumber", "to_application": convert_position},
             "title": {},
             "artist": {},
             "album_artist": {"field": "albumartist"},
@@ -221,12 +297,15 @@ CONF = {
             "musicbrainz_recordingid": {"field": "musicbrainz_trackid"},
             "test": {},
             "pictures": {},
+            "license": {},
+            "copyright": {},
         },
     },
 }
 
 ALL_FIELDS = [
     "track_number",
+    "disc_number",
     "title",
     "artist",
     "album_artist",
@@ -236,14 +315,17 @@ ALL_FIELDS = [
     "musicbrainz_artistid",
     "musicbrainz_albumartistid",
     "musicbrainz_recordingid",
+    "license",
+    "copyright",
 ]
 
 
 class Metadata(object):
-    def __init__(self, path):
+    def __init__(self, filething, kind=mutagen.File):
-        self._file = mutagen.File(path)
+        self._file = kind(filething)
         if self._file is None:
-            raise ValueError("Cannot parse metadata from {}".format(path))
+            raise ValueError("Cannot parse metadata from {}".format(filething))
+        self.fallback = self.load_fallback(filething, self._file)
         ft = self.get_file_type(self._file)
         try:
             self._conf = CONF[ft]
@@ -253,7 +335,40 @@ class Metadata(object):
     def get_file_type(self, f):
         return f.__class__.__name__
 
+    def load_fallback(self, filething, parent):
+        """
+        In some situations, such as Ogg Theora files tagged with MusicBrainz Picard,
+        part of the tags are only available in the ogg vorbis comments
+        """
+        try:
+            filething.seek(0)
+        except AttributeError:
+            pass
+        if isinstance(parent, mutagen.oggtheora.OggTheora):
+            try:
+                return Metadata(filething, kind=mutagen.oggvorbis.OggVorbis)
+            except (ValueError, mutagen._util.MutagenError):
+                raise
+                pass
+
     def get(self, key, default=NODEFAULT):
+        try:
+            return self._get_from_self(key)
+        except TagNotFound:
+            if not self.fallback:
+                if default != NODEFAULT:
+                    return default
+                else:
+                    raise
+            else:
+                return self.fallback.get(key, default=default)
+        except UnsupportedTag:
+            if not self.fallback:
+                raise
+            else:
+                return self.fallback.get(key, default=default)
+
+    def _get_from_self(self, key, default=NODEFAULT):
         try:
             field_conf = self._conf["fields"][key]
         except KeyError:
@@ -275,7 +390,7 @@ class Metadata(object):
             v = field.to_python(v)
         return v
 
-    def all(self):
+    def all(self, ignore_parse_errors=True):
         """
         Return a dict containing all metadata of the file
         """
@@ -286,11 +401,22 @@ class Metadata(object):
                 data[field] = self.get(field, None)
             except (TagNotFound, forms.ValidationError):
                 data[field] = None
+            except ParseError as e:
+                if not ignore_parse_errors:
+                    raise
+                logger.warning("Unparsable field {}: {}".format(field, str(e)))
+                data[field] = None
 
         return data
 
-    def get_picture(self, picture_type="cover_front"):
+    def get_picture(self, *picture_types):
-        ptype = getattr(mutagen.id3.PictureType, picture_type.upper())
+        if not picture_types:
+            raise ValueError("You need to request at least one picture type")
+        ptypes = [
+            getattr(mutagen.id3.PictureType, picture_type.upper())
+            for picture_type in picture_types
+        ]
+
         try:
             pictures = self.get("pictures")
         except (UnsupportedTag, TagNotFound):
@@ -298,6 +424,9 @@ class Metadata(object):
 
         cleaner = self._conf.get("clean_pictures", lambda v: v)
         pictures = cleaner(pictures)
-        for p in pictures:
+        if not pictures:
-            if p["type"] == ptype:
+            return
-                return p
+        for ptype in ptypes:
+            for p in pictures:
+                if p["type"] == ptype:
+                    return p

api/funkwhale_api/music/migrations/0033_auto_20181023_1837.py

@@ -0,0 +1,53 @@
+# Generated by Django 2.0.9 on 2018-10-23 18:37
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import funkwhale_api.music.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0032_track_file_to_upload'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='UploadVersion',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('mimetype', models.CharField(choices=[('audio/ogg', 'ogg'), ('audio/mpeg', 'mp3'), ('audio/x-flac', 'flac')], max_length=50)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('accessed_date', models.DateTimeField(blank=True, null=True)),
+                ('audio_file', models.FileField(max_length=255, upload_to=funkwhale_api.music.models.get_file_path)),
+                ('bitrate', models.PositiveIntegerField()),
+                ('size', models.IntegerField()),
+                ('upload', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='music.Upload')),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='album',
+            name='from_activity',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='federation.Activity'),
+        ),
+        migrations.AlterField(
+            model_name='artist',
+            name='from_activity',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='federation.Activity'),
+        ),
+        migrations.AlterField(
+            model_name='track',
+            name='from_activity',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='federation.Activity'),
+        ),
+        migrations.AlterField(
+            model_name='work',
+            name='from_activity',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='federation.Activity'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='uploadversion',
+            unique_together={('upload', 'mimetype', 'bitrate')},
+        ),
+    ]

api/funkwhale_api/music/migrations/0034_auto_20181127_0325.py

@@ -0,0 +1,36 @@
+# Generated by Django 2.0.9 on 2018-11-27 03:25
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0033_auto_20181023_1837'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='License',
+            fields=[
+                ('code', models.CharField(max_length=100, primary_key=True, serialize=False)),
+                ('url', models.URLField(max_length=500)),
+                ('copyleft', models.BooleanField()),
+                ('commercial', models.BooleanField()),
+                ('attribution', models.BooleanField()),
+                ('derivative', models.BooleanField()),
+                ('redistribute', models.BooleanField()),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='uploadversion',
+            name='mimetype',
+            field=models.CharField(choices=[('audio/ogg', 'ogg'), ('audio/mpeg', 'mp3'), ('audio/x-flac', 'flac'), ('audio/flac', 'flac')], max_length=50),
+        ),
+        migrations.AddField(
+            model_name='track',
+            name='license',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tracks', to='music.License'),
+        ),
+    ]

api/funkwhale_api/music/migrations/0035_auto_20181203_1515.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.0.9 on 2018-12-03 15:15
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0034_auto_20181127_0325'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='track',
+            name='copyright',
+            field=models.CharField(blank=True, max_length=500, null=True),
+        ),
+        migrations.AlterField(
+            model_name='track',
+            name='license',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='tracks', to='music.License'),
+        ),
+    ]

api/funkwhale_api/music/migrations/0036_track_disc_number.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.9 on 2018-12-04 15:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0035_auto_20181203_1515'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='track',
+            name='disc_number',
+            field=models.PositiveIntegerField(blank=True, null=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0037_auto_20190103_1757.py

@@ -0,0 +1,43 @@
+# Generated by Django 2.0.9 on 2019-01-03 17:57
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0036_track_disc_number'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='track',
+            options={'ordering': ['album', 'disc_number', 'position']},
+        ),
+        migrations.AlterField(
+            model_name='album',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='artist',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='track',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='upload',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='work',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -11,7 +11,7 @@ from django.conf import settings
 from django.contrib.postgres.fields import JSONField
 from django.core.files.base import ContentFile
 from django.core.serializers.json import DjangoJSONEncoder
-from django.db import models
+from django.db import models, transaction
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.urls import reverse
@@ -29,7 +29,7 @@ from funkwhale_api.federation import models as federation_models
 from funkwhale_api.federation import utils as federation_utils
 from . import importers, metadata, utils
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 
 
 def empty_dict():
@@ -44,7 +44,7 @@ class APIModelMixin(models.Model):
         "federation.Activity", null=True, blank=True, on_delete=models.SET_NULL
     )
     api_includes = []
-    creation_date = models.DateTimeField(default=timezone.now)
+    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
     import_hooks = []
 
     class Meta:
@@ -113,6 +113,33 @@ class APIModelMixin(models.Model):
         return super().save(**kwargs)
 
 
+class License(models.Model):
+    code = models.CharField(primary_key=True, max_length=100)
+    url = models.URLField(max_length=500)
+
+    # if true, license is a copyleft license, meaning that derivative
+    # work must be shared under the same license
+    copyleft = models.BooleanField()
+    # if true, commercial use of the work is allowed
+    commercial = models.BooleanField()
+    # if true, attribution to the original author is required when reusing
+    # the work
+    attribution = models.BooleanField()
+    # if true, derivative work are allowed
+    derivative = models.BooleanField()
+    # if true, redistribution of the wor is allowed
+    redistribute = models.BooleanField()
+
+    @property
+    def conf(self):
+        from . import licenses
+
+        for row in licenses.LICENSES:
+            if self.code == row["code"]:
+                return row
+        logger.warning("%s do not match any registered license", self.code)
+
+
 class ArtistQuerySet(models.QuerySet):
     def with_albums_count(self):
         return self.annotate(_albums_count=models.Count("albums"))
@@ -124,8 +151,8 @@ class ArtistQuerySet(models.QuerySet):
 
     def annotate_playable_by_actor(self, actor):
         tracks = (
-            Track.objects.playable_by(actor)
+            Upload.objects.playable_by(actor)
-            .filter(artist=models.OuterRef("id"))
+            .filter(track__artist=models.OuterRef("id"))
             .order_by("id")
             .values("id")[:1]
         )
@@ -134,10 +161,11 @@ class ArtistQuerySet(models.QuerySet):
 
     def playable_by(self, actor, include=True):
         tracks = Track.objects.playable_by(actor, include)
+        matches = self.filter(tracks__in=tracks).values_list("pk")
         if include:
-            return self.filter(tracks__in=tracks)
+            return self.filter(pk__in=matches)
         else:
-            return self.exclude(tracks__in=tracks)
+            return self.exclude(pk__in=matches)
 
 
 class Artist(APIModelMixin):
@@ -192,8 +220,8 @@ class AlbumQuerySet(models.QuerySet):
 
     def annotate_playable_by_actor(self, actor):
         tracks = (
-            Track.objects.playable_by(actor)
+            Upload.objects.playable_by(actor)
-            .filter(album=models.OuterRef("id"))
+            .filter(track__album=models.OuterRef("id"))
             .order_by("id")
             .values("id")[:1]
         )
@@ -202,10 +230,15 @@ class AlbumQuerySet(models.QuerySet):
 
     def playable_by(self, actor, include=True):
         tracks = Track.objects.playable_by(actor, include)
+        matches = self.filter(tracks__in=tracks).values_list("pk")
         if include:
-            return self.filter(tracks__in=tracks)
+            return self.filter(pk__in=matches)
         else:
-            return self.exclude(tracks__in=tracks)
+            return self.exclude(pk__in=matches)
+
+    def with_prefetched_tracks_and_playable_uploads(self, actor):
+        tracks = Track.objects.with_playable_uploads(actor)
+        return self.prefetch_related(models.Prefetch("tracks", queryset=tracks))
 
 
 class Album(APIModelMixin):
@@ -398,24 +431,23 @@ class TrackQuerySet(models.QuerySet):
 
     def playable_by(self, actor, include=True):
         files = Upload.objects.playable_by(actor, include)
+        matches = self.filter(uploads__in=files).values_list("pk")
         if include:
-            return self.filter(uploads__in=files)
+            return self.filter(pk__in=matches)
         else:
-            return self.exclude(uploads__in=files)
+            return self.exclude(pk__in=matches)
 
-    def annotate_duration(self):
+    def with_playable_uploads(self, actor):
-        first_upload = Upload.objects.filter(track=models.OuterRef("pk")).order_by("pk")
+        uploads = Upload.objects.playable_by(actor).select_related("track")
-        return self.annotate(
+        return self.prefetch_related(
-            duration=models.Subquery(first_upload.values("duration")[:1])
+            models.Prefetch("uploads", queryset=uploads, to_attr="playable_uploads")
         )
 
-    def annotate_file_data(self):
+    def order_for_album(self):
-        first_upload = Upload.objects.filter(track=models.OuterRef("pk")).order_by("pk")
+        """
-        return self.annotate(
+        Order by disc number then position
-            bitrate=models.Subquery(first_upload.values("bitrate")[:1]),
+        """
-            size=models.Subquery(first_upload.values("size")[:1]),
+        return self.order_by("disc_number", "position", "title")
-            mimetype=models.Subquery(first_upload.values("mimetype")[:1]),
-        )
 
 
 def get_artist(release_list):
@@ -427,6 +459,7 @@ def get_artist(release_list):
 class Track(APIModelMixin):
     title = models.CharField(max_length=255)
     artist = models.ForeignKey(Artist, related_name="tracks", on_delete=models.CASCADE)
+    disc_number = models.PositiveIntegerField(null=True, blank=True)
     position = models.PositiveIntegerField(null=True, blank=True)
     album = models.ForeignKey(
         Album, related_name="tracks", null=True, blank=True, on_delete=models.CASCADE
@@ -434,6 +467,14 @@ class Track(APIModelMixin):
     work = models.ForeignKey(
         Work, related_name="tracks", null=True, blank=True, on_delete=models.CASCADE
     )
+    license = models.ForeignKey(
+        License,
+        null=True,
+        blank=True,
+        on_delete=models.DO_NOTHING,
+        related_name="tracks",
+    )
+    copyright = models.CharField(max_length=500, null=True, blank=True)
     federation_namespace = "tracks"
     musicbrainz_model = "recording"
     api = musicbrainz.api.recordings
@@ -454,7 +495,7 @@ class Track(APIModelMixin):
     tags = TaggableManager(blank=True)
 
     class Meta:
-        ordering = ["album", "position"]
+        ordering = ["album", "disc_number", "position"]
 
     def __str__(self):
         return self.title
@@ -551,6 +592,17 @@ class Track(APIModelMixin):
     def listen_url(self):
         return reverse("api:v1:listen-detail", kwargs={"uuid": self.uuid})
 
+    @property
+    def local_license(self):
+        """
+        Since license primary keys are strings, and we can get the data
+        from our hardcoded licenses.LICENSES list, there is no need
+        for extra SQL joins / queries.
+        """
+        from . import licenses
+
+        return licenses.LICENSES_BY_ID.get(self.license_id)
+
 
 class UploadQuerySet(models.QuerySet):
     def playable_by(self, actor, include=True):
@@ -566,6 +618,9 @@ class UploadQuerySet(models.QuerySet):
     def for_federation(self):
         return self.filter(import_status="finished", mimetype__startswith="audio/")
 
+    def with_file(self):
+        return self.exclude(audio_file=None).exclude(audio_file="")
+
 
 TRACK_FILE_IMPORT_STATUS_CHOICES = (
     ("pending", "Pending"),
@@ -576,6 +631,9 @@ TRACK_FILE_IMPORT_STATUS_CHOICES = (
 
 
 def get_file_path(instance, filename):
+    if isinstance(instance, UploadVersion):
+        return common_utils.ChunkedPath("transcoded")(instance, filename)
+
     if instance.library.actor.get_user():
         return common_utils.ChunkedPath("tracks")(instance, filename)
     else:
@@ -600,7 +658,7 @@ class Upload(models.Model):
         blank=True,
         max_length=500,
     )
-    creation_date = models.DateTimeField(default=timezone.now)
+    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
     modification_date = models.DateTimeField(default=timezone.now, null=True)
     accessed_date = models.DateTimeField(null=True, blank=True)
     duration = models.IntegerField(null=True, blank=True)
@@ -687,9 +745,14 @@ class Upload(models.Model):
 
     @property
     def extension(self):
-        if not self.audio_file:
+        try:
-            return
+            return utils.MIMETYPE_TO_EXTENSION[self.mimetype]
-        return os.path.splitext(self.audio_file.name)[-1].replace(".", "", 1)
+        except KeyError:
+            pass
+        if self.audio_file:
+            return os.path.splitext(self.audio_file.name)[-1].replace(".", "", 1)
+        if self.in_place_path:
+            return os.path.splitext(self.in_place_path)[-1].replace(".", "", 1)
 
     def get_file_size(self):
         if self.audio_file:
@@ -739,6 +802,67 @@ class Upload(models.Model):
     def listen_url(self):
         return self.track.listen_url + "?upload={}".format(self.uuid)
 
+    def get_transcoded_version(self, format):
+        mimetype = utils.EXTENSION_TO_MIMETYPE[format]
+        existing_versions = list(self.versions.filter(mimetype=mimetype))
+        if existing_versions:
+            # we found an existing version, no need to transcode again
+            return existing_versions[0]
+
+        return self.create_transcoded_version(mimetype, format)
+
+    @transaction.atomic
+    def create_transcoded_version(self, mimetype, format):
+        # we create the version with an empty file, then
+        # we'll write to it
+        f = ContentFile(b"")
+        version = self.versions.create(
+            mimetype=mimetype, bitrate=self.bitrate or 128000, size=0
+        )
+        # we keep the same name, but we update the extension
+        new_name = os.path.splitext(os.path.basename(self.audio_file.name))[
+            0
+        ] + ".{}".format(format)
+        version.audio_file.save(new_name, f)
+        utils.transcode_file(
+            input=self.audio_file,
+            output=version.audio_file,
+            input_format=utils.MIMETYPE_TO_EXTENSION[self.mimetype],
+            output_format=utils.MIMETYPE_TO_EXTENSION[mimetype],
+        )
+        version.size = version.audio_file.size
+        version.save(update_fields=["size"])
+
+        return version
+
+    @property
+    def in_place_path(self):
+        if not self.source or not self.source.startswith("file://"):
+            return
+        return self.source.lstrip("file://")
+
+
+MIMETYPE_CHOICES = [(mt, ext) for ext, mt in utils.AUDIO_EXTENSIONS_AND_MIMETYPE]
+
+
+class UploadVersion(models.Model):
+    upload = models.ForeignKey(
+        Upload, related_name="versions", on_delete=models.CASCADE
+    )
+    mimetype = models.CharField(max_length=50, choices=MIMETYPE_CHOICES)
+    creation_date = models.DateTimeField(default=timezone.now)
+    accessed_date = models.DateTimeField(null=True, blank=True)
+    audio_file = models.FileField(upload_to=get_file_path, max_length=255)
+    bitrate = models.PositiveIntegerField()
+    size = models.IntegerField()
+
+    class Meta:
+        unique_together = ("upload", "mimetype", "bitrate")
+
+    @property
+    def filename(self):
+        return self.upload.filename
+
 
 IMPORT_STATUS_CHOICES = (
     ("pending", "Pending"),
@@ -983,7 +1107,7 @@ def update_request_status(sender, instance, created, **kwargs):
 
 @receiver(models.signals.post_save, sender=Album)
 def warm_album_covers(sender, instance, **kwargs):
-    if not instance.cover:
+    if not instance.cover or not settings.CREATE_IMAGE_THUMBNAILS:
         return
     album_covers_warmer = VersatileImageFieldWarmer(
         instance_or_queryset=instance, rendition_key_set="square", image_attr="cover"

api/funkwhale_api/music/serializers.py

@@ -1,4 +1,8 @@
+import urllib.parse
+
 from django.db import transaction
+from django import urls
+from django.conf import settings
 from rest_framework import serializers
 from taggit.models import Tag
 from versatileimagefield.serializers import VersatileImageFieldSerializer
@@ -7,6 +11,7 @@ from funkwhale_api.activity import serializers as activity_serializers
 from funkwhale_api.common import serializers as common_serializers
 from funkwhale_api.common import utils as common_utils
 from funkwhale_api.federation import routes
+from funkwhale_api.federation import utils as federation_utils
 
 from . import filters, models, tasks
 
@@ -14,6 +19,21 @@ from . import filters, models, tasks
 cover_field = VersatileImageFieldSerializer(allow_null=True, sizes="square")
 
 
+class LicenseSerializer(serializers.Serializer):
+    id = serializers.SerializerMethodField()
+    url = serializers.URLField()
+    code = serializers.CharField()
+    name = serializers.CharField()
+    redistribute = serializers.BooleanField()
+    derivative = serializers.BooleanField()
+    commercial = serializers.BooleanField()
+    attribution = serializers.BooleanField()
+    copyleft = serializers.BooleanField()
+
+    def get_id(self, obj):
+        return obj["identifiers"][0]
+
+
 class ArtistAlbumSerializer(serializers.ModelSerializer):
     tracks_count = serializers.SerializerMethodField()
     cover = cover_field
@@ -59,7 +79,7 @@ class ArtistSimpleSerializer(serializers.ModelSerializer):
 
 class AlbumTrackSerializer(serializers.ModelSerializer):
     artist = ArtistSimpleSerializer(read_only=True)
-    is_playable = serializers.SerializerMethodField()
+    uploads = serializers.SerializerMethodField()
     listen_url = serializers.SerializerMethodField()
     duration = serializers.SerializerMethodField()
 
@@ -73,16 +93,17 @@ class AlbumTrackSerializer(serializers.ModelSerializer):
             "artist",
             "creation_date",
             "position",
-            "is_playable",
+            "disc_number",
+            "uploads",
             "listen_url",
             "duration",
+            "copyright",
+            "license",
         )
 
-    def get_is_playable(self, obj):
+    def get_uploads(self, obj):
-        try:
+        uploads = getattr(obj, "playable_uploads", [])
-            return bool(obj.is_playable_by_actor)
+        return TrackUploadSerializer(uploads, many=True).data
-        except AttributeError:
-            return None
 
     def get_listen_url(self, obj):
         return obj.listen_url
@@ -115,15 +136,14 @@ class AlbumSerializer(serializers.ModelSerializer):
         )
 
     def get_tracks(self, o):
-        ordered_tracks = sorted(
+        ordered_tracks = o.tracks.all()
-            o.tracks.all(),
-            key=lambda v: (v.position, v.title) if v.position else (99999, v.title),
-        )
         return AlbumTrackSerializer(ordered_tracks, many=True).data
 
     def get_is_playable(self, obj):
         try:
-            return any([bool(t.is_playable_by_actor) for t in obj.tracks.all()])
+            return any(
+                [bool(getattr(t, "playable_uploads", [])) for t in obj.tracks.all()]
+            )
         except AttributeError:
             return None
 
@@ -145,16 +165,26 @@ class TrackAlbumSerializer(serializers.ModelSerializer):
         )
 
 
+class TrackUploadSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.Upload
+        fields = (
+            "uuid",
+            "listen_url",
+            "size",
+            "duration",
+            "bitrate",
+            "mimetype",
+            "extension",
+        )
+
+
 class TrackSerializer(serializers.ModelSerializer):
     artist = ArtistSimpleSerializer(read_only=True)
     album = TrackAlbumSerializer(read_only=True)
     lyrics = serializers.SerializerMethodField()
-    is_playable = serializers.SerializerMethodField()
+    uploads = serializers.SerializerMethodField()
     listen_url = serializers.SerializerMethodField()
-    duration = serializers.SerializerMethodField()
-    bitrate = serializers.SerializerMethodField()
-    size = serializers.SerializerMethodField()
-    mimetype = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Track
@@ -166,13 +196,12 @@ class TrackSerializer(serializers.ModelSerializer):
             "artist",
             "creation_date",
             "position",
+            "disc_number",
             "lyrics",
-            "is_playable",
+            "uploads",
             "listen_url",
-            "duration",
+            "copyright",
-            "bitrate",
+            "license",
-            "size",
-            "mimetype",
         )
 
     def get_lyrics(self, obj):
@@ -181,37 +210,12 @@ class TrackSerializer(serializers.ModelSerializer):
     def get_listen_url(self, obj):
         return obj.listen_url
 
-    def get_is_playable(self, obj):
+    def get_uploads(self, obj):
-        try:
+        uploads = getattr(obj, "playable_uploads", [])
-            return bool(obj.is_playable_by_actor)
+        return TrackUploadSerializer(uploads, many=True).data
-        except AttributeError:
-            return None
-
-    def get_duration(self, obj):
-        try:
-            return obj.duration
-        except AttributeError:
-            return None
-
-    def get_bitrate(self, obj):
-        try:
-            return obj.bitrate
-        except AttributeError:
-            return None
-
-    def get_size(self, obj):
-        try:
-            return obj.size
-        except AttributeError:
-            return None
-
-    def get_mimetype(self, obj):
-        try:
-            return obj.mimetype
-        except AttributeError:
-            return None
 
 
+@common_serializers.track_fields_for_update("name", "description", "privacy_level")
 class LibraryForOwnerSerializer(serializers.ModelSerializer):
     uploads_count = serializers.SerializerMethodField()
     size = serializers.SerializerMethodField()
@@ -236,6 +240,11 @@ class LibraryForOwnerSerializer(serializers.ModelSerializer):
     def get_size(self, o):
         return getattr(o, "_size", 0)
 
+    def on_updated_fields(self, obj, before, after):
+        routes.outbox.dispatch(
+            {"type": "Update", "object": {"type": "Library"}}, context={"library": obj}
+        )
+
 
 class UploadSerializer(serializers.ModelSerializer):
     track = TrackSerializer(required=False, allow_null=True)
@@ -376,3 +385,100 @@ class TrackActivitySerializer(activity_serializers.ModelSerializer):
 
     def get_type(self, obj):
         return "Audio"
+
+
+class OembedSerializer(serializers.Serializer):
+    format = serializers.ChoiceField(choices=["json"])
+    url = serializers.URLField()
+    maxheight = serializers.IntegerField(required=False)
+    maxwidth = serializers.IntegerField(required=False)
+
+    def validate(self, validated_data):
+        try:
+            match = common_utils.spa_resolve(
+                urllib.parse.urlparse(validated_data["url"]).path
+            )
+        except urls.exceptions.Resolver404:
+            raise serializers.ValidationError(
+                "Invalid URL {}".format(validated_data["url"])
+            )
+        data = {
+            "version": "1.0",
+            "type": "rich",
+            "provider_name": settings.APP_NAME,
+            "provider_url": settings.FUNKWHALE_URL,
+            "height": validated_data.get("maxheight") or 400,
+            "width": validated_data.get("maxwidth") or 600,
+        }
+        embed_id = None
+        embed_type = None
+        if match.url_name == "library_track":
+            qs = models.Track.objects.select_related("artist", "album__artist").filter(
+                pk=int(match.kwargs["pk"])
+            )
+            try:
+                track = qs.get()
+            except models.Track.DoesNotExist:
+                raise serializers.ValidationError(
+                    "No track matching id {}".format(match.kwargs["pk"])
+                )
+            embed_type = "track"
+            embed_id = track.pk
+            data["title"] = "{} by {}".format(track.title, track.artist.name)
+            if track.album.cover:
+                data["thumbnail_url"] = federation_utils.full_url(
+                    track.album.cover.crop["400x400"].url
+                )
+                data["thumbnail_width"] = 400
+                data["thumbnail_height"] = 400
+            data["description"] = track.full_name
+            data["author_name"] = track.artist.name
+            data["height"] = 150
+            data["author_url"] = federation_utils.full_url(
+                common_utils.spa_reverse(
+                    "library_artist", kwargs={"pk": track.artist.pk}
+                )
+            )
+        elif match.url_name == "library_album":
+            qs = models.Album.objects.select_related("artist").filter(
+                pk=int(match.kwargs["pk"])
+            )
+            try:
+                album = qs.get()
+            except models.Album.DoesNotExist:
+                raise serializers.ValidationError(
+                    "No album matching id {}".format(match.kwargs["pk"])
+                )
+            embed_type = "album"
+            embed_id = album.pk
+            if album.cover:
+                data["thumbnail_url"] = federation_utils.full_url(
+                    album.cover.crop["400x400"].url
+                )
+                data["thumbnail_width"] = 400
+                data["thumbnail_height"] = 400
+            data["title"] = "{} by {}".format(album.title, album.artist.name)
+            data["description"] = "{} by {}".format(album.title, album.artist.name)
+            data["author_name"] = album.artist.name
+            data["height"] = 400
+            data["author_url"] = federation_utils.full_url(
+                common_utils.spa_reverse(
+                    "library_artist", kwargs={"pk": album.artist.pk}
+                )
+            )
+        else:
+            raise serializers.ValidationError(
+                "Unsupported url: {}".format(validated_data["url"])
+            )
+        data[
+            "html"
+        ] = '<iframe width="{}" height="{}" scrolling="no" frameborder="no" src="{}"></iframe>'.format(
+            data["width"],
+            data["height"],
+            settings.FUNKWHALE_EMBED_URL
+            + "?type={}&id={}".format(embed_type, embed_id),
+        )
+        return data
+
+    def create(self, data):
+        return data

api/funkwhale_api/music/spa_views.py

@@ -0,0 +1,168 @@
+import urllib.parse
+
+from django.conf import settings
+from django.urls import reverse
+
+from funkwhale_api.common import utils
+
+from . import models
+
+
+def library_track(request, pk):
+    queryset = models.Track.objects.filter(pk=pk).select_related("album", "artist")
+    try:
+        obj = queryset.get()
+    except models.Track.DoesNotExist:
+        return []
+    track_url = utils.join_url(
+        settings.FUNKWHALE_URL,
+        utils.spa_reverse("library_track", kwargs={"pk": obj.pk}),
+    )
+    metas = [
+        {"tag": "meta", "property": "og:url", "content": track_url},
+        {"tag": "meta", "property": "og:title", "content": obj.title},
+        {"tag": "meta", "property": "og:type", "content": "music.song"},
+        {"tag": "meta", "property": "music:album:disc", "content": obj.disc_number},
+        {"tag": "meta", "property": "music:album:track", "content": obj.position},
+        {
+            "tag": "meta",
+            "property": "music:musician",
+            "content": utils.join_url(
+                settings.FUNKWHALE_URL,
+                utils.spa_reverse("library_artist", kwargs={"pk": obj.artist.pk}),
+            ),
+        },
+        {
+            "tag": "meta",
+            "property": "music:album",
+            "content": utils.join_url(
+                settings.FUNKWHALE_URL,
+                utils.spa_reverse("library_album", kwargs={"pk": obj.album.pk}),
+            ),
+        },
+    ]
+    if obj.album.cover:
+        metas.append(
+            {
+                "tag": "meta",
+                "property": "og:image",
+                "content": utils.join_url(
+                    settings.FUNKWHALE_URL, obj.album.cover.crop["400x400"].url
+                ),
+            }
+        )
+
+    if obj.uploads.playable_by(None).exists():
+        metas.append(
+            {
+                "tag": "meta",
+                "property": "og:audio",
+                "content": utils.join_url(settings.FUNKWHALE_URL, obj.listen_url),
+            }
+        )
+
+        metas.append(
+            {
+                "tag": "link",
+                "rel": "alternate",
+                "type": "application/json+oembed",
+                "href": (
+                    utils.join_url(settings.FUNKWHALE_URL, reverse("api:v1:oembed"))
+                    + "?format=json&url={}".format(urllib.parse.quote_plus(track_url))
+                ),
+            }
+        )
+    return metas
+
+
+def library_album(request, pk):
+    queryset = models.Album.objects.filter(pk=pk).select_related("artist")
+    try:
+        obj = queryset.get()
+    except models.Album.DoesNotExist:
+        return []
+    album_url = utils.join_url(
+        settings.FUNKWHALE_URL,
+        utils.spa_reverse("library_album", kwargs={"pk": obj.pk}),
+    )
+    metas = [
+        {"tag": "meta", "property": "og:url", "content": album_url},
+        {"tag": "meta", "property": "og:title", "content": obj.title},
+        {"tag": "meta", "property": "og:type", "content": "music.album"},
+        {
+            "tag": "meta",
+            "property": "music:musician",
+            "content": utils.join_url(
+                settings.FUNKWHALE_URL,
+                utils.spa_reverse("library_artist", kwargs={"pk": obj.artist.pk}),
+            ),
+        },
+    ]
+
+    if obj.release_date:
+        metas.append(
+            {
+                "tag": "meta",
+                "property": "music:release_date",
+                "content": str(obj.release_date),
+            }
+        )
+
+    if obj.cover:
+        metas.append(
+            {
+                "tag": "meta",
+                "property": "og:image",
+                "content": utils.join_url(
+                    settings.FUNKWHALE_URL, obj.cover.crop["400x400"].url
+                ),
+            }
+        )
+
+    if models.Upload.objects.filter(track__album=obj).playable_by(None).exists():
+        metas.append(
+            {
+                "tag": "link",
+                "rel": "alternate",
+                "type": "application/json+oembed",
+                "href": (
+                    utils.join_url(settings.FUNKWHALE_URL, reverse("api:v1:oembed"))
+                    + "?format=json&url={}".format(urllib.parse.quote_plus(album_url))
+                ),
+            }
+        )
+    return metas
+
+
+def library_artist(request, pk):
+    queryset = models.Artist.objects.filter(pk=pk)
+    try:
+        obj = queryset.get()
+    except models.Artist.DoesNotExist:
+        return []
+    artist_url = utils.join_url(
+        settings.FUNKWHALE_URL,
+        utils.spa_reverse("library_artist", kwargs={"pk": obj.pk}),
+    )
+    # we use latest album's cover as artist image
+    latest_album = (
+        obj.albums.exclude(cover="").exclude(cover=None).order_by("release_date").last()
+    )
+    metas = [
+        {"tag": "meta", "property": "og:url", "content": artist_url},
+        {"tag": "meta", "property": "og:title", "content": obj.name},
+        {"tag": "meta", "property": "og:type", "content": "profile"},
+    ]
+
+    if latest_album and latest_album.cover:
+        metas.append(
+            {
+                "tag": "meta",
+                "property": "og:image",
+                "content": utils.join_url(
+                    settings.FUNKWHALE_URL, latest_album.cover.crop["400x400"].url
+                ),
+            }
+        )
+
+    return metas

api/funkwhale_api/music/tasks.py

@@ -1,4 +1,5 @@
 import collections
+import datetime
 import logging
 import os
 
@@ -10,11 +11,12 @@ from django.dispatch import receiver
 from musicbrainzngs import ResponseError
 from requests.exceptions import RequestException
 
-from funkwhale_api.common import channels
+from funkwhale_api.common import channels, preferences
 from funkwhale_api.federation import routes
 from funkwhale_api.federation import library as lb
 from funkwhale_api.taskapp import celery
 
+from . import licenses
 from . import lyrics as lyrics_utils
 from . import models
 from . import metadata
@@ -189,7 +191,7 @@ def process_upload(upload):
             final_metadata = collections.ChainMap(
                 additional_data, import_metadata, file_metadata
             )
-            additional_data["cover_data"] = m.get_picture("cover_front")
+            additional_data["cover_data"] = m.get_picture("cover_front", "other")
         additional_data["upload_source"] = upload.source
         track = get_track_from_import_metadata(final_metadata)
     except UploadImportError as e:
@@ -272,9 +274,12 @@ def federation_audio_track_to_metadata(payload):
         "title": payload["name"],
         "album": payload["album"]["name"],
         "track_number": payload["position"],
+        "disc_number": payload.get("disc"),
         "artist": payload["artists"][0]["name"],
         "album_artist": payload["album"]["artists"][0]["name"],
         "date": payload["album"].get("released"),
+        "license": payload.get("license"),
+        "copyright": payload.get("copyright"),
         # musicbrainz
         "musicbrainz_recordingid": str(musicbrainz_recordingid)
         if musicbrainz_recordingid
@@ -493,8 +498,11 @@ def get_track_from_import_metadata(data):
         "mbid": track_mbid,
         "artist": artist,
         "position": track_number,
+        "disc_number": data.get("disc_number"),
         "fid": track_fid,
         "from_activity_id": from_activity_id,
+        "license": licenses.match(data.get("license"), data.get("copyright")),
+        "copyright": data.get("copyright"),
     }
     if data.get("fdate"):
         defaults["creation_date"] = data.get("fdate")
@@ -526,3 +534,19 @@ def broadcast_import_status_update_to_owner(old_status, new_status, upload, **kw
             },
         },
     )
+
+
+@celery.app.task(name="music.clean_transcoding_cache")
+def clean_transcoding_cache():
+    delay = preferences.get("music__transcoding_cache_duration")
+    if delay < 1:
+        return  # cache clearing disabled
+    limit = timezone.now() - datetime.timedelta(minutes=delay)
+    candidates = (
+        models.UploadVersion.objects.filter(
+            (Q(accessed_date__lt=limit) | Q(accessed_date=None))
+        )
+        .only("audio_file", "id")
+        .order_by("id")
+    )
+    return candidates.delete()

api/funkwhale_api/music/utils.py

@@ -2,6 +2,7 @@ import mimetypes
 
 import magic
 import mutagen
+import pydub
 
 from funkwhale_api.common.search import normalize_query, get_query  # noqa
 
@@ -32,6 +33,7 @@ AUDIO_EXTENSIONS_AND_MIMETYPE = [
     ("ogg", "audio/ogg"),
     ("mp3", "audio/mpeg"),
     ("flac", "audio/x-flac"),
+    ("flac", "audio/flac"),
 ]
 
 EXTENSION_TO_MIMETYPE = {ext: mt for ext, mt in AUDIO_EXTENSIONS_AND_MIMETYPE}
@@ -68,3 +70,10 @@ def get_actor_from_request(request):
         actor = request.user.actor
 
     return actor
+
+
+def transcode_file(input, output, input_format, output_format, **kwargs):
+    with input.open("rb"):
+        audio = pydub.AudioSegment.from_file(input, format=input_format)
+    with output.open("wb"):
+        return audio.export(output, format=output_format, **kwargs)

api/funkwhale_api/music/views.py

@@ -11,41 +11,42 @@ from rest_framework import mixins
 from rest_framework import permissions
 from rest_framework import settings as rest_settings
 from rest_framework import views, viewsets
-from rest_framework.decorators import detail_route, list_route
+from rest_framework.decorators import action
 from rest_framework.response import Response
 from taggit.models import Tag
 
-from funkwhale_api.common import utils as common_utils
 from funkwhale_api.common import permissions as common_permissions
+from funkwhale_api.common import preferences
+from funkwhale_api.common import utils as common_utils
 from funkwhale_api.federation.authentication import SignatureAuthentication
 from funkwhale_api.federation import api_serializers as federation_api_serializers
 from funkwhale_api.federation import routes
 
-from . import filters, models, serializers, tasks, utils
+from . import filters, licenses, models, serializers, tasks, utils
 
 logger = logging.getLogger(__name__)
 
 
 def get_libraries(filter_uploads):
-    def view(self, request, *args, **kwargs):
+    def libraries(self, request, *args, **kwargs):
         obj = self.get_object()
         actor = utils.get_actor_from_request(request)
         uploads = models.Upload.objects.all()
         uploads = filter_uploads(obj, uploads)
         uploads = uploads.playable_by(actor)
-        libraries = models.Library.objects.filter(
+        qs = models.Library.objects.filter(
             pk__in=uploads.values_list("library", flat=True)
-        )
+        ).annotate(_uploads_count=Count("uploads"))
-        libraries = libraries.select_related("actor")
+        qs = qs.select_related("actor")
-        page = self.paginate_queryset(libraries)
+        page = self.paginate_queryset(qs)
         if page is not None:
             serializer = federation_api_serializers.LibrarySerializer(page, many=True)
             return self.get_paginated_response(serializer.data)
 
-        serializer = federation_api_serializers.LibrarySerializer(libraries, many=True)
+        serializer = federation_api_serializers.LibrarySerializer(qs, many=True)
         return Response(serializer.data)
 
-    return view
+    return libraries
 
 
 class TagViewSetMixin(object):
@@ -61,7 +62,7 @@ class ArtistViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = models.Artist.objects.all()
     serializer_class = serializers.ArtistWithAlbumsSerializer
     permission_classes = [common_permissions.ConditionalAuthentication]
-    filter_class = filters.ArtistFilter
+    filterset_class = filters.ArtistFilter
     ordering_fields = ("id", "name", "creation_date")
 
     def get_queryset(self):
@@ -70,9 +71,9 @@ class ArtistViewSet(viewsets.ReadOnlyModelViewSet):
         albums = albums.annotate_playable_by_actor(
             utils.get_actor_from_request(self.request)
         )
-        return queryset.prefetch_related(Prefetch("albums", queryset=albums)).distinct()
+        return queryset.prefetch_related(Prefetch("albums", queryset=albums))
 
-    libraries = detail_route(methods=["get"])(
+    libraries = action(methods=["get"], detail=True)(
         get_libraries(
             filter_uploads=lambda o, uploads: uploads.filter(
                 Q(track__artist=o) | Q(track__album__artist=o)
@@ -88,25 +89,19 @@ class AlbumViewSet(viewsets.ReadOnlyModelViewSet):
     serializer_class = serializers.AlbumSerializer
     permission_classes = [common_permissions.ConditionalAuthentication]
     ordering_fields = ("creation_date", "release_date", "title")
-    filter_class = filters.AlbumFilter
+    filterset_class = filters.AlbumFilter
 
     def get_queryset(self):
         queryset = super().get_queryset()
-        tracks = models.Track.objects.annotate_playable_by_actor(
+        tracks = (
-            utils.get_actor_from_request(self.request)
+            models.Track.objects.select_related("artist")
-        ).select_related("artist")
+            .with_playable_uploads(utils.get_actor_from_request(self.request))
-        if (
+            .order_for_album()
-            hasattr(self, "kwargs")
+        )
-            and self.kwargs
-            and self.request.method.lower() == "get"
-        ):
-            # we are detailing a single album, so we can add the overhead
-            # to fetch additional data
-            tracks = tracks.annotate_duration()
         qs = queryset.prefetch_related(Prefetch("tracks", queryset=tracks))
-        return qs.distinct()
+        return qs
 
-    libraries = detail_route(methods=["get"])(
+    libraries = action(methods=["get"], detail=True)(
         get_libraries(filter_uploads=lambda o, uploads: uploads.filter(track__album=o))
     )
 
@@ -149,7 +144,9 @@ class LibraryViewSet(
         )
         instance.delete()
 
-    @detail_route(methods=["get"])
+    follows = action
+
+    @action(methods=["get"], detail=True)
     @transaction.non_atomic_requests
     def follows(self, request, *args, **kwargs):
         library = self.get_object()
@@ -177,7 +174,7 @@ class TrackViewSet(TagViewSetMixin, viewsets.ReadOnlyModelViewSet):
     queryset = models.Track.objects.all().for_nested_serialization()
     serializer_class = serializers.TrackSerializer
     permission_classes = [common_permissions.ConditionalAuthentication]
-    filter_class = filters.TrackFilter
+    filterset_class = filters.TrackFilter
     ordering_fields = (
         "creation_date",
         "title",
@@ -193,20 +190,12 @@ class TrackViewSet(TagViewSetMixin, viewsets.ReadOnlyModelViewSet):
         if user.is_authenticated and filter_favorites == "true":
             queryset = queryset.filter(track_favorites__user=user)
 
-        queryset = queryset.annotate_playable_by_actor(
+        queryset = queryset.with_playable_uploads(
             utils.get_actor_from_request(self.request)
-        ).annotate_duration()
+        )
-        if (
+        return queryset
-            hasattr(self, "kwargs")
-            and self.kwargs
-            and self.request.method.lower() == "get"
-        ):
-            # we are detailing a single track, so we can add the overhead
-            # to fetch additional data
-            queryset = queryset.annotate_file_data()
-        return queryset.distinct()
 
-    @detail_route(methods=["get"])
+    @action(methods=["get"], detail=True)
     @transaction.non_atomic_requests
     def lyrics(self, request, *args, **kwargs):
         try:
@@ -231,7 +220,7 @@ class TrackViewSet(TagViewSetMixin, viewsets.ReadOnlyModelViewSet):
         serializer = serializers.LyricsSerializer(lyrics)
         return Response(serializer.data)
 
-    libraries = detail_route(methods=["get"])(
+    libraries = action(methods=["get"], detail=True)(
         get_libraries(filter_uploads=lambda o, uploads: uploads.filter(track=o))
     )
 
@@ -267,12 +256,31 @@ def get_file_path(audio_file):
         return path.encode("utf-8")
 
 
-def handle_serve(upload, user):
+def should_transcode(upload, format):
+    if not preferences.get("music__transcoding_enabled"):
+        return False
+    if format is None:
+        return False
+    if format not in utils.EXTENSION_TO_MIMETYPE:
+        # format should match supported formats
+        return False
+    if upload.mimetype is None:
+        # upload should have a mimetype, otherwise we cannot transcode
+        return False
+    if upload.mimetype == utils.EXTENSION_TO_MIMETYPE[format]:
+        # requested format sould be different than upload mimetype, otherwise
+        # there is no need to transcode
+        return False
+    return True
+
+
+def handle_serve(upload, user, format=None):
     f = upload
     # we update the accessed_date
-    f.accessed_date = timezone.now()
+    now = timezone.now()
-    f.save(update_fields=["accessed_date"])
+    upload.accessed_date = now
-
+    upload.save(update_fields=["accessed_date"])
+    f = upload
     if f.audio_file:
         file_path = get_file_path(f.audio_file)
 
@@ -298,6 +306,14 @@ def handle_serve(upload, user):
     elif f.source and f.source.startswith("file://"):
         file_path = get_file_path(f.source.replace("file://", "", 1))
     mt = f.mimetype
+
+    if should_transcode(f, format):
+        transcoded_version = upload.get_transcoded_version(format)
+        transcoded_version.accessed_date = now
+        transcoded_version.save(update_fields=["accessed_date"])
+        f = transcoded_version
+        file_path = get_file_path(f.audio_file)
+        mt = f.mimetype
     if mt:
         response = Response(content_type=mt)
     else:
@@ -337,7 +353,8 @@ class ListenViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
         if not upload:
             return Response(status=404)
 
-        return handle_serve(upload, user=request.user)
+        format = request.GET.get("to")
+        return handle_serve(upload, user=request.user, format=format)
 
 
 class UploadViewSet(
@@ -360,7 +377,7 @@ class UploadViewSet(
     ]
     owner_field = "library.actor.user"
     owner_checks = ["read", "write"]
-    filter_class = filters.UploadFilter
+    filterset_class = filters.UploadFilter
     ordering_fields = (
         "creation_date",
         "import_date",
@@ -373,7 +390,7 @@ class UploadViewSet(
         qs = super().get_queryset()
         return qs.filter(library__actor=self.request.user.actor)
 
-    @list_route(methods=["post"])
+    @action(methods=["post"], detail=False)
     def action(self, request, *args, **kwargs):
         queryset = self.get_queryset()
         serializer = serializers.UploadActionSerializer(request.data, queryset=queryset)
@@ -433,28 +450,29 @@ class Search(views.APIView):
             "artist__name__unaccent",
         ]
         query_obj = utils.get_query(query, search_fields)
-        return (
+        qs = (
             models.Track.objects.all()
             .filter(query_obj)
             .select_related("artist", "album__artist")
-        )[: self.max_results]
+        )
+        return common_utils.order_for_search(qs, "title")[: self.max_results]
 
     def get_albums(self, query):
         search_fields = ["mbid", "title__unaccent", "artist__name__unaccent"]
         query_obj = utils.get_query(query, search_fields)
-        return (
+        qs = (
             models.Album.objects.all()
             .filter(query_obj)
             .select_related()
-            .prefetch_related("tracks")
+            .prefetch_related("tracks__artist")
-        )[: self.max_results]
+        )
+        return common_utils.order_for_search(qs, "title")[: self.max_results]
 
     def get_artists(self, query):
         search_fields = ["mbid", "name__unaccent"]
         query_obj = utils.get_query(query, search_fields)
-        return (models.Artist.objects.all().filter(query_obj).with_albums())[
+        qs = models.Artist.objects.all().filter(query_obj).with_albums()
-            : self.max_results
+        return common_utils.order_for_search(qs, "name")[: self.max_results]
-        ]
 
     def get_tags(self, query):
         search_fields = ["slug", "name__unaccent"]
@@ -468,3 +486,38 @@ class Search(views.APIView):
         )
 
         return qs.filter(query_obj)[: self.max_results]
+
+
+class LicenseViewSet(viewsets.ReadOnlyModelViewSet):
+    permission_classes = [common_permissions.ConditionalAuthentication]
+    serializer_class = serializers.LicenseSerializer
+    queryset = models.License.objects.all().order_by("code")
+    lookup_value_regex = ".*"
+
+    def get_queryset(self):
+        # ensure our licenses are up to date in DB
+        licenses.load(licenses.LICENSES)
+        return super().get_queryset()
+
+    def get_serializer(self, *args, **kwargs):
+        if len(args) == 0:
+            return super().get_serializer(*args, **kwargs)
+
+        # our serializer works with license dict, not License instances
+        # so we pass those instead
+        instance_or_qs = args[0]
+        try:
+            first_arg = instance_or_qs.conf
+        except AttributeError:
+            first_arg = [i.conf for i in instance_or_qs if i.conf]
+        return super().get_serializer(*((first_arg,) + args[1:]), **kwargs)
+
+
+class OembedView(views.APIView):
+    permission_classes = [common_permissions.ConditionalAuthentication]
+
+    def get(self, request, *args, **kwargs):
+        serializer = serializers.OembedSerializer(data=request.GET)
+        serializer.is_valid(raise_exception=True)
+        embed_data = serializer.save()
+        return Response(embed_data)

api/funkwhale_api/musicbrainz/views.py

@@ -1,5 +1,5 @@
 from rest_framework import viewsets
-from rest_framework.decorators import list_route
+from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
@@ -47,19 +47,19 @@ class ReleaseBrowse(APIView):
 class SearchViewSet(viewsets.ViewSet):
     permission_classes = [ConditionalAuthentication]
 
-    @list_route(methods=["get"])
+    @action(methods=["get"], detail=False)
     def recordings(self, request, *args, **kwargs):
         query = request.GET["query"]
         results = api.recordings.search(query)
         return Response(results)
 
-    @list_route(methods=["get"])
+    @action(methods=["get"], detail=False)
     def releases(self, request, *args, **kwargs):
         query = request.GET["query"]
         results = api.releases.search(query)
         return Response(results)
 
-    @list_route(methods=["get"])
+    @action(methods=["get"], detail=False)
     def artists(self, request, *args, **kwargs):
         query = request.GET["query"]
         results = api.artists.search(query)

api/funkwhale_api/playlists/factories.py

@@ -1,12 +1,12 @@
 import factory
 
-from funkwhale_api.factories import registry
+from funkwhale_api.factories import registry, NoUpdateOnCreate
 from funkwhale_api.music.factories import TrackFactory
 from funkwhale_api.users.factories import UserFactory
 
 
 @registry.register
-class PlaylistFactory(factory.django.DjangoModelFactory):
+class PlaylistFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     name = factory.Faker("name")
     user = factory.SubFactory(UserFactory)
 
@@ -15,7 +15,7 @@ class PlaylistFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class PlaylistTrackFactory(factory.django.DjangoModelFactory):
+class PlaylistTrackFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     playlist = factory.SubFactory(PlaylistFactory)
     track = factory.SubFactory(TrackFactory)
 

api/funkwhale_api/playlists/filters.py

@@ -7,8 +7,8 @@ from . import models
 
 
 class PlaylistFilter(filters.FilterSet):
-    q = filters.CharFilter(name="_", method="filter_q")
+    q = filters.CharFilter(field_name="_", method="filter_q")
-    playable = filters.BooleanFilter(name="_", method="filter_playable")
+    playable = filters.BooleanFilter(field_name="_", method="filter_playable")
 
     class Meta:
         model = models.Playlist

api/funkwhale_api/playlists/models.py

@@ -38,22 +38,21 @@ class PlaylistQuerySet(models.QuerySet):
         )
         return self.prefetch_related(plt_prefetch)
 
-    def annotate_playable_by_actor(self, actor):
+    def with_playable_plts(self, actor):
-        plts = (
+        return self.prefetch_related(
-            PlaylistTrack.objects.playable_by(actor)
+            models.Prefetch(
-            .filter(playlist=models.OuterRef("id"))
+                "playlist_tracks",
-            .order_by("id")
+                queryset=PlaylistTrack.objects.playable_by(actor),
-            .values("id")[:1]
+                to_attr="playable_plts",
+            )
         )
-        subquery = models.Subquery(plts)
-        return self.annotate(is_playable_by_actor=subquery)
 
     def playable_by(self, actor, include=True):
         plts = PlaylistTrack.objects.playable_by(actor, include)
         if include:
-            return self.filter(playlist_tracks__in=plts)
+            return self.filter(playlist_tracks__in=plts).distinct()
         else:
-            return self.exclude(playlist_tracks__in=plts)
+            return self.exclude(playlist_tracks__in=plts).distinct()
 
 
 class Playlist(models.Model):
@@ -148,7 +147,7 @@ class Playlist(models.Model):
 
 class PlaylistTrackQuerySet(models.QuerySet):
     def for_nested_serialization(self, actor=None):
-        tracks = music_models.Track.objects.annotate_playable_by_actor(actor)
+        tracks = music_models.Track.objects.with_playable_uploads(actor)
         tracks = tracks.select_related("artist", "album__artist")
         return self.prefetch_related(
             models.Prefetch("track", queryset=tracks, to_attr="_prefetched_track")
@@ -156,8 +155,8 @@ class PlaylistTrackQuerySet(models.QuerySet):
 
     def annotate_playable_by_actor(self, actor):
         tracks = (
-            music_models.Track.objects.playable_by(actor)
+            music_models.Upload.objects.playable_by(actor)
-            .filter(pk=models.OuterRef("track"))
+            .filter(track__pk=models.OuterRef("track"))
             .order_by("id")
             .values("id")[:1]
         )
@@ -167,9 +166,9 @@ class PlaylistTrackQuerySet(models.QuerySet):
     def playable_by(self, actor, include=True):
         tracks = music_models.Track.objects.playable_by(actor, include)
         if include:
-            return self.filter(track__pk__in=tracks)
+            return self.filter(track__pk__in=tracks).distinct()
         else:
-            return self.exclude(track__pk__in=tracks)
+            return self.exclude(track__pk__in=tracks).distinct()
 
 
 class PlaylistTrack(models.Model):

api/funkwhale_api/playlists/serializers.py

@@ -93,7 +93,7 @@ class PlaylistSerializer(serializers.ModelSerializer):
 
     def get_is_playable(self, obj):
         try:
-            return bool(obj.is_playable_by_actor)
+            return bool(obj.playable_plts)
         except AttributeError:
             return None
 

api/funkwhale_api/playlists/views.py

@@ -1,7 +1,7 @@
 from django.db import transaction
 from django.db.models import Count
 from rest_framework import exceptions, mixins, viewsets
-from rest_framework.decorators import detail_route
+from rest_framework.decorators import action
 from rest_framework.permissions import IsAuthenticatedOrReadOnly
 from rest_framework.response import Response
 
@@ -33,10 +33,10 @@ class PlaylistViewSet(
         IsAuthenticatedOrReadOnly,
     ]
     owner_checks = ["write"]
-    filter_class = filters.PlaylistFilter
+    filterset_class = filters.PlaylistFilter
     ordering_fields = ("id", "name", "creation_date", "modification_date")
 
-    @detail_route(methods=["get"])
+    @action(methods=["get"], detail=True)
     def tracks(self, request, *args, **kwargs):
         playlist = self.get_object()
         plts = playlist.playlist_tracks.all().for_nested_serialization(
@@ -46,7 +46,7 @@ class PlaylistViewSet(
         data = {"count": len(plts), "results": serializer.data}
         return Response(data, status=200)
 
-    @detail_route(methods=["post"])
+    @action(methods=["post"], detail=True)
     @transaction.atomic
     def add(self, request, *args, **kwargs):
         playlist = self.get_object()
@@ -67,7 +67,7 @@ class PlaylistViewSet(
         data = {"count": len(plts), "results": serializer.data}
         return Response(data, status=201)
 
-    @detail_route(methods=["delete"])
+    @action(methods=["delete"], detail=True)
     @transaction.atomic
     def clear(self, request, *args, **kwargs):
         playlist = self.get_object()
@@ -78,7 +78,7 @@ class PlaylistViewSet(
     def get_queryset(self):
         return self.queryset.filter(
             fields.privacy_level_query(self.request.user)
-        ).annotate_playable_by_actor(music_utils.get_actor_from_request(self.request))
+        ).with_playable_plts(music_utils.get_actor_from_request(self.request))
 
     def perform_create(self, serializer):
         return serializer.save(

api/funkwhale_api/providers/acoustid/__init__.py

@@ -1,27 +0,0 @@
-import acoustid
-
-from dynamic_preferences.registries import global_preferences_registry
-
-
-class Client(object):
-    def __init__(self, api_key):
-        self.api_key = api_key
-
-    def match(self, file_path):
-        return acoustid.match(self.api_key, file_path, parse=False)
-
-    def get_best_match(self, file_path):
-        results = self.match(file_path=file_path)
-        MIN_SCORE_FOR_MATCH = 0.8
-        try:
-            rows = results["results"]
-        except KeyError:
-            return
-        for row in rows:
-            if row["score"] >= MIN_SCORE_FOR_MATCH:
-                return row
-
-
-def get_acoustid_client():
-    manager = global_preferences_registry.manager()
-    return Client(api_key=manager["providers_acoustid__api_key"])

api/funkwhale_api/providers/acoustid/dynamic_preferences_registry.py

@@ -1,16 +0,0 @@
-from django import forms
-from dynamic_preferences.registries import global_preferences_registry
-from dynamic_preferences.types import Section, StringPreference
-
-acoustid = Section("providers_acoustid")
-
-
-@global_preferences_registry.register
-class APIKey(StringPreference):
-    section = acoustid
-    name = "api_key"
-    default = ""
-    verbose_name = "Acoustid API key"
-    help_text = "The API key used to query AcoustID. Get one at https://acoustid.org/new-application."
-    widget = forms.PasswordInput
-    field_kwargs = {"required": False}

api/funkwhale_api/radios/factories.py

@@ -1,11 +1,11 @@
 import factory
 
-from funkwhale_api.factories import registry
+from funkwhale_api.factories import registry, NoUpdateOnCreate
 from funkwhale_api.users.factories import UserFactory
 
 
 @registry.register
-class RadioFactory(factory.django.DjangoModelFactory):
+class RadioFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     name = factory.Faker("name")
     description = factory.Faker("paragraphs")
     user = factory.SubFactory(UserFactory)
@@ -16,7 +16,7 @@ class RadioFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register
-class RadioSessionFactory(factory.django.DjangoModelFactory):
+class RadioSessionFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     user = factory.SubFactory(UserFactory)
 
     class Meta:
@@ -24,7 +24,7 @@ class RadioSessionFactory(factory.django.DjangoModelFactory):
 
 
 @registry.register(name="radios.CustomRadioSession")
-class CustomRadioSessionFactory(factory.django.DjangoModelFactory):
+class CustomRadioSessionFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     user = factory.SubFactory(UserFactory)
     radio_type = "custom"
     custom_radio = factory.SubFactory(

api/funkwhale_api/radios/views.py

@@ -1,6 +1,6 @@
 from django.db.models import Q
 from rest_framework import mixins, permissions, status, viewsets
-from rest_framework.decorators import detail_route, list_route
+from rest_framework.decorators import action
 from rest_framework.response import Response
 
 from funkwhale_api.common import permissions as common_permissions
@@ -23,7 +23,7 @@ class RadioViewSet(
         permissions.IsAuthenticated,
         common_permissions.OwnerPermission,
     ]
-    filter_class = filtersets.RadioFilter
+    filterset_class = filtersets.RadioFilter
     owner_field = "user"
     owner_checks = ["write"]
 
@@ -40,7 +40,7 @@ class RadioViewSet(
     def perform_update(self, serializer):
         return serializer.save(user=self.request.user)
 
-    @detail_route(methods=["get"])
+    @action(methods=["get"], detail=True)
     def tracks(self, request, *args, **kwargs):
         radio = self.get_object()
         tracks = radio.get_candidates().for_nested_serialization()
@@ -50,14 +50,14 @@ class RadioViewSet(
             serializer = TrackSerializer(page, many=True)
             return self.get_paginated_response(serializer.data)
 
-    @list_route(methods=["get"])
+    @action(methods=["get"], detail=False)
     def filters(self, request, *args, **kwargs):
         serializer = serializers.FilterSerializer(
             filters.registry.exposed_filters, many=True
         )
         return Response(serializer.data)
 
-    @list_route(methods=["post"])
+    @action(methods=["post"], detail=False)
     def validate(self, request, *args, **kwargs):
         try:
             f_list = request.data["filters"]

api/funkwhale_api/subsonic/filters.py

@@ -4,7 +4,7 @@ from funkwhale_api.music import models as music_models
 
 
 class AlbumList2FilterSet(filters.FilterSet):
-    type = filters.CharFilter(name="_", method="filter_type")
+    type = filters.CharFilter(field_name="_", method="filter_type")
 
     class Meta:
         model = music_models.Album

api/funkwhale_api/subsonic/renderers.py

@@ -2,17 +2,30 @@ import xml.etree.ElementTree as ET
 
 from rest_framework import renderers
 
+import funkwhale_api
+
+
+def structure_payload(data):
+    payload = {
+        "status": "ok",
+        "version": "1.16.0",
+        "type": "funkwhale",
+        "funkwhaleVersion": funkwhale_api.__version__,
+    }
+    payload.update(data)
+    if "detail" in payload:
+        payload["error"] = {"code": 0, "message": payload.pop("detail")}
+    if "error" in payload:
+        payload["status"] = "failed"
+    return payload
+
 
 class SubsonicJSONRenderer(renderers.JSONRenderer):
     def render(self, data, accepted_media_type=None, renderer_context=None):
         if not data:
             # when stream view is called, we don't have any data
             return super().render(data, accepted_media_type, renderer_context)
-        final = {"subsonic-response": {"status": "ok", "version": "1.16.0"}}
+        final = {"subsonic-response": structure_payload(data)}
-        final["subsonic-response"].update(data)
-        if "error" in final:
-            # an error was returned
-            final["subsonic-response"]["status"] = "failed"
         return super().render(final, accepted_media_type, renderer_context)
 
 
@@ -23,15 +36,8 @@ class SubsonicXMLRenderer(renderers.JSONRenderer):
         if not data:
             # when stream view is called, we don't have any data
             return super().render(data, accepted_media_type, renderer_context)
-        final = {
+        final = structure_payload(data)
-            "xmlns": "http://subsonic.org/restapi",
+        final["xmlns"] = "http://subsonic.org/restapi"
-            "status": "ok",
-            "version": "1.16.0",
-        }
-        final.update(data)
-        if "error" in final:
-            # an error was returned
-            final["status"] = "failed"
         tree = dict_to_xml_tree("subsonic-response", final)
         return b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(
             tree, encoding="utf-8"

api/funkwhale_api/subsonic/serializers.py

@@ -226,6 +226,30 @@ def get_music_directory_data(artist):
     return data
 
 
+def get_folders(user):
+    return []
+
+
+def get_user_detail_data(user):
+    return {
+        "username": user.username,
+        "email": user.email,
+        "scrobblingEnabled": "true",
+        "adminRole": "false",
+        "settingsRole": "false",
+        "commentRole": "false",
+        "podcastRole": "false",
+        "coverArtRole": "false",
+        "shareRole": "false",
+        "uploadRole": "true",
+        "downloadRole": "true",
+        "playlistRole": "true",
+        "streamRole": "true",
+        "jukeboxRole": "true",
+        "folder": [f["id"] for f in get_folders(user)],
+    }
+
+
 class ScrobbleSerializer(serializers.Serializer):
     submission = serializers.BooleanField(default=True, required=False)
     id = serializers.PrimaryKeyRelatedField(

api/funkwhale_api/subsonic/views.py

@@ -1,20 +1,23 @@
 import datetime
+import functools
 
 from django.conf import settings
 from django.utils import timezone
 from rest_framework import exceptions
 from rest_framework import permissions as rest_permissions
 from rest_framework import renderers, response, viewsets
-from rest_framework.decorators import list_route
+from rest_framework.decorators import action
 from rest_framework.serializers import ValidationError
 
+import funkwhale_api
 from funkwhale_api.activity import record
-from funkwhale_api.common import preferences
+from funkwhale_api.common import preferences, utils as common_utils
 from funkwhale_api.favorites.models import TrackFavorite
 from funkwhale_api.music import models as music_models
 from funkwhale_api.music import utils
 from funkwhale_api.music import views as music_views
 from funkwhale_api.playlists import models as playlists_models
+from funkwhale_api.users import models as users_models
 
 from . import authentication, filters, negotiation, serializers
 
@@ -23,6 +26,7 @@ def find_object(
     queryset, model_field="pk", field="id", cast=int, filter_playable=False
 ):
     def decorator(func):
+        @functools.wraps(func)
         def inner(self, request, *args, **kwargs):
             data = request.GET or request.POST
             try:
@@ -55,7 +59,7 @@ def find_object(
 
             if filter_playable:
                 actor = utils.get_actor_from_request(request)
-                qs = qs.playable_by(actor).distinct()
+                qs = qs.playable_by(actor)
 
             try:
                 obj = qs.get(**{model_field: value})
@@ -95,7 +99,10 @@ class SubsonicViewSet(viewsets.GenericViewSet):
     def handle_exception(self, exc):
         # subsonic API sends 200 status code with custom error
         # codes in the payload
-        mapping = {exceptions.AuthenticationFailed: (40, "Wrong username or password.")}
+        mapping = {
+            exceptions.AuthenticationFailed: (40, "Wrong username or password."),
+            exceptions.NotAuthenticated: (10, "Required parameter is missing."),
+        }
         payload = {"status": "failed"}
         if exc.__class__ in mapping:
             code, message = mapping[exc.__class__]
@@ -105,12 +112,13 @@ class SubsonicViewSet(viewsets.GenericViewSet):
 
         return response.Response(payload, status=200)
 
-    @list_route(methods=["get", "post"], permission_classes=[])
+    @action(detail=False, methods=["get", "post"], permission_classes=[])
     def ping(self, request, *args, **kwargs):
         data = {"status": "ok", "version": "1.16.0"}
         return response.Response(data, status=200)
 
-    @list_route(
+    @action(
+        detail=False,
         methods=["get", "post"],
         url_name="get_license",
         permissions_classes=[],
@@ -121,6 +129,8 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         data = {
             "status": "ok",
             "version": "1.16.0",
+            "type": "funkwhale",
+            "funkwhaleVersion": funkwhale_api.__version__,
             "license": {
                 "valid": "true",
                 "email": "valid@valid.license",
@@ -129,7 +139,12 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         }
         return response.Response(data, status=200)
 
-    @list_route(methods=["get", "post"], url_name="get_artists", url_path="getArtists")
+    @action(
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_artists",
+        url_path="getArtists",
+    )
     def get_artists(self, request, *args, **kwargs):
         artists = music_models.Artist.objects.all().playable_by(
             utils.get_actor_from_request(request)
@@ -139,7 +154,12 @@ class SubsonicViewSet(viewsets.GenericViewSet):
 
         return response.Response(payload, status=200)
 
-    @list_route(methods=["get", "post"], url_name="get_indexes", url_path="getIndexes")
+    @action(
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_indexes",
+        url_path="getIndexes",
+    )
     def get_indexes(self, request, *args, **kwargs):
         artists = music_models.Artist.objects.all().playable_by(
             utils.get_actor_from_request(request)
@@ -149,7 +169,12 @@ class SubsonicViewSet(viewsets.GenericViewSet):
 
         return response.Response(payload, status=200)
 
-    @list_route(methods=["get", "post"], url_name="get_artist", url_path="getArtist")
+    @action(
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_artist",
+        url_path="getArtist",
+    )
     @find_object(music_models.Artist.objects.all(), filter_playable=True)
     def get_artist(self, request, *args, **kwargs):
         artist = kwargs.pop("obj")
@@ -158,7 +183,9 @@ class SubsonicViewSet(viewsets.GenericViewSet):
 
         return response.Response(payload, status=200)
 
-    @list_route(methods=["get", "post"], url_name="get_song", url_path="getSong")
+    @action(
+        detail=False, methods=["get", "post"], url_name="get_song", url_path="getSong"
+    )
     @find_object(music_models.Track.objects.all(), filter_playable=True)
     def get_song(self, request, *args, **kwargs):
         track = kwargs.pop("obj")
@@ -167,8 +194,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
 
         return response.Response(payload, status=200)
 
-    @list_route(
+    @action(
-        methods=["get", "post"], url_name="get_artist_info2", url_path="getArtistInfo2"
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_artist_info2",
+        url_path="getArtistInfo2",
     )
     @find_object(music_models.Artist.objects.all(), filter_playable=True)
     def get_artist_info2(self, request, *args, **kwargs):
@@ -176,7 +206,9 @@ class SubsonicViewSet(viewsets.GenericViewSet):
 
         return response.Response(payload, status=200)
 
-    @list_route(methods=["get", "post"], url_name="get_album", url_path="getAlbum")
+    @action(
+        detail=False, methods=["get", "post"], url_name="get_album", url_path="getAlbum"
+    )
     @find_object(
         music_models.Album.objects.select_related("artist"), filter_playable=True
     )
@@ -186,46 +218,88 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         payload = {"album": data}
         return response.Response(payload, status=200)
 
-    @list_route(methods=["get", "post"], url_name="stream", url_path="stream")
+    @action(detail=False, methods=["get", "post"], url_name="stream", url_path="stream")
     @find_object(music_models.Track.objects.all(), filter_playable=True)
     def stream(self, request, *args, **kwargs):
+        data = request.GET or request.POST
         track = kwargs.pop("obj")
         queryset = track.uploads.select_related("track__album__artist", "track__artist")
         upload = queryset.first()
         if not upload:
             return response.Response(status=404)
-        return music_views.handle_serve(upload=upload, user=request.user)
 
-    @list_route(methods=["get", "post"], url_name="star", url_path="star")
+        format = data.get("format", "raw")
+        if format == "raw":
+            format = None
+        return music_views.handle_serve(upload=upload, user=request.user, format=format)
+
+    @action(detail=False, methods=["get", "post"], url_name="star", url_path="star")
     @find_object(music_models.Track.objects.all())
     def star(self, request, *args, **kwargs):
         track = kwargs.pop("obj")
         TrackFavorite.add(user=request.user, track=track)
         return response.Response({"status": "ok"})
 
-    @list_route(methods=["get", "post"], url_name="unstar", url_path="unstar")
+    @action(detail=False, methods=["get", "post"], url_name="unstar", url_path="unstar")
     @find_object(music_models.Track.objects.all())
     def unstar(self, request, *args, **kwargs):
         track = kwargs.pop("obj")
         request.user.track_favorites.filter(track=track).delete()
         return response.Response({"status": "ok"})
 
-    @list_route(
+    @action(
-        methods=["get", "post"], url_name="get_starred2", url_path="getStarred2"
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_starred2",
+        url_path="getStarred2",
     )
     def get_starred2(self, request, *args, **kwargs):
         favorites = request.user.track_favorites.all()
         data = {"starred2": {"song": serializers.get_starred_tracks_data(favorites)}}
         return response.Response(data)
 
-    @list_route(methods=["get", "post"], url_name="get_starred", url_path="getStarred")
+    @action(
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_random_songs",
+        url_path="getRandomSongs",
+    )
+    def get_random_songs(self, request, *args, **kwargs):
+        data = request.GET or request.POST
+        actor = utils.get_actor_from_request(request)
+        queryset = music_models.Track.objects.all()
+        queryset = queryset.playable_by(actor)
+        try:
+            size = int(data["size"])
+        except (TypeError, KeyError, ValueError):
+            size = 50
+
+        queryset = (
+            queryset.playable_by(actor).prefetch_related("uploads").order_by("?")[:size]
+        )
+        data = {
+            "randomSongs": {
+                "song": serializers.GetSongSerializer(queryset, many=True).data
+            }
+        }
+        return response.Response(data)
+
+    @action(
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_starred",
+        url_path="getStarred",
+    )
     def get_starred(self, request, *args, **kwargs):
         favorites = request.user.track_favorites.all()
         data = {"starred": {"song": serializers.get_starred_tracks_data(favorites)}}
         return response.Response(data)
 
-    @list_route(
+    @action(
-        methods=["get", "post"], url_name="get_album_list2", url_path="getAlbumList2"
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_album_list2",
+        url_path="getAlbumList2",
     )
     def get_album_list2(self, request, *args, **kwargs):
         queryset = music_models.Album.objects.with_tracks_count().order_by(
@@ -252,7 +326,9 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         data = {"albumList2": {"album": serializers.get_album_list2_data(queryset)}}
         return response.Response(data)
 
-    @list_route(methods=["get", "post"], url_name="search3", url_path="search3")
+    @action(
+        detail=False, methods=["get", "post"], url_name="search3", url_path="search3"
+    )
     def search3(self, request, *args, **kwargs):
         data = request.GET or request.POST
         query = str(data.get("query", "")).replace("*", "")
@@ -310,12 +386,16 @@ class SubsonicViewSet(viewsets.GenericViewSet):
                     utils.get_query(query, c["search_fields"])
                 )
             queryset = queryset.playable_by(actor)
+            queryset = common_utils.order_for_search(queryset, c["search_fields"][0])
             queryset = queryset[offset : offset + size]
             payload["searchResult3"][c["subsonic"]] = c["serializer"](queryset)
         return response.Response(payload)
 
-    @list_route(
+    @action(
-        methods=["get", "post"], url_name="get_playlists", url_path="getPlaylists"
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_playlists",
+        url_path="getPlaylists",
     )
     def get_playlists(self, request, *args, **kwargs):
         playlists = request.user.playlists.with_tracks_count().select_related("user")
@@ -326,8 +406,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         }
         return response.Response(data)
 
-    @list_route(
+    @action(
-        methods=["get", "post"], url_name="get_playlist", url_path="getPlaylist"
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_playlist",
+        url_path="getPlaylist",
     )
     @find_object(playlists_models.Playlist.objects.with_tracks_count())
     def get_playlist(self, request, *args, **kwargs):
@@ -335,8 +418,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         data = {"playlist": serializers.get_playlist_detail_data(playlist)}
         return response.Response(data)
 
-    @list_route(
+    @action(
-        methods=["get", "post"], url_name="update_playlist", url_path="updatePlaylist"
+        detail=False,
+        methods=["get", "post"],
+        url_name="update_playlist",
+        url_path="updatePlaylist",
     )
     @find_object(lambda request: request.user.playlists.all(), field="playlistId")
     def update_playlist(self, request, *args, **kwargs):
@@ -377,8 +463,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         data = {"status": "ok"}
         return response.Response(data)
 
-    @list_route(
+    @action(
-        methods=["get", "post"], url_name="delete_playlist", url_path="deletePlaylist"
+        detail=False,
+        methods=["get", "post"],
+        url_name="delete_playlist",
+        url_path="deletePlaylist",
     )
     @find_object(lambda request: request.user.playlists.all())
     def delete_playlist(self, request, *args, **kwargs):
@@ -387,8 +476,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         data = {"status": "ok"}
         return response.Response(data)
 
-    @list_route(
+    @action(
-        methods=["get", "post"], url_name="create_playlist", url_path="createPlaylist"
+        detail=False,
+        methods=["get", "post"],
+        url_name="create_playlist",
+        url_path="createPlaylist",
     )
     def create_playlist(self, request, *args, **kwargs):
         data = request.GET or request.POST
@@ -426,7 +518,43 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         data = {"playlist": serializers.get_playlist_detail_data(playlist)}
         return response.Response(data)
 
-    @list_route(
+    @action(
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_avatar",
+        url_path="getAvatar",
+    )
+    @find_object(
+        queryset=users_models.User.objects.exclude(avatar=None).exclude(avatar=""),
+        model_field="username__iexact",
+        field="username",
+        cast=str,
+    )
+    def get_avatar(self, request, *args, **kwargs):
+        user = kwargs.pop("obj")
+        mapping = {"nginx": "X-Accel-Redirect", "apache2": "X-Sendfile"}
+        path = music_views.get_file_path(user.avatar)
+        file_header = mapping[settings.REVERSE_PROXY_TYPE]
+        # let the proxy set the content-type
+        r = response.Response({}, content_type="")
+        r[file_header] = path
+        return r
+
+    @action(
+        detail=False, methods=["get", "post"], url_name="get_user", url_path="getUser"
+    )
+    @find_object(
+        queryset=lambda request: users_models.User.objects.filter(pk=request.user.pk),
+        model_field="username__iexact",
+        field="username",
+        cast=str,
+    )
+    def get_user(self, request, *args, **kwargs):
+        data = {"user": serializers.get_user_detail_data(request.user)}
+        return response.Response(data)
+
+    @action(
+        detail=False,
         methods=["get", "post"],
         url_name="get_music_folders",
         url_path="getMusicFolders",
@@ -435,8 +563,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         data = {"musicFolders": {"musicFolder": [{"id": 1, "name": "Music"}]}}
         return response.Response(data)
 
-    @list_route(
+    @action(
-        methods=["get", "post"], url_name="get_cover_art", url_path="getCoverArt"
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_cover_art",
+        url_path="getCoverArt",
     )
     def get_cover_art(self, request, *args, **kwargs):
         data = request.GET or request.POST
@@ -472,7 +603,9 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         r[file_header] = path
         return r
 
-    @list_route(methods=["get", "post"], url_name="scrobble", url_path="scrobble")
+    @action(
+        detail=False, methods=["get", "post"], url_name="scrobble", url_path="scrobble"
+    )
     def scrobble(self, request, *args, **kwargs):
         data = request.GET or request.POST
         serializer = serializers.ScrobbleSerializer(

api/funkwhale_api/users/admin.py

@@ -51,7 +51,7 @@ class UserAdmin(AuthUserAdmin):
         "privacy_level",
         "permission_settings",
         "permission_library",
-        "permission_federation",
+        "permission_moderation",
     ]
 
     fieldsets = (
@@ -67,10 +67,9 @@ class UserAdmin(AuthUserAdmin):
                     "is_active",
                     "is_staff",
                     "is_superuser",
-                    "permission_upload",
                     "permission_library",
                     "permission_settings",
-                    "permission_federation",
+                    "permission_moderation",
                 )
             },
         ),