MirrorMirror
/
funkwhale
mirror of https://dev.funkwhale.audio/funkwhale/funkwhale.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release/0.18'

This commit is contained in:
Eliot Berriot 2019-01-22 12:04:58 +01:00
parent de7b131468 f44d2b0694
commit 338e1a8520
No known key found for this signature in database
GPG Key ID: DD6965E2476E5C27
340 changed files with 45695 additions and 56594 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.env.dev
View File

@ -12,3 +12,9 @@ MUSIC_DIRECTORY_PATH=/music
BROWSABLE_API_ENABLED=True
FORWARDED_PROTO=http
LDAP_ENABLED=False
# Uncomment this if you're using traefik/https
# FORCE_HTTPS_URLS=True
# Customize to your needs
POSTGRES_VERSION=11

2
.gitignore vendored
View File

@ -93,4 +93,6 @@ po/*.po
docs/swagger
_build
front/src/translations.json
front/src/translations/*.json
front/locales/en_US/LC_MESSAGES/app.po
*.prof

56
.gitlab-ci.yml
View File

@ -1,7 +1,8 @@
variables:
IMAGE_NAME: funkwhale/funkwhale
IMAGE: $IMAGE_NAME:$CI_COMMIT_REF_NAME
IMAGE_LATEST: $IMAGE_NAME:latest
ALL_IN_ONE_IMAGE_NAME: funkwhale/all-in-one
ALL_IN_ONE_IMAGE: $ALL_IN_ONE_IMAGE_NAME:$CI_COMMIT_REF_NAME
PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
PYTHONDONTWRITEBYTECODE: "true"
REVIEW_DOMAIN: preview.funkwhale.audio
@ -131,16 +132,15 @@ flake8:
test_api:
services:
- postgres:9.4
- postgres:11
- redis:3
stage: test
image: funkwhale/funkwhale:latest
image: funkwhale/funkwhale:develop
cache:
key: "$CI_PROJECT_ID__pip_cache"
paths:
- "$PIP_CACHE_DIR"
variables:
DJANGO_ALLOWED_HOSTS: "localhost"
DATABASE_URL: "postgresql://postgres@postgres/postgres"
FUNKWHALE_URL: "https://funkwhale.ci"
DJANGO_SETTINGS_MODULE: config.settings.local
@ -148,11 +148,10 @@ test_api:
- branches
before_script:
- cd api
- apt-get update
- grep "^[^#;]" requirements.apt | grep -Fv "python3-dev" | xargs apt-get install -y --no-install-recommends
- pip install -r requirements/base.txt
- pip install -r requirements/local.txt
- pip install -r requirements/test.txt
- sed -i '/Pillow/d' requirements/base.txt
- pip3 install -r requirements/base.txt
- pip3 install -r requirements/local.txt
- pip3 install -r requirements/test.txt
script:
- pytest --cov=funkwhale_api tests/
tags:
@ -166,7 +165,7 @@ test_front:
only:
- branches
script:
- yarn install
- yarn install --check-files
- yarn test:unit
cache:
key: "funkwhale__front_dependencies"
@ -194,11 +193,6 @@ build_front:
# cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169
- yarn build | tee /dev/stderr | (! grep -i 'ERROR in')
- chmod -R 755 dist
cache:
key: "funkwhale__front_dependencies"
paths:
- front/node_modules
- front/yarn.lock
artifacts:
name: "front_${CI_COMMIT_REF_NAME}"
paths:
@ -207,6 +201,7 @@ build_front:
- tags@funkwhale/funkwhale
- master@funkwhale/funkwhale
- develop@funkwhale/funkwhale
tags:
- docker
@ -236,9 +231,11 @@ pages:
docker_release:
stage: deploy
image: bash
before_script:
- docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
- cp -r front/dist api/frontend
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
- cd api
script:
- docker build -t $IMAGE .
@ -249,15 +246,42 @@ docker_release:
tags:
- docker-build
docker_all_in_one_release:
stage: deploy
image: bash
variables:
ALL_IN_ONE_REF: master
ALL_IN_ONE_ARTIFACT_URL: https://github.com/thetarkus/docker-funkwhale/archive/$ALL_IN_ONE_REF.zip
BUILD_PATH: all_in_one
before_script:
- docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
script:
- wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
- unzip -o all_in_one.zip -d tmpdir
- mv tmpdir/docker-funkwhale-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
- cp -r api $BUILD_PATH/src/api
- cp -r front $BUILD_PATH/src/front
- cd $BUILD_PATH
- ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
- docker build -t $ALL_IN_ONE_IMAGE .
- docker push $ALL_IN_ONE_IMAGE
only:
- develop@funkwhale/funkwhale
- tags@funkwhale/funkwhale
tags:
- docker-build
build_api:
# Simply publish a zip containing api/ directory
stage: deploy
image: busybox
image: bash
artifacts:
name: "api_${CI_COMMIT_REF_NAME}"
paths:
- api
script:
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
- chmod -R 750 api
- echo Done!
only:

6
.gitlab/issue_templates/Bug.md
View File

@ -34,6 +34,12 @@ Describe the expected behaviour.
## Context
<!--
The version of your instance can be found on the footer : Source code (x.y)
-->
**Funkwhale version(s) affected**: x.y
<!--
If relevant, share additional context here like:

293
CHANGELOG
View File

@ -10,6 +10,297 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.
.. towncrier
0.18 "Naomi" (2019-01-22)
-------------------------
This release is dedicated to Naomi, an early contributor and beta tester of Funkwhale.
Her positivity, love and support have been incredibly helpful and helped shape the project
as you can enjoy it today. Thank you so much Naomi <3
Upgrade instructions are available at
https://docs.funkwhale.audio/index.html, ensure you also execute the intructions
marked with ``[manual action required]`` and ``[manual action suggested]``.
See ``Full changelog`` below for an exhaustive list of changes!
Audio transcoding is back!
^^^^^^^^^^^^^^^^^^^^^^^^^^
After removal of our first, buggy transcoding implementation, we're proud to announce
that this feature is back. It is enabled by default, and can be configured/disabled
in your instance settings!
This feature works in the browser, with federated/non-federated tracks and using Subsonic clients.
Transcoded tracks are generated on the fly, and cached for a configurable amount of time,
to reduce the load on the server.
Licensing and copyright information
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Funkwhale is now able to parse copyright and license data from file and store
this information. Apart from displaying it on each track detail page,
no additional behaviour is currently implemented to use this new data, but this
will change in future releases.
License and copyright data is also broadcasted over federation.
License matching is done on the content of the ``License`` tag in the files,
with a fallback on the ``Copyright`` tag.
Funkwhale will successfully extract licensing data for the following licenses:
- Creative Commons 0 (Public Domain)
- Creative Commons 1.0 (All declinations)
- Creative Commons 2.0 (All declinations)
- Creative Commons 2.5 (All declinations and countries)
- Creative Commons 3.0 (All declinations and countries)
- Creative Commons 4.0 (All declinations)
Support for other licenses such as Art Libre or WTFPL will be added in future releases.
Instance-level moderation tools
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
This release includes a first set of moderation tools that will give more control
to admins about the way their instance federate with other instance and accounts on the network.
Using these tools, it's now possible to:
- Browse known accounts and domains, and associated data (storage size, software version, etc.)
- Purge data belonging to given accounts and domains
- Block or partially restrict interactions with any account or domain
All those features are usable using a brand new "moderation" permission, meaning
you can appoints one or nultiple moderators to help with this task.
I'd like to thank all Mastodon contributors, because some of the these tools are heavily
inspired from what's being done in Mastodon. Thank you so much!
Iframe widget to embed public tracks and albums [manual action required]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Funkwhale now support embedding a lightweight audio player on external websites
for album and tracks that are available in public libraries. Important pages,
such as artist, album and track pages also include OpenGraph tags that will
enable previews on compatible apps (like sharing a Funkwhale track link on Mastodon
or Twitter).
To achieve that, we had to tweak the way Funkwhale front-end is served. You'll have
to modify your nginx configuration when upgrading to keep your instance working.
**On docker setups**, edit your ``/srv/funkwhale/nginx/funkwhale.template`` and replace
the ``location /api/`` and `location /` blocks by the following snippets::
location / {
include /etc/nginx/funkwhale_proxy.conf;
# this is needed if you have file import via upload enabled
client_max_body_size ${NGINX_MAX_BODY_SIZE};
proxy_pass http://funkwhale-api/;
}
location /front/ {
alias /frontend/;
}
The change of configuration will be picked when restarting your nginx container.
**On non-docker setups**, edit your ``/etc/nginx/sites-available/funkwhale.conf`` file,
and replace the ``location /api/`` and `location /` blocks by the following snippets::
location / {
include /etc/nginx/funkwhale_proxy.conf;
# this is needed if you have file import via upload enabled
client_max_body_size ${NGINX_MAX_BODY_SIZE};
proxy_pass http://funkwhale-api/;
}
location /front/ {
alias ${FUNKWHALE_FRONTEND_PATH}/;
}
Replace ``${FUNKWHALE_FRONTEND_PATH}`` by the corresponding variable from your .env file,
which should be ``/srv/funkwhale/front/dist`` by default, then reload your nginx process with
``sudo systemctl reload nginx``.
Alternative docker deployment method
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Thanks to the awesome done by @thetarkus at https://github.com/thetarkus/docker-funkwhale,
we're now able to provide an alternative and easier Docker deployment method!
In contrast with our current, multi-container offer, this method integrates
all Funkwhale processes and services (database, redis, etc.) into a single, easier to deploy container.
Both method will coexist in parallel, as each one has pros and cons. You can learn more
about this exciting new deployment option by visiting https://docs.funkwhale.audio/installation/docker.html!
Automatically load .env file
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
On non-docker deployments, earlier versions required you to source
the config/.env file before launching any Funkwhale command, with ``export $(cat config/.env | grep -v ^# | xargs)``
This led to more complex and error prode deployment / setup.
This is not the case anymore, and Funkwhale will automatically load this file if it's available.
Delete pre 0.17 federated tracks [manual action suggested]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
If you were using Funkwhale before the 0.17 release and federated with other instances,
it's possible that you still have some unplayable federated files in the database.
To purge the database of those entries, you can run the following command:
On docker setups::
docker-compose run --rm api python manage.py script delete_pre_017_federated_uploads --no-input
On non-docker setups::
python manage.py script delete_pre_017_federated_uploads --no-input
Enable gzip compression [manual action suggested]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Gzip compression will be enabled on new instances by default
and will reduce the amount of bandwidth consumed by your instance.
If you with to benefit from gzip compression on your instance,
edit your reverse proxy virtualhost file (located at ``/etc/nginx/sites-available/funkwhale.conf``) and add the following snippet
in the server block, then reload your nginx server::
server {
# ... exiting configuration
# compression settings
gzip on;
gzip_comp_level 5;
gzip_min_length 256;
gzip_proxied any;
gzip_vary on;
gzip_types
application/atom+xml
application/javascript
application/json
application/ld+json
application/activity+json
application/manifest+json
application/rss+xml
application/vnd.geo+json
application/vnd.ms-fontobject
application/x-font-ttf
application/x-web-app-manifest+json
application/xhtml+xml
application/xml
font/opentype
image/bmp
image/svg+xml
image/x-icon
text/cache-manifest
text/css
text/plain
text/vcard
text/vnd.rim.location.xloc
text/vtt
text/x-component
text/x-cross-domain-policy;
# end of compression settings
}
Full changelog
^^^^^^^^^^^^^^
Features:
- Allow embedding of albums and tracks available in public libraries via an <iframe> (#578)
- Audio transcoding is back! (#272)
- First set of instance level moderation tools (#580, !521)
- Store licensing and copyright information from file metadata, if available (#308)
Enhancements:
- Add UI elements for multi-disc albums (#631)
- Added alternative funkwhale/all-in-one docker image (#614)
- Broadcast library updates (name, description, visibility) over federation
- Based Docker image on alpine to have a smaller (and faster to build) image
- Improved front-end performance by stripping unused dependencies, reducing bundle size
and enabling gzip compression
- Improved accessibility by using main/section/nav tags and aria-labels in most critical places (#612)
- The progress bar in the player now display loading state / buffer loading (#586)
- Added "type: funkwhale" and "funkwhale-version" in Subsonic responses (#573)
- Documented keyboard shortcuts, list is now available by pressing "h" or in the footer (#611)
- Documented which Subsonic endpoints are implemented (#575)
- Hide invitation code field during signup when it's not required (#410)
- Importer will now pick embedded images in files with OTHER type if no COVER_FRONT is present
- Improved keyboard accessibility on player, queue and various controls (#576)
- Improved performance when listing playable tracks, albums and artists
- Increased default upload limit from 30 to 100MB (#654)
- Load env file in config/.env automatically to avoid sourcing it by hand (#626)
- More resilient date parsing during audio import, will not crash anymore on
invalid dates (#622)
- Now start radios immediatly, skipping any existing tracks in queue (#585)
- Officially support connecting to a password protected redis server, with
the redis://:password@localhost:6379/0 scheme (#640)
- Performance improvement when fetching favorites, down to a single, small http request
- Removed "Activity" page, since all the data is available on the "Browse" page (#600)
- Removed the need to specify the DJANGO_ALLOWED_HOSTS variable
- Restructured the footer, added useful links and removed unused content
- Show short entries first in search results to improve UX
- Store disc number and order tracks by disc number / position) (#507)
- Strip EXIF metadata from uploaded avatars to avoid leaking private data (#374)
- Support blind key rotation in HTTP Signatures (#658)
- Support setting a server URL in settings.json (#650)
- Updated default docker postgres version from 9.4 to 11 (#656)
- Updated lots of dependencies (especially django 2.0->2.1), and removed unused dependencies (#657)
- Improved test suite speed by reducing / disabling expensive operations (#648)
Bugfixes:
- Fixed parsing of embedded file cover for ogg files tagged with MusicBrainz (#469)
- Upgraded core dependencies to fix websocket/messaging issues and possible memory leaks (#643)
- Fix ".None" extension when downloading Flac file (#473)
- Fixed None extension when downloading an in-place imported file (#621)
- Added a script to prune pre 0.17 federated tracks (#564)
- Advertise public libraries properly in ActivityPub representations (#553)
- Allow opus file upload (#598)
- Do not display "view on MusicBrainz" button if we miss the mbid (#422)
- Do not try to create unaccent extension if it's already present (#663)
- Ensure admin links in sidebar are displayed for users with relavant permissions, and only them (#597)
- Fix broken websocket connexion under Chrome browser (#589)
- Fix play button not starting playback with empty queue (#632)
- Fixed a styling inconsistency on about page when instance description was missing (#659)
- Fixed a UI discrepency in playlist tracks count (#647)
- Fixed greyed tracks in radio builder and detail page (#637)
- Fixed inconsistencies in subsonic error responses (#616)
- Fixed incorrect icon for "next track" in player control (#613)
- Fixed malformed search string when redirecting to LyricsWiki (#608)
- Fixed missing track count on various library cards (#581)
- Fixed skipped track when appending multiple tracks to the queue under certain conditions (#209)
- Fixed wrong album/track count on artist page (#599)
- Hide unplayable/emtpy playlists in "Browse playlist" pages (#424)
- Initial UI render using correct language from browser (#644)
- Invalid URI for reverse proxy websocket with apache (#617)
- Properly encode Wikipedia and lyrics search urls (#470)
- Refresh profile after user settings update to avoid cache issues (#606)
- Use role=button instead of empty links for player controls (#610)
Documentation:
- Deploy documentation from the master branch instead of the develop branch to avoid inconsistencies (#642)
- Document how to find and use library id when importing files in CLI (#562)
- Fix documentation typos (#645)
0.17 (2018-10-07)
-----------------
@ -120,7 +411,7 @@ Then, add the following block at the end of your docker-compose.yml file::
- .env
environment:
# Override those variables in your .env file if needed
- "NGINX_MAX_BODY_SIZE=${NGINX_MAX_BODY_SIZE-30M}"
- "NGINX_MAX_BODY_SIZE=${NGINX_MAX_BODY_SIZE-100M}"
volumes:
- "./nginx/funkwhale.template:/etc/nginx/conf.d/funkwhale.template:ro"
- "./nginx/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro"

12
CONTRIBUTING.rst
View File

@ -35,18 +35,20 @@ Setup front-end only development environment
cd funkwhale
cd front
2. Install [nodejs](https://nodejs.org/en/download/package-manager/) and [yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)
2. Install `nodejs <https://nodejs.org/en/download/package-manager/>`_ and `yarn <https://yarnpkg.com/lang/en/docs/install/#debian-stable>`_
3. Install the dependencies::
yarn install
4. Launch the development server::
# this will serve the front-end on http://localhost:8000
# this will serve the front-end on http://localhost:8000/front/
VUE_PORT=8000 yarn serve
5. Make the front-end talk with an existing server (like https://demo.funkwhale.audio),
by clicking on the corresponding link in the footer
6. Start hacking!
Setup your development environment
@ -307,7 +309,7 @@ A typical fragment looks like that:
Fixed broken audio player on Chrome 42 for ogg files (#567)
If the work fixes one or more issues, the issue number should be included at the
end of the fragment (``(#567)`` is the issue number in the previous example.
end of the fragment (``(#567)`` is the issue number in the previous example).
If your work is not related to a specific issue, use the merge request
identifier instead, like this:
@ -389,7 +391,7 @@ This is regular pytest, so you can use any arguments/options that pytest usually
# Stop on first failure
docker-compose -f dev.yml run --rm api pytest -x
# Run a specific test file
docker-compose -f dev.yml run --rm api pytest tests/test_acoustid.py
docker-compose -f dev.yml run --rm api pytest tests/music/test_models.py
Writing tests
^^^^^^^^^^^^^
@ -507,7 +509,7 @@ useful when testing components that depend on each other:
# here, we ensure no email was sent
mocked_notify.assert_not_called()
Views: you can find some readable views tests in :file:`tests/users/test_views.py`
Views: you can find some readable views tests in file: ``api/tests/users/test_views.py``
.. note::

60
api/Dockerfile
View File

@ -1,27 +1,47 @@
FROM python:3.6
FROM alpine:3.8
ENV PYTHONUNBUFFERED 1
RUN \
echo 'installing dependencies' && \
apk add \
bash \
git \
gettext \
musl-dev \
gcc \
postgresql-dev \
python3-dev \
py3-psycopg2 \
py3-pillow \
libldap \
ffmpeg \
libpq \
libmagic \
libffi-dev \
zlib-dev \
openldap-dev && \
\
\
ln -s /usr/bin/python3 /usr/bin/python
# Requirements have to be pulled and installed here, otherwise caching won't work
RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
COPY ./requirements.apt /requirements.apt
RUN apt-get update; \
grep "^[^#;]" requirements.apt | \
grep -Fv "python3-dev" | \
xargs apt-get install -y --no-install-recommends; \
rm -rf /usr/share/doc/* /usr/share/locale/*
RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
RUN mkdir /requirements
COPY ./requirements/base.txt /requirements/base.txt
RUN pip install -r /requirements/base.txt
COPY ./requirements/production.txt /requirements/production.txt
RUN pip install -r /requirements/production.txt
RUN \
echo 'fixing requirements file for alpine' && \
sed -i '/Pillow/d' /requirements/base.txt && \
\
\
echo 'installing pip requirements' && \
pip3 install --no-cache-dir --upgrade pip && \
pip3 install --no-cache-dir setuptools wheel && \
pip3 install --no-cache-dir -r /requirements/base.txt
COPY . /app
# Since youtube-dl code is updated fairly often, we split it here
RUN pip install --upgrade youtube-dl
WORKDIR /app
ARG install_dev_deps=0
COPY ./requirements/*.txt /requirements/
RUN \
if [ "$install_dev_deps" = "1" ] ; then echo "Installing dev dependencies" && pip3 install --no-cache-dir -r /requirements/local.txt -r /requirements/test.txt ; else echo "Skipping dev deps installation" ; fi
ENTRYPOINT ["./compose/django/entrypoint.sh"]
CMD ["./compose/django/daphne.sh"]
COPY . /app
WORKDIR /app

2
api/compose/django/daphne.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash -eux
python /app/manage.py collectstatic --noinput
/usr/local/bin/daphne -b 0.0.0.0 -p 5000 config.asgi:application --proxy-headers
daphne -b 0.0.0.0 -p 5000 config.asgi:application --proxy-headers

2
api/compose/django/dev-entrypoint.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash
#!/bin/sh
set -e
exec "$@"

2
api/compose/django/entrypoint.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh
set -e
# This entrypoint is used to play nicely with the current cookiecutter configuration.
# Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple

6
api/config/api_urls.py
View File

@ -10,7 +10,7 @@ from funkwhale_api.playlists import views as playlists_views
from funkwhale_api.subsonic.views import SubsonicViewSet
router = routers.SimpleRouter()
router.register(r"settings", GlobalPreferencesViewSet, base_name="settings")
router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
router.register(r"activity", activity_views.ActivityViewSet, "activity")
router.register(r"tags", views.TagViewSet, "tags")
router.register(r"tracks", views.TrackViewSet, "tracks")
@ -19,6 +19,7 @@ router.register(r"libraries", views.LibraryViewSet, "libraries")
router.register(r"listen", views.ListenViewSet, "listen")
router.register(r"artists", views.ArtistViewSet, "artists")
router.register(r"albums", views.AlbumViewSet, "albums")
router.register(r"licenses", views.LicenseViewSet, "licenses")
router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
router.register(
r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks"
@ -26,10 +27,11 @@ router.register(
v1_patterns = router.urls
subsonic_router = routers.SimpleRouter(trailing_slash=False)
subsonic_router.register(r"subsonic/rest", SubsonicViewSet, base_name="subsonic")
subsonic_router.register(r"subsonic/rest", SubsonicViewSet, basename="subsonic")
v1_patterns += [
url(r"^oembed/$", views.OembedView.as_view(), name="oembed"),
url(
r"^instance/",
include(("funkwhale_api.instance.urls", "instance"), namespace="instance"),

53
api/config/settings/common.py
View File

@ -13,7 +13,7 @@ from __future__ import absolute_import, unicode_literals
import datetime
import logging
from urllib.parse import urlparse, urlsplit
from urllib.parse import urlsplit
import environ
from celery.schedules import crontab
@ -69,12 +69,23 @@ else:
FUNKWHALE_HOSTNAME = _parsed.netloc
FUNKWHALE_PROTOCOL = _parsed.scheme
FUNKWHALE_PROTOCOL = FUNKWHALE_PROTOCOL.lower()
FUNKWHALE_HOSTNAME = FUNKWHALE_HOSTNAME.lower()
FUNKWHALE_URL = "{}://{}".format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME)
FUNKWHALE_SPA_HTML_ROOT = env(
"FUNKWHALE_SPA_HTML_ROOT", default=FUNKWHALE_URL + "/front/"
)
FUNKWHALE_SPA_HTML_CACHE_DURATION = env.int(
"FUNKWHALE_SPA_HTML_CACHE_DURATION", default=60 * 15
)
FUNKWHALE_EMBED_URL = env(
"FUNKWHALE_EMBED_URL", default=FUNKWHALE_SPA_HTML_ROOT + "embed.html"
)
APP_NAME = "Funkwhale"
# XXX: deprecated, see #186
FEDERATION_ENABLED = env.bool("FEDERATION_ENABLED", default=True)
FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME)
FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME).lower()
# XXX: deprecated, see #186
FEDERATION_COLLECTION_PAGE_SIZE = env.int("FEDERATION_COLLECTION_PAGE_SIZE", default=50)
# XXX: deprecated, see #186
@ -83,7 +94,7 @@ FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
)
# XXX: deprecated, see #186
FEDERATION_ACTOR_FETCH_DELAY = env.int("FEDERATION_ACTOR_FETCH_DELAY", default=60 * 12)
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS")
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNAME]
# APP CONFIGURATION
# ------------------------------------------------------------------------------
@ -145,10 +156,10 @@ LOCAL_APPS = (
"funkwhale_api.requests",
"funkwhale_api.favorites",
"funkwhale_api.federation",
"funkwhale_api.moderation",
"funkwhale_api.radios",
"funkwhale_api.history",
"funkwhale_api.playlists",
"funkwhale_api.providers.acoustid",
"funkwhale_api.subsonic",
)
@ -159,7 +170,7 @@ INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
# MIDDLEWARE CONFIGURATION
# ------------------------------------------------------------------------------
MIDDLEWARE = (
# Make sure djangosecure.middleware.SecurityMiddleware is listed first
"funkwhale_api.common.middleware.SPAFallbackMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"corsheaders.middleware.CorsMiddleware",
"django.middleware.common.CommonMiddleware",
@ -305,8 +316,7 @@ FILE_UPLOAD_PERMISSIONS = 0o644
# URL Configuration
# ------------------------------------------------------------------------------
ROOT_URLCONF = "config.urls"
# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = "config.wsgi.application"
SPA_URLCONF = "config.spa_urls"
ASGI_APPLICATION = "config.routing.application"
# This ensures that Django will be able to detect a secure connection
@ -315,7 +325,7 @@ SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
# AUTHENTICATION CONFIGURATION
# ------------------------------------------------------------------------------
AUTHENTICATION_BACKENDS = (
"django.contrib.auth.backends.ModelBackend",
"funkwhale_api.users.auth_backends.ModelBackend",
"allauth.account.auth_backends.AuthenticationBackend",
)
SESSION_COOKIE_HTTPONLY = False
@ -400,15 +410,20 @@ if AUTH_LDAP_ENABLED:
AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify"
CACHE_DEFAULT = "redis://127.0.0.1:6379/0"
CACHES = {"default": env.cache_url("CACHE_URL", default=CACHE_DEFAULT)}
CACHES = {
"default": env.cache_url("CACHE_URL", default=CACHE_DEFAULT),
"local": {
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
"LOCATION": "local-cache",
},
}
CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
cache_url = urlparse(CACHES["default"]["LOCATION"])
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {"hosts": [(cache_url.hostname, cache_url.port)]},
"CONFIG": {"hosts": [CACHES["default"]["LOCATION"]]},
}
}
@ -435,7 +450,12 @@ CELERY_BEAT_SCHEDULE = {
"task": "federation.clean_music_cache",
"schedule": crontab(hour="*/2"),
"options": {"expires": 60 * 2},
}
},
"music.clean_transcoding_cache": {
"task": "music.clean_transcoding_cache",
"schedule": crontab(hour="*"),
"options": {"expires": 60 * 2},
},
}
JWT_AUTH = {
@ -516,6 +536,7 @@ MUSICBRAINZ_HOSTNAME = env("MUSICBRAINZ_HOSTNAME", default="musicbrainz.org")
# Custom Admin URL, use {% url 'admin:index' %}
ADMIN_URL = env("DJANGO_ADMIN_URL", default="^api/admin/")
CSRF_USE_SESSIONS = True
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
# Playlist settings
# XXX: deprecated, see #186
@ -570,3 +591,9 @@ VERSATILEIMAGEFIELD_RENDITION_KEY_SETS = {
]
}
VERSATILEIMAGEFIELD_SETTINGS = {"create_images_on_demand": False}
RSA_KEY_SIZE = 2048
# for performance gain in tests, since we don't need to actually create the
# thumbnails
CREATE_IMAGE_THUMBNAILS = env.bool("CREATE_IMAGE_THUMBNAILS", default=True)
# we rotate actor keys at most every two days by default
ACTOR_KEY_ROTATION_DELAY = env.int("ACTOR_KEY_ROTATION_DELAY", default=3600 * 48)

19
api/config/settings/local.py
View File

@ -14,6 +14,7 @@ from .common import * # noqa
# DEBUG
# ------------------------------------------------------------------------------
DEBUG = env.bool("DJANGO_DEBUG", default=True)
FORCE_HTTPS_URLS = env.bool("FORCE_HTTPS_URLS", default=False)
TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG
# SECRET CONFIGURATION
@ -31,7 +32,6 @@ EMAIL_PORT = 1025
# django-debug-toolbar
# ------------------------------------------------------------------------------
MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)
# INTERNAL_IPS = ('127.0.0.1', '10.0.2.2',)
@ -39,20 +39,24 @@ DEBUG_TOOLBAR_CONFIG = {
"DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
"SHOW_TEMPLATE_CONTEXT": True,
"SHOW_TOOLBAR_CALLBACK": lambda request: True,
"JQUERY_URL": "",
"JQUERY_URL": "/staticfiles/admin/js/vendor/jquery/jquery.js",
}
# django-extensions
# ------------------------------------------------------------------------------
# INSTALLED_APPS += ('django_extensions', )
INSTALLED_APPS += ("debug_toolbar",)
# Debug toolbar is slow, we disable it for tests
DEBUG_TOOLBAR_ENABLED = env.bool("DEBUG_TOOLBAR_ENABLED", default=DEBUG)
if DEBUG_TOOLBAR_ENABLED:
MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)
INSTALLED_APPS += ("debug_toolbar",)
# TESTING
# ------------------------------------------------------------------------------
TEST_RUNNER = "django.test.runner.DiscoverRunner"
# CELERY
# In development, all tasks will be executed locally by blocking until the task returns
CELERY_TASK_ALWAYS_EAGER = False
# END CELERY
@ -72,3 +76,10 @@ LOGGING = {
},
}
CSRF_TRUSTED_ORIGINS = [o for o in ALLOWED_HOSTS]
if env.bool("WEAK_PASSWORDS", default=False):
# Faster during tests
PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",)
MIDDLEWARE = ("funkwhale_api.common.middleware.DevHttpsMiddleware",) + MIDDLEWARE

18
api/config/spa_urls.py Normal file
View File

@ -0,0 +1,18 @@
from django import urls
from funkwhale_api.music import spa_views
urlpatterns = [
urls.re_path(
r"^library/tracks/(?P<pk>\d+)/?$", spa_views.library_track, name="library_track"
),
urls.re_path(
r"^library/albums/(?P<pk>\d+)/?$", spa_views.library_album, name="library_album"
),
urls.re_path(
r"^library/artists/(?P<pk>\d+)/?$",
spa_views.library_artist,
name="library_artist",
),
]

39
api/config/wsgi.py
View File

@ -1,39 +0,0 @@
"""
WSGI config for funkwhale_api project.
This module contains the WSGI application used by Django's development server
and any production WSGI deployments. It should expose a module-level variable
named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
this application via the ``WSGI_APPLICATION`` setting.
Usually you will have the standard Django WSGI application here, but it also
might make sense to replace the whole Django WSGI application with a custom one
that later delegates to the Django one. For example, you could introduce WSGI
middleware here, or combine a Django application with an application of another
framework.
"""
import os
from django.core.wsgi import get_wsgi_application
from whitenoise.django import DjangoWhiteNoise
# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks
# if running multiple sites in the same mod_wsgi process. To fix this, use
# mod_wsgi daemon mode with each site in its own daemon process, or use
# os.environ["DJANGO_SETTINGS_MODULE"] = "config.settings.production"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
# This application object is used by any WSGI server configured to use this
# file. This includes Django's development server, if the WSGI_APPLICATION
# setting points here.
application = get_wsgi_application()
# Use Whitenoise to serve static files
# See: https://whitenoise.readthedocs.org/
application = DjangoWhiteNoise(application)
# Apply WSGI middleware here.
# from helloworld.wsgi import HelloWorldApplication
# application = HelloWorldApplication(application)

26
api/docker/Dockerfile.test
View File

@ -1,26 +0,0 @@
FROM python:3.6
ENV PYTHONUNBUFFERED 1
# Requirements have to be pulled and installed here, otherwise caching won't work
RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
COPY ./requirements.apt /requirements.apt
RUN apt-get update; \
grep "^[^#;]" requirements.apt | \
grep -Fv "python3-dev" | \
xargs apt-get install -y --no-install-recommends; \
rm -rf /usr/share/doc/* /usr/share/locale/*
RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
RUN mkdir /requirements
COPY ./requirements/base.txt /requirements/base.txt
RUN pip install -r /requirements/base.txt
COPY ./requirements/local.txt /requirements/local.txt
RUN pip install -r /requirements/local.txt
COPY ./requirements/test.txt /requirements/test.txt
RUN pip install -r /requirements/test.txt
COPY . /app
WORKDIR /app
ENTRYPOINT ["compose/django/dev-entrypoint.sh"]

2
api/funkwhale_api/__init__.py
View File

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
__version__ = "0.17"
__version__ = "0.18"
__version_info__ = tuple(
[
int(num) if num.isdigit() else num

2
api/funkwhale_api/common/channels.py
View File

@ -5,7 +5,7 @@ from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
from django.core.serializers.json import DjangoJSONEncoder
logger = logging.getLogger(__file__)
logger = logging.getLogger(__name__)
channel_layer = get_channel_layer()
group_add = async_to_sync(channel_layer.group_add)

14
api/funkwhale_api/common/decorators.py Normal file
View File

@ -0,0 +1,14 @@
from rest_framework import response
from rest_framework import decorators
def action_route(serializer_class):
@decorators.action(methods=["post"], detail=False)
def action(self, request, *args, **kwargs):
queryset = self.get_queryset()
serializer = serializer_class(request.data, queryset=queryset)
serializer.is_valid(raise_exception=True)
result = serializer.save()
return response.Response(result, status=200)
return action

159
api/funkwhale_api/common/middleware.py Normal file
View File

@ -0,0 +1,159 @@
import html
import requests
from django import http
from django.conf import settings
from django.core.cache import caches
from django import urls
from . import preferences
from . import utils
EXCLUDED_PATHS = ["/api", "/federation", "/.well-known"]
def should_fallback_to_spa(path):
if path == "/":
return True
return not any([path.startswith(m) for m in EXCLUDED_PATHS])
def serve_spa(request):
html = get_spa_html(settings.FUNKWHALE_SPA_HTML_ROOT)
head, tail = html.split("</head>", 1)
if not preferences.get("common__api_authentication_required"):
try:
request_tags = get_request_head_tags(request) or []
except urls.exceptions.Resolver404:
# we don't have any custom tags for this route
request_tags = []
else:
# API is not open, we don't expose any custom data
request_tags = []
default_tags = get_default_head_tags(request.path)
unique_attributes = ["name", "property"]
final_tags = request_tags
skip = []
for t in final_tags:
for attr in unique_attributes:
if attr in t:
skip.append(t[attr])
for t in default_tags:
existing = False
for attr in unique_attributes:
if t.get(attr) in skip:
existing = True
break
if not existing:
final_tags.append(t)
# let's inject our meta tags in the HTML
head += "\n" + "\n".join(render_tags(final_tags)) + "\n</head>"
return http.HttpResponse(head + tail)
def get_spa_html(spa_url):
cache_key = "spa-html:{}".format(spa_url)
cached = caches["local"].get(cache_key)
if cached:
return cached
response = requests.get(
utils.join_url(spa_url, "index.html"),
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
)
response.raise_for_status()
content = response.text
caches["local"].set(cache_key, content, settings.FUNKWHALE_SPA_HTML_CACHE_DURATION)
return content
def get_default_head_tags(path):
instance_name = preferences.get("instance__name")
short_description = preferences.get("instance__short_description")
app_name = settings.APP_NAME
parts = [instance_name, app_name]
return [
{"tag": "meta", "property": "og:type", "content": "website"},
{
"tag": "meta",
"property": "og:site_name",
"content": " - ".join([p for p in parts if p]),
},
{"tag": "meta", "property": "og:description", "content": short_description},
{
"tag": "meta",
"property": "og:image",
"content": utils.join_url(settings.FUNKWHALE_URL, "/front/favicon.png"),
},
{
"tag": "meta",
"property": "og:url",
"content": utils.join_url(settings.FUNKWHALE_URL, path),
},
]
def render_tags(tags):
"""
Given a dict like {'tag': 'meta', 'hello': 'world'}
return a html ready tag like
<meta hello="world" />
"""
for tag in tags:
yield "<{tag} {attrs} />".format(
tag=tag.pop("tag"),
attrs=" ".join(
[
'{}="{}"'.format(a, html.escape(str(v)))
for a, v in sorted(tag.items())
if v
]
),
)
def get_request_head_tags(request):
match = urls.resolve(request.path, urlconf=settings.SPA_URLCONF)
return match.func(request, *match.args, **match.kwargs)
class SPAFallbackMiddleware:
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
response = self.get_response(request)
if response.status_code == 404 and should_fallback_to_spa(request.path):
return serve_spa(request)
return response
class DevHttpsMiddleware:
"""
In development, it's sometimes difficult to have django use HTTPS
when we have django behind nginx behind traefix.
We thus use a simple setting (in dev ONLY) to control that.
"""
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
if settings.FORCE_HTTPS_URLS:
setattr(request.__class__, "scheme", "https")
setattr(
request,
"get_host",
lambda: request.__class__.get_host(request).replace(":80", ":443"),
)
return self.get_response(request)

14
api/funkwhale_api/common/migrations/0001_initial.py
View File

@ -3,8 +3,20 @@ from django.db import migrations
from django.contrib.postgres.operations import UnaccentExtension
class CustomUnaccentExtension(UnaccentExtension):
def database_forwards(self, app_label, schema_editor, from_state, to_state):
check_sql = "SELECT 1 FROM pg_extension WHERE extname = 'unaccent'"
with schema_editor.connection.cursor() as cursor:
cursor.execute(check_sql)
result = cursor.fetchall()
if result:
return
return super().database_forwards(app_label, schema_editor, from_state, to_state)
class Migration(migrations.Migration):
dependencies = []
operations = [UnaccentExtension()]
operations = [CustomUnaccentExtension()]

2
api/funkwhale_api/common/scripts/__init__.py
View File

@ -2,6 +2,7 @@ from . import create_actors
from . import create_image_variations
from . import django_permissions_to_user_permissions
from . import migrate_to_user_libraries
from . import delete_pre_017_federated_uploads
from . import test
@ -10,5 +11,6 @@ __all__ = [
"create_image_variations",
"django_permissions_to_user_permissions",
"migrate_to_user_libraries",
"delete_pre_017_federated_uploads",
"test",
]

14
api/funkwhale_api/common/scripts/delete_pre_017_federated_uploads.py Normal file
View File

@ -0,0 +1,14 @@
"""
Compute different sizes of image used for Album covers and User avatars
"""
from funkwhale_api.music.models import Upload
def main(command, **kwargs):
queryset = Upload.objects.filter(
source__startswith="http", source__contains="/federation/music/file/"
).exclude(source__contains="youtube")
total = queryset.count()
command.stdout.write("{} uploads found".format(total))
queryset.delete()

1
api/funkwhale_api/common/scripts/django_permissions_to_user_permissions.py
View File

@ -10,7 +10,6 @@ from funkwhale_api.users import models
mapping = {
"dynamic_preferences.change_globalpreferencemodel": "settings",
"music.add_importbatch": "library",
"federation.change_library": "federation",
}

59
api/funkwhale_api/common/serializers.py
View File

@ -1,8 +1,12 @@
import collections
import io
import PIL
import os
from rest_framework import serializers
from django.core.exceptions import ObjectDoesNotExist
from django.core.files.uploadedfile import SimpleUploadedFile
from django.utils.encoding import smart_text
from django.utils.translation import ugettext_lazy as _
@ -119,7 +123,7 @@ class ActionSerializer(serializers.Serializer):
if type(value) in [list, tuple]:
return self.queryset.filter(
**{"{}__in".format(self.pk_field): value}
).order_by("id")
).order_by(self.pk_field)
raise serializers.ValidationError(
"{} is not a valid value for objects. You must provide either a "
@ -159,3 +163,56 @@ class ActionSerializer(serializers.Serializer):
"result": result,
}
return payload
def track_fields_for_update(*fields):
"""
Apply this decorator to serializer to call function when specific values
are updated on an object:
.. code-block:: python
@track_fields_for_update('privacy_level')
class LibrarySerializer(serializers.ModelSerializer):
def on_updated_privacy_level(self, obj, old_value, new_value):
print('Do someting')
"""
def decorator(serializer_class):
original_update = serializer_class.update
def new_update(self, obj, validated_data):
tracked_fields_before = {f: getattr(obj, f) for f in fields}
obj = original_update(self, obj, validated_data)
tracked_fields_after = {f: getattr(obj, f) for f in fields}
if tracked_fields_before != tracked_fields_after:
self.on_updated_fields(obj, tracked_fields_before, tracked_fields_after)
return obj
serializer_class.update = new_update
return serializer_class
return decorator
class StripExifImageField(serializers.ImageField):
def to_internal_value(self, data):
file_obj = super().to_internal_value(data)
image = PIL.Image.open(file_obj)
data = list(image.getdata())
image_without_exif = PIL.Image.new(image.mode, image.size)
image_without_exif.putdata(data)
with io.BytesIO() as output:
image_without_exif.save(
output,
format=PIL.Image.EXTENSION[os.path.splitext(file_obj.name)[-1]],
quality=100,
)
content = output.getvalue()
return SimpleUploadedFile(
file_obj.name, content, content_type=file_obj.content_type
)

42
api/funkwhale_api/common/utils.py
View File

@ -3,10 +3,13 @@ from django.utils.deconstruct import deconstructible
import os
import shutil
import uuid
import xml.etree.ElementTree as ET
from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
from django.db import transaction
from django.conf import settings
from django import urls
from django.db import models, transaction
def rename_file(instance, field_name, new_name, allow_missing_file=False):
@ -107,3 +110,40 @@ def chunk_queryset(source_qs, chunk_size):
if nb_items < chunk_size:
return
def join_url(start, end):
if start.endswith("/") and end.startswith("/"):
return start + end[1:]
if not start.endswith("/") and not end.startswith("/"):
return start + "/" + end
return start + end
def spa_reverse(name, args=[], kwargs={}):
return urls.reverse(name, urlconf=settings.SPA_URLCONF, args=args, kwargs=kwargs)
def spa_resolve(path):
return urls.resolve(path, urlconf=settings.SPA_URLCONF)
def parse_meta(html):
# dirty but this is only for testing so we don't really care,
# we convert the html string to xml so it can be parsed as xml
html = '<?xml version="1.0"?>' + html
tree = ET.fromstring(html)
meta = [elem for elem in tree.iter() if elem.tag in ["meta", "link"]]
return [dict([("tag", elem.tag)] + list(elem.items())) for elem in meta]
def order_for_search(qs, field):
"""
When searching, it's often more useful to have short results first,
this function will order the given qs based on the length of the given field
"""
return qs.annotate(__size=models.functions.Length(field)).order_by("__size")

15
api/funkwhale_api/common/validators.py
View File

@ -1,6 +1,7 @@
import mimetypes
from os.path import splitext
from django.core import validators
from django.core.exceptions import ValidationError
from django.core.files.images import get_image_dimensions
from django.template.defaultfilters import filesizeformat
@ -150,3 +151,17 @@ class FileValidator(object):
}
raise ValidationError(message)
class DomainValidator(validators.URLValidator):
message = "Enter a valid domain name."
def __call__(self, value):
"""
This is a bit hackish but since we don't have any built-in domain validator,
we use the url one, and prepend http:// in front of it.
If it fails, we know the domain is not valid.
"""
super().__call__("http://{}".format(value))
return value

3
api/funkwhale_api/downloader/__init__.py
View File

@ -1,3 +0,0 @@
from .downloader import download
__all__ = ["download"]

19
api/funkwhale_api/downloader/downloader.py
View File

@ -1,19 +0,0 @@
import os
import youtube_dl
from django.conf import settings
def download(
url, target_directory=settings.MEDIA_ROOT, name="%(id)s.%(ext)s", bitrate=192
):
target_path = os.path.join(target_directory, name)
ydl_opts = {
"quiet": True,
"outtmpl": target_path,
"postprocessors": [{"key": "FFmpegExtractAudio", "preferredcodec": "vorbis"}],
}
_downloader = youtube_dl.YoutubeDL(ydl_opts)
info = _downloader.extract_info(url)
info["audio_file_path"] = target_path % {"id": info["id"], "ext": "ogg"}
return info

11
api/funkwhale_api/factories.py
View File

@ -28,3 +28,14 @@ def ManyToManyFromList(field_name):
field.add(*extracted)
return inner
class NoUpdateOnCreate:
"""
Factory boy calls save after the initial create. In most case, this
is not needed, so we disable this behaviour
"""
@classmethod
def _after_postgeneration(cls, instance, create, results=None):
return

4
api/funkwhale_api/favorites/factories.py
View File

@ -1,12 +1,12 @@
import factory
from funkwhale_api.factories import registry
from funkwhale_api.factories import registry, NoUpdateOnCreate
from funkwhale_api.music.factories import TrackFactory
from funkwhale_api.users.factories import UserFactory
@registry.register
class TrackFavorite(factory.django.DjangoModelFactory):
class TrackFavorite(NoUpdateOnCreate, factory.django.DjangoModelFactory):
track = factory.SubFactory(TrackFactory)
user = factory.SubFactory(UserFactory)

24
api/funkwhale_api/favorites/views.py
View File

@ -1,5 +1,5 @@
from rest_framework import mixins, status, viewsets
from rest_framework.decorators import list_route
from rest_framework.decorators import action
from rest_framework.permissions import IsAuthenticatedOrReadOnly
from rest_framework.response import Response
@ -20,7 +20,7 @@ class TrackFavoriteViewSet(
viewsets.GenericViewSet,
):
filter_class = filters.TrackFavoriteFilter
filterset_class = filters.TrackFavoriteFilter
serializer_class = serializers.UserTrackFavoriteSerializer
queryset = models.TrackFavorite.objects.all().select_related("user")
permission_classes = [
@ -51,7 +51,7 @@ class TrackFavoriteViewSet(
queryset = queryset.filter(
fields.privacy_level_query(self.request.user, "user__privacy_level")
)
tracks = Track.objects.annotate_playable_by_actor(
tracks = Track.objects.with_playable_uploads(
music_utils.get_actor_from_request(self.request)
).select_related("artist", "album__artist")
queryset = queryset.prefetch_related(Prefetch("track", queryset=tracks))
@ -62,7 +62,7 @@ class TrackFavoriteViewSet(
favorite = models.TrackFavorite.add(track=track, user=self.request.user)
return favorite
@list_route(methods=["delete", "post"])
@action(methods=["delete", "post"], detail=False)
def remove(self, request, *args, **kwargs):
try:
pk = int(request.data["track"])
@ -71,3 +71,19 @@ class TrackFavoriteViewSet(
return Response({}, status=400)
favorite.delete()
return Response([], status=status.HTTP_204_NO_CONTENT)
@action(methods=["get"], detail=False)
def all(self, request, *args, **kwargs):
"""
Return all the favorites of the current user, with only limited data
to have a performant endpoint and avoid lots of queries just to display
favorites status in the UI
"""
if not request.user.is_authenticated:
return Response({"results": [], "count": 0}, status=200)
favorites = list(
request.user.track_favorites.values("id", "track").order_by("id")
)
payload = {"results": favorites, "count": len(favorites)}
return Response(payload, status=200)

131
api/funkwhale_api/federation/activity.py
View File

@ -1,6 +1,8 @@
import uuid
import logging
from django.core.cache import cache
from django.conf import settings
from django.db import transaction, IntegrityError
from django.db.models import Q
@ -42,28 +44,68 @@ ACTIVITY_TYPES = [
"View",
]
OBJECT_TYPES = [
"Article",
"Audio",
"Collection",
"Document",
"Event",
"Image",
"Note",
"OrderedCollection",
"Page",
"Place",
"Profile",
"Relationship",
"Tombstone",
"Video",
] + ACTIVITY_TYPES
FUNKWHALE_OBJECT_TYPES = [
("Domain", "Domain"),
("Artist", "Artist"),
("Album", "Album"),
("Track", "Track"),
("Library", "Library"),
]
OBJECT_TYPES = (
[
"Application",
"Article",
"Audio",
"Collection",
"Document",
"Event",
"Group",
"Image",
"Note",
"Object",
"OrderedCollection",
"Organization",
"Page",
"Person",
"Place",
"Profile",
"Relationship",
"Service",
"Tombstone",
"Video",
]
+ ACTIVITY_TYPES
+ FUNKWHALE_OBJECT_TYPES
)
BROADCAST_TO_USER_ACTIVITIES = ["Follow", "Accept"]
def should_reject(id, actor_id=None, payload={}):
from funkwhale_api.moderation import models as moderation_models
policies = moderation_models.InstancePolicy.objects.active()
media_types = ["Audio", "Artist", "Album", "Track", "Library", "Image"]
relevant_values = [
recursive_gettattr(payload, "type", permissive=True),
recursive_gettattr(payload, "object.type", permissive=True),
recursive_gettattr(payload, "target.type", permissive=True),
]
# if one of the payload types match our internal media types, then
# we apply policies that reject media
if set(media_types) & set(relevant_values):
policy_type = Q(block_all=True) | Q(reject_media=True)
else:
policy_type = Q(block_all=True)
query = policies.matching_url_query(id) & policy_type
if actor_id:
query |= policies.matching_url_query(actor_id) & policy_type
return policies.filter(query).exists()
@transaction.atomic
def receive(activity, on_behalf_of):
from . import models
@ -76,6 +118,16 @@ def receive(activity, on_behalf_of):
data=activity, context={"actor": on_behalf_of, "local_recipients": True}
)
serializer.is_valid(raise_exception=True)
if should_reject(
id=serializer.validated_data["id"],
actor_id=serializer.validated_data["actor"].fid,
payload=activity,
):
logger.info(
"[federation] Discarding activity due to instance policies %s",
serializer.validated_data.get("id"),
)
return
try:
copy = serializer.save()
except IntegrityError:
@ -186,6 +238,21 @@ class InboxRouter(Router):
return
ACTOR_KEY_ROTATION_LOCK_CACHE_KEY = "federation:actor-key-rotation-lock:{}"
def should_rotate_actor_key(actor_id):
lock = cache.get(ACTOR_KEY_ROTATION_LOCK_CACHE_KEY.format(actor_id))
return lock is None
def schedule_key_rotation(actor_id, delay):
from . import tasks
cache.set(ACTOR_KEY_ROTATION_LOCK_CACHE_KEY.format(actor_id), True, timeout=delay)
tasks.rotate_actor_key.apply_async(kwargs={"actor_id": actor_id}, countdown=delay)
class OutboxRouter(Router):
@transaction.atomic
def dispatch(self, routing, context):
@ -206,6 +273,15 @@ class OutboxRouter(Router):
# a route can yield zero, one or more activity payloads
if e:
activities_data.append(e)
deletions = [
a["actor"].id
for a in activities_data
if a["payload"]["type"] == "Delete"
]
for actor_id in deletions:
# we way need to triggers a blind key rotation
if should_rotate_actor_key(actor_id):
schedule_key_rotation(actor_id, settings.ACTOR_KEY_ROTATION_DELAY)
inbox_items_by_activity_uuid = {}
deliveries_by_activity_uuid = {}
prepared_activities = []
@ -267,7 +343,7 @@ class OutboxRouter(Router):
return activities
def recursive_gettattr(obj, key):
def recursive_gettattr(obj, key, permissive=False):
"""
Given a dictionary such as {'user': {'name': 'Bob'}} and
a dotted string such as user.name, returns 'Bob'.
@ -276,7 +352,12 @@ def recursive_gettattr(obj, key):
"""
v = obj
for k in key.split("."):
v = v.get(k)
try:
v = v.get(k)
except (TypeError, AttributeError):
if not permissive:
raise
return
if v is None:
return
@ -386,15 +467,3 @@ def get_actors_from_audience(urls):
if not final_query:
return models.Actor.objects.none()
return models.Actor.objects.filter(final_query)
def get_inbox_urls(actor_queryset):
"""
Given an actor queryset, returns a deduplicated set containing
all inbox or shared inbox urls where we should deliver our payloads for
those actors
"""
values = actor_queryset.values("inbox_url", "shared_inbox_url")
urls = set([actor["shared_inbox_url"] or actor["inbox_url"] for actor in values])
return sorted(urls)

23
api/funkwhale_api/federation/actors.py
View File

@ -25,17 +25,18 @@ def get_actor_data(actor_url):
raise ValueError("Invalid actor payload: {}".format(response.text))
def get_actor(fid):
try:
actor = models.Actor.objects.get(fid=fid)
except models.Actor.DoesNotExist:
actor = None
fetch_delta = datetime.timedelta(
minutes=preferences.get("federation__actor_fetch_delay")
)
if actor and actor.last_fetch_date > timezone.now() - fetch_delta:
# cache is hot, we can return as is
return actor
def get_actor(fid, skip_cache=False):
if not skip_cache:
try:
actor = models.Actor.objects.get(fid=fid)
except models.Actor.DoesNotExist:
actor = None
fetch_delta = datetime.timedelta(
minutes=preferences.get("federation__actor_fetch_delay")
)
if actor and actor.last_fetch_date > timezone.now() - fetch_delta:
# cache is hot, we can return as is
return actor
data = get_actor_data(fid)
serializer = serializers.ActorSerializer(data=data)
serializer.is_valid(raise_exception=True)

6
api/funkwhale_api/federation/admin.py
View File

@ -24,6 +24,12 @@ def redeliver_activities(modeladmin, request, queryset):
redeliver_activities.short_description = "Redeliver"
@admin.register(models.Domain)
class DomainAdmin(admin.ModelAdmin):
list_display = ["name", "creation_date"]
search_fields = ["name"]
@admin.register(models.Activity)
class ActivityAdmin(admin.ModelAdmin):
list_display = ["type", "fid", "url", "actor", "creation_date"]

22
api/funkwhale_api/federation/api_views.py
View File

@ -13,6 +13,7 @@ from funkwhale_api.music import models as music_models
from . import activity
from . import api_serializers
from . import exceptions
from . import filters
from . import models
from . import routes
@ -42,7 +43,7 @@ class LibraryFollowViewSet(
)
serializer_class = api_serializers.LibraryFollowSerializer
permission_classes = [permissions.IsAuthenticated]
filter_class = filters.LibraryFollowFilter
filterset_class = filters.LibraryFollowFilter
ordering_fields = ("creation_date",)
def get_queryset(self):
@ -65,7 +66,7 @@ class LibraryFollowViewSet(
context["actor"] = self.request.user.actor
return context
@decorators.detail_route(methods=["post"])
@decorators.action(methods=["post"], detail=True)
def accept(self, request, *args, **kwargs):
try:
follow = self.queryset.get(
@ -76,7 +77,7 @@ class LibraryFollowViewSet(
update_follow(follow, approved=True)
return response.Response(status=204)
@decorators.detail_route(methods=["post"])
@decorators.action(methods=["post"], detail=True)
def reject(self, request, *args, **kwargs):
try:
follow = self.queryset.get(
@ -104,7 +105,7 @@ class LibraryViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
qs = super().get_queryset()
return qs.viewable_by(actor=self.request.user.actor)
@decorators.detail_route(methods=["post"])
@decorators.action(methods=["post"], detail=True)
def scan(self, request, *args, **kwargs):
library = self.get_object()
if library.actor.get_user():
@ -121,18 +122,23 @@ class LibraryViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
)
return response.Response({"status": "skipped"}, 200)
@decorators.list_route(methods=["post"])
@decorators.action(methods=["post"], detail=False)
def fetch(self, request, *args, **kwargs):
try:
fid = request.data["fid"]
except KeyError:
return response.Response({"fid": ["This field is required"]})
try:
library = utils.retrieve(
library = utils.retrieve_ap_object(
fid,
queryset=self.queryset,
serializer_class=serializers.LibrarySerializer,
)
except exceptions.BlockedActorOrDomain:
return response.Response(
{"detail": "This domain/account is blocked on your instance."},
status=400,
)
except requests.exceptions.RequestException as e:
return response.Response(
{"detail": "Error while fetching the library: {}".format(str(e))},
@ -162,14 +168,14 @@ class InboxItemViewSet(
)
serializer_class = api_serializers.InboxItemSerializer
permission_classes = [permissions.IsAuthenticated]
filter_class = filters.InboxItemFilter
filterset_class = filters.InboxItemFilter
ordering_fields = ("activity__creation_date",)
def get_queryset(self):
qs = super().get_queryset()
return qs.filter(actor=self.request.user.actor)
@decorators.list_route(methods=["post"])
@decorators.action(methods=["post"], detail=False)
def action(self, request, *args, **kwargs):
queryset = self.get_queryset()
serializer = api_serializers.InboxItemActionSerializer(

44
api/funkwhale_api/federation/authentication.py
View File

@ -1,8 +1,14 @@
import cryptography
from django.contrib.auth.models import AnonymousUser
from rest_framework import authentication, exceptions
import logging
from . import actors, keys, signing, utils
from django.contrib.auth.models import AnonymousUser
from rest_framework import authentication, exceptions as rest_exceptions
from funkwhale_api.moderation import models as moderation_models
from . import actors, exceptions, keys, signing, utils
logger = logging.getLogger(__name__)
class SignatureAuthentication(authentication.BaseAuthentication):
@ -14,20 +20,42 @@ class SignatureAuthentication(authentication.BaseAuthentication):
except KeyError:
return
except ValueError as e:
raise exceptions.AuthenticationFailed(str(e))
raise rest_exceptions.AuthenticationFailed(str(e))
try:
actor = actors.get_actor(key_id.split("#")[0])
actor_url = key_id.split("#")[0]
except (TypeError, IndexError, AttributeError):
raise rest_exceptions.AuthenticationFailed("Invalid key id")
policies = (
moderation_models.InstancePolicy.objects.active()
.filter(block_all=True)
.matching_url(actor_url)
)
if policies.exists():
raise exceptions.BlockedActorOrDomain()
try:
actor = actors.get_actor(actor_url)
except Exception as e:
raise exceptions.AuthenticationFailed(str(e))
logger.info(
"Discarding HTTP request from blocked actor/domain %s", actor_url
)
raise rest_exceptions.AuthenticationFailed(str(e))
if not actor.public_key:
raise exceptions.AuthenticationFailed("No public key found")
raise rest_exceptions.AuthenticationFailed("No public key found")
try:
signing.verify_django(request, actor.public_key.encode("utf-8"))
except cryptography.exceptions.InvalidSignature:
raise exceptions.AuthenticationFailed("Invalid signature")
# in case of invalid signature, we refetch the actor object
# to load a potentially new public key. This process is called
# Blind key rotation, and is described at
# https://blog.dereferenced.org/the-case-for-blind-key-rotation
# if signature verification fails after that, then we return a 403 error
actor = actors.get_actor(actor_url, skip_cache=True)
signing.verify_django(request, actor.public_key.encode("utf-8"))
return actor

7
api/funkwhale_api/federation/exceptions.py
View File

@ -1,6 +1,13 @@
from rest_framework import exceptions
class MalformedPayload(ValueError):
pass
class MissingSignature(KeyError):
pass
class BlockedActorOrDomain(exceptions.AuthenticationFailed):
pass

69
api/funkwhale_api/federation/factories.py
View File

@ -7,7 +7,7 @@ from django.conf import settings
from django.utils import timezone
from django.utils.http import http_date
from funkwhale_api.factories import registry
from funkwhale_api.factories import registry, NoUpdateOnCreate
from funkwhale_api.users import factories as user_factories
from . import keys, models
@ -67,24 +67,40 @@ def create_user(actor):
@registry.register
class ActorFactory(factory.DjangoModelFactory):
class DomainFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
name = factory.Faker("domain_name")
class Meta:
model = "federation.Domain"
django_get_or_create = ("name",)
@registry.register
class ActorFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
public_key = None
private_key = None
preferred_username = factory.Faker("user_name")
summary = factory.Faker("paragraph")
domain = factory.Faker("domain_name")
domain = factory.SubFactory(DomainFactory)
fid = factory.LazyAttribute(
lambda o: "https://{}/users/{}".format(o.domain, o.preferred_username)
lambda o: "https://{}/users/{}".format(o.domain.name, o.preferred_username)
)
followers_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}followers".format(o.domain, o.preferred_username)
lambda o: "https://{}/users/{}followers".format(
o.domain.name, o.preferred_username
)
)
inbox_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}/inbox".format(o.domain, o.preferred_username)
lambda o: "https://{}/users/{}/inbox".format(
o.domain.name, o.preferred_username
)
)
outbox_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}/outbox".format(o.domain, o.preferred_username)
lambda o: "https://{}/users/{}/outbox".format(
o.domain.name, o.preferred_username
)
)
keys = factory.LazyFunction(keys.get_key_pair)
class Meta:
model = models.Actor
@ -95,7 +111,9 @@ class ActorFactory(factory.DjangoModelFactory):
return
from funkwhale_api.users.factories import UserFactory
self.domain = settings.FEDERATION_HOSTNAME
self.domain = models.Domain.objects.get_or_create(
name=settings.FEDERATION_HOSTNAME
)[0]
self.save(update_fields=["domain"])
if not create:
if extracted and hasattr(extracted, "pk"):
@ -108,19 +126,9 @@ class ActorFactory(factory.DjangoModelFactory):
else:
self.user = UserFactory(actor=self, **kwargs)
@factory.post_generation
def keys(self, create, extracted, **kwargs):
if not create:
# Simple build, do nothing.
return
if not extracted:
private, public = keys.get_key_pair()
self.private_key = private.decode("utf-8")
self.public_key = public.decode("utf-8")
@registry.register
class FollowFactory(factory.DjangoModelFactory):
class FollowFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
target = factory.SubFactory(ActorFactory)
actor = factory.SubFactory(ActorFactory)
@ -132,28 +140,23 @@ class FollowFactory(factory.DjangoModelFactory):
@registry.register
class MusicLibraryFactory(factory.django.DjangoModelFactory):
class MusicLibraryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory)
privacy_level = "me"
name = factory.Faker("sentence")
description = factory.Faker("sentence")
uploads_count = 0
fid = factory.Faker("federation_url")
followers_url = factory.LazyAttribute(
lambda o: o.fid + "/followers" if o.fid else None
)
class Meta:
model = "music.Library"
@factory.post_generation
def followers_url(self, create, extracted, **kwargs):
if not create:
# Simple build, do nothing.
return
self.followers_url = extracted or self.fid + "/followers"
@registry.register
class LibraryScan(factory.django.DjangoModelFactory):
class LibraryScan(NoUpdateOnCreate, factory.django.DjangoModelFactory):
library = factory.SubFactory(MusicLibraryFactory)
actor = factory.SubFactory(ActorFactory)
total_files = factory.LazyAttribute(lambda o: o.library.uploads_count)
@ -163,7 +166,7 @@ class LibraryScan(factory.django.DjangoModelFactory):
@registry.register
class ActivityFactory(factory.django.DjangoModelFactory):
class ActivityFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory)
url = factory.Faker("federation_url")
payload = factory.LazyFunction(lambda: {"type": "Create"})
@ -173,7 +176,7 @@ class ActivityFactory(factory.django.DjangoModelFactory):
@registry.register
class InboxItemFactory(factory.django.DjangoModelFactory):
class InboxItemFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory, local=True)
activity = factory.SubFactory(ActivityFactory)
type = "to"
@ -183,7 +186,7 @@ class InboxItemFactory(factory.django.DjangoModelFactory):
@registry.register
class DeliveryFactory(factory.django.DjangoModelFactory):
class DeliveryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
activity = factory.SubFactory(ActivityFactory)
inbox_url = factory.Faker("url")
@ -192,7 +195,7 @@ class DeliveryFactory(factory.django.DjangoModelFactory):
@registry.register
class LibraryFollowFactory(factory.DjangoModelFactory):
class LibraryFollowFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
target = factory.SubFactory(MusicLibraryFactory)
actor = factory.SubFactory(ActorFactory)

18
api/funkwhale_api/federation/fields.py Normal file
View File

@ -0,0 +1,18 @@
from rest_framework import serializers
from . import models
class ActorRelatedField(serializers.EmailField):
def to_representation(self, value):
return value.full_username
def to_internal_value(self, value):
value = super().to_internal_value(value)
username, domain = value.split("@")
try:
return models.Actor.objects.get(
preferred_username=username, domain_id=domain
)
except models.Actor.DoesNotExist:
raise serializers.ValidationError("Invalid actor name")

5
api/funkwhale_api/federation/keys.py
View File

@ -1,6 +1,8 @@
import re
import urllib.parse
from django.conf import settings
from cryptography.hazmat.backends import default_backend as crypto_default_backend
from cryptography.hazmat.primitives import serialization as crypto_serialization
from cryptography.hazmat.primitives.asymmetric import rsa
@ -8,7 +10,8 @@ from cryptography.hazmat.primitives.asymmetric import rsa
KEY_ID_REGEX = re.compile(r"keyId=\"(?P<id>.*)\"")
def get_key_pair(size=2048):
def get_key_pair(size=None):
size = size or settings.RSA_KEY_SIZE
key = rsa.generate_private_key(
backend=crypto_default_backend(), public_exponent=65537, key_size=size
)

23
api/funkwhale_api/federation/migrations/0013_auto_20181226_1935.py Normal file
View File

@ -0,0 +1,23 @@
# Generated by Django 2.0.9 on 2018-12-26 19:35
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [("federation", "0012_auto_20180920_1803")]
operations = [
migrations.AlterField(
model_name="actor",
name="private_key",
field=models.TextField(blank=True, max_length=5000, null=True),
),
migrations.AlterField(
model_name="actor",
name="public_key",
field=models.TextField(blank=True, max_length=5000, null=True),
),
]

46
api/funkwhale_api/federation/migrations/0014_auto_20181205_0958.py Normal file
View File

@ -0,0 +1,46 @@
# Generated by Django 2.0.9 on 2018-12-05 09:58
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [("federation", "0013_auto_20181226_1935")]
operations = [
migrations.CreateModel(
name="Domain",
fields=[
(
"name",
models.CharField(max_length=255, primary_key=True, serialize=False),
),
(
"creation_date",
models.DateTimeField(default=django.utils.timezone.now),
),
],
),
migrations.AlterField(
model_name="actor",
name="domain",
field=models.CharField(max_length=1000, null=True),
),
migrations.RenameField("actor", "domain", "old_domain"),
migrations.AddField(
model_name="actor",
name="domain",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="actors",
to="federation.Domain",
),
),
migrations.AlterUniqueTogether(name="actor", unique_together=set()),
migrations.AlterUniqueTogether(
name="actor", unique_together={("domain", "preferred_username")}
),
]

56
api/funkwhale_api/federation/migrations/0015_populate_domains.py Normal file
View File

@ -0,0 +1,56 @@
# Generated by Django 2.0.9 on 2018-11-14 08:55
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
def populate_domains(apps, schema_editor):
Domain = apps.get_model("federation", "Domain")
Actor = apps.get_model("federation", "Actor")
domains = set(
[v.lower() for v in Actor.objects.values_list("old_domain", flat=True)]
)
for domain in sorted(domains):
print("Populating domain {}...".format(domain))
first_actor = (
Actor.objects.order_by("creation_date")
.exclude(creation_date=None)
.filter(old_domain__iexact=domain)
.first()
)
if first_actor:
first_seen = first_actor.creation_date
else:
first_seen = django.utils.timezone.now()
Domain.objects.update_or_create(
name=domain, defaults={"creation_date": first_seen}
)
for domain in Domain.objects.all():
Actor.objects.filter(old_domain__iexact=domain.name).update(domain=domain)
def skip(apps, schema_editor):
pass
class Migration(migrations.Migration):
dependencies = [("federation", "0014_auto_20181205_0958")]
operations = [
migrations.RunPython(populate_domains, skip),
migrations.AlterField(
model_name="actor",
name="domain",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="actors",
to="federation.Domain",
),
),
]

25
api/funkwhale_api/federation/migrations/0016_auto_20181227_1605.py Normal file
View File

@ -0,0 +1,25 @@
# Generated by Django 2.0.9 on 2018-12-27 16:05
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import funkwhale_api.federation.models
class Migration(migrations.Migration):
dependencies = [("federation", "0015_populate_domains")]
operations = [
migrations.AddField(
model_name="domain",
name="nodeinfo",
field=django.contrib.postgres.fields.jsonb.JSONField(
default=funkwhale_api.federation.models.empty_dict, max_length=50000
),
),
migrations.AddField(
model_name="domain",
name="nodeinfo_fetch_date",
field=models.DateTimeField(blank=True, default=None, null=True),
),
]

151
api/funkwhale_api/federation/models.py
View File

@ -13,6 +13,7 @@ from django.urls import reverse
from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common import validators as common_validators
from funkwhale_api.music import utils as music_utils
from . import utils as federation_utils
@ -61,6 +62,83 @@ class ActorQuerySet(models.QuerySet):
return qs
def with_uploads_count(self):
return self.annotate(
uploads_count=models.Count("libraries__uploads", distinct=True)
)
class DomainQuerySet(models.QuerySet):
def external(self):
return self.exclude(pk=settings.FEDERATION_HOSTNAME)
def with_actors_count(self):
return self.annotate(actors_count=models.Count("actors", distinct=True))
def with_outbox_activities_count(self):
return self.annotate(
outbox_activities_count=models.Count(
"actors__outbox_activities", distinct=True
)
)
class Domain(models.Model):
name = models.CharField(
primary_key=True,
max_length=255,
validators=[common_validators.DomainValidator()],
)
creation_date = models.DateTimeField(default=timezone.now)
nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True)
nodeinfo = JSONField(default=empty_dict, max_length=50000, blank=True)
objects = DomainQuerySet.as_manager()
def __str__(self):
return self.name
def save(self, **kwargs):
lowercase_fields = ["name"]
for field in lowercase_fields:
v = getattr(self, field, None)
if v:
setattr(self, field, v.lower())
super().save(**kwargs)
def get_stats(self):
from funkwhale_api.music import models as music_models
data = Domain.objects.filter(pk=self.pk).aggregate(
actors=models.Count("actors", distinct=True),
outbox_activities=models.Count("actors__outbox_activities", distinct=True),
libraries=models.Count("actors__libraries", distinct=True),
received_library_follows=models.Count(
"actors__libraries__received_follows", distinct=True
),
emitted_library_follows=models.Count(
"actors__library_follows", distinct=True
),
)
data["artists"] = music_models.Artist.objects.filter(
from_activity__actor__domain_id=self.pk
).count()
data["albums"] = music_models.Album.objects.filter(
from_activity__actor__domain_id=self.pk
).count()
data["tracks"] = music_models.Track.objects.filter(
from_activity__actor__domain_id=self.pk
).count()
uploads = music_models.Upload.objects.filter(library__actor__domain_id=self.pk)
data["uploads"] = uploads.count()
data["media_total_size"] = uploads.aggregate(v=models.Sum("size"))["v"] or 0
data["media_downloaded_size"] = (
uploads.with_file().aggregate(v=models.Sum("size"))["v"] or 0
)
return data
class Actor(models.Model):
ap_type = "Actor"
@ -74,7 +152,7 @@ class Actor(models.Model):
shared_inbox_url = models.URLField(max_length=500, null=True, blank=True)
type = models.CharField(choices=TYPE_CHOICES, default="Person", max_length=25)
name = models.CharField(max_length=200, null=True, blank=True)
domain = models.CharField(max_length=1000)
domain = models.ForeignKey(Domain, on_delete=models.CASCADE, related_name="actors")
summary = models.CharField(max_length=500, null=True, blank=True)
preferred_username = models.CharField(max_length=200, null=True, blank=True)
public_key = models.TextField(max_length=5000, null=True, blank=True)
@ -105,41 +183,14 @@ class Actor(models.Model):
@property
def full_username(self):
return "{}@{}".format(self.preferred_username, self.domain)
return "{}@{}".format(self.preferred_username, self.domain_id)
def __str__(self):
return "{}@{}".format(self.preferred_username, self.domain)
def save(self, **kwargs):
lowercase_fields = ["domain"]
for field in lowercase_fields:
v = getattr(self, field, None)
if v:
setattr(self, field, v.lower())
super().save(**kwargs)
return "{}@{}".format(self.preferred_username, self.domain_id)
@property
def is_local(self):
return self.domain == settings.FEDERATION_HOSTNAME
@property
def is_system(self):
from . import actors
return all(
[
settings.FEDERATION_HOSTNAME == self.domain,
self.preferred_username in actors.SYSTEM_ACTORS,
]
)
@property
def system_conf(self):
from . import actors
if self.is_system:
return actors.SYSTEM_ACTORS[self.preferred_username]
return self.domain_id == settings.FEDERATION_HOSTNAME
def get_approved_followers(self):
follows = self.received_follows.filter(approved=True)
@ -163,6 +214,44 @@ class Actor(models.Model):
data["total"] = sum(data.values())
return data
def get_stats(self):
from funkwhale_api.music import models as music_models
data = Actor.objects.filter(pk=self.pk).aggregate(
outbox_activities=models.Count("outbox_activities", distinct=True),
libraries=models.Count("libraries", distinct=True),
received_library_follows=models.Count(
"libraries__received_follows", distinct=True
),
emitted_library_follows=models.Count("library_follows", distinct=True),
)
data["artists"] = music_models.Artist.objects.filter(
from_activity__actor=self.pk
).count()
data["albums"] = music_models.Album.objects.filter(
from_activity__actor=self.pk
).count()
data["tracks"] = music_models.Track.objects.filter(
from_activity__actor=self.pk
).count()
uploads = music_models.Upload.objects.filter(library__actor=self.pk)
data["uploads"] = uploads.count()
data["media_total_size"] = uploads.aggregate(v=models.Sum("size"))["v"] or 0
data["media_downloaded_size"] = (
uploads.with_file().aggregate(v=models.Sum("size"))["v"] or 0
)
return data
@property
def keys(self):
return self.private_key, self.public_key
@keys.setter
def keys(self, v):
self.private_key = v[0].decode("utf-8")
self.public_key = v[1].decode("utf-8")
class InboxItem(models.Model):
"""

41
api/funkwhale_api/federation/routes.py
View File

@ -82,7 +82,7 @@ def inbox_undo_follow(payload, context):
serializer = serializers.UndoFollowSerializer(data=payload, context=context)
if not serializer.is_valid(raise_exception=context.get("raise_exception", False)):
logger.debug(
"Discarding invalid follow undo from {}: %s",
"Discarding invalid follow undo from %s: %s",
context["actor"].fid,
serializer.errors,
)
@ -195,6 +195,45 @@ def outbox_delete_library(context):
}
@outbox.register({"type": "Update", "object.type": "Library"})
def outbox_update_library(context):
library = context["library"]
serializer = serializers.ActivitySerializer(
{"type": "Update", "object": serializers.LibrarySerializer(library).data}
)
yield {
"type": "Update",
"actor": library.actor,
"payload": with_recipients(
serializer.data, to=[{"type": "followers", "target": library}]
),
}
@inbox.register({"type": "Update", "object.type": "Library"})
def inbox_update_library(payload, context):
actor = context["actor"]
library_id = payload["object"].get("id")
if not library_id:
logger.debug("Discarding deletion of empty library")
return
if not actor.libraries.filter(fid=library_id).exists():
logger.debug("Discarding deletion of unkwnown library %s", library_id)
return
serializer = serializers.LibrarySerializer(data=payload["object"])
if serializer.is_valid():
serializer.save()
else:
logger.debug(
"Discarding update of library %s because of payload errors: %s",
library_id,
serializer.errors,
)
@inbox.register({"type": "Delete", "object.type": "Audio"})
def inbox_delete_audio(payload, context):
actor = context["actor"]

23
api/funkwhale_api/federation/serializers.py
View File

@ -114,7 +114,7 @@ class ActorSerializer(serializers.Serializer):
if maf is not None:
kwargs["manually_approves_followers"] = maf
domain = urllib.parse.urlparse(kwargs["fid"]).netloc
kwargs["domain"] = domain
kwargs["domain"] = models.Domain.objects.get_or_create(pk=domain)[0]
for endpoint, url in self.initial_data.get("endpoints", {}).items():
if endpoint == "sharedInbox":
kwargs["shared_inbox_url"] = url
@ -560,14 +560,14 @@ class LibrarySerializer(PaginatedCollectionSerializer):
r = super().to_representation(conf)
r["audience"] = (
"https://www.w3.org/ns/activitystreams#Public"
if library.privacy_level == "public"
if library.privacy_level == "everyone"
else ""
)
r["followers"] = library.followers_url
return r
def create(self, validated_data):
actor = utils.retrieve(
actor = utils.retrieve_ap_object(
validated_data["actor"],
queryset=models.Actor,
serializer_class=ActorSerializer,
@ -729,8 +729,11 @@ class AlbumSerializer(MusicEntitySerializer):
class TrackSerializer(MusicEntitySerializer):
position = serializers.IntegerField(min_value=0, allow_null=True, required=False)
disc = serializers.IntegerField(min_value=1, allow_null=True, required=False)
artists = serializers.ListField(child=ArtistSerializer(), min_length=1)
album = AlbumSerializer()
license = serializers.URLField(allow_null=True, required=False)
copyright = serializers.CharField(allow_null=True, required=False)
def to_representation(self, instance):
d = {
@ -740,6 +743,11 @@ class TrackSerializer(MusicEntitySerializer):
"published": instance.creation_date.isoformat(),
"musicbrainzId": str(instance.mbid) if instance.mbid else None,
"position": instance.position,
"disc": instance.disc_number,
"license": instance.local_license["identifiers"][0]
if instance.local_license
else None,
"copyright": instance.copyright if instance.copyright else None,
"artists": [
ArtistSerializer(
instance.artist, context={"include_ap_context": False}
@ -880,3 +888,12 @@ class CollectionSerializer(serializers.Serializer):
if self.context.get("include_ap_context", True):
d["@context"] = AP_CONTEXT
return d
class NodeInfoLinkSerializer(serializers.Serializer):
href = serializers.URLField()
rel = serializers.URLField()
class NodeInfoSerializer(serializers.Serializer):
links = serializers.ListField(child=NodeInfoLinkSerializer(), min_length=1)

2
api/funkwhale_api/federation/signing.py
View File

@ -85,7 +85,7 @@ def verify_django(django_request, public_key):
def get_auth(private_key, private_key_id):
return requests_http_signature.HTTPSignatureAuth(
use_auth_header=False,
headers=["(request-target)", "user-agent", "host", "date", "content-type"],
headers=["(request-target)", "user-agent", "host", "date"],
algorithm="rsa-sha256",
key=private_key.encode("utf-8"),
key_id=private_key_id,

92
api/funkwhale_api/federation/tasks.py
View File

@ -1,6 +1,7 @@
import datetime
import logging
import os
import requests
from django.conf import settings
from django.db.models import Q, F
@ -13,7 +14,9 @@ from funkwhale_api.common import session
from funkwhale_api.music import models as music_models
from funkwhale_api.taskapp import celery
from . import keys
from . import models, signing
from . import serializers
from . import routes
logger = logging.getLogger(__name__)
@ -147,3 +150,92 @@ def deliver_to_remote(delivery):
delivery.attempts = F("attempts") + 1
delivery.is_delivered = True
delivery.save(update_fields=["last_attempt_date", "attempts", "is_delivered"])
def fetch_nodeinfo(domain_name):
s = session.get_session()
wellknown_url = "https://{}/.well-known/nodeinfo".format(domain_name)
response = s.get(
url=wellknown_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL
)
response.raise_for_status()
serializer = serializers.NodeInfoSerializer(data=response.json())
serializer.is_valid(raise_exception=True)
nodeinfo_url = None
for link in serializer.validated_data["links"]:
if link["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0":
nodeinfo_url = link["href"]
break
response = s.get(
url=nodeinfo_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL
)
response.raise_for_status()
return response.json()
@celery.app.task(name="federation.update_domain_nodeinfo")
@celery.require_instance(
models.Domain.objects.external(), "domain", id_kwarg_name="domain_name"
)
def update_domain_nodeinfo(domain):
now = timezone.now()
try:
nodeinfo = {"status": "ok", "payload": fetch_nodeinfo(domain.name)}
except (requests.RequestException, serializers.serializers.ValidationError) as e:
nodeinfo = {"status": "error", "error": str(e)}
domain.nodeinfo_fetch_date = now
domain.nodeinfo = nodeinfo
domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date"])
def delete_qs(qs):
label = qs.model._meta.label
result = qs.delete()
related = sum(result[1].values())
logger.info(
"Purged %s %s objects (and %s related entities)", result[0], label, related
)
def handle_purge_actors(ids, only=[]):
"""
Empty only means we purge everything
Otherwise, we purge only the requested bits: media
"""
# purge follows (received emitted)
if not only:
delete_qs(models.LibraryFollow.objects.filter(target__actor_id__in=ids))
delete_qs(models.Follow.objects.filter(actor_id__in=ids))
# purge audio content
if not only or "media" in only:
delete_qs(models.LibraryFollow.objects.filter(actor_id__in=ids))
delete_qs(models.Follow.objects.filter(target_id__in=ids))
delete_qs(music_models.Upload.objects.filter(library__actor_id__in=ids))
delete_qs(music_models.Library.objects.filter(actor_id__in=ids))
# purge remaining activities / deliveries
if not only:
delete_qs(models.InboxItem.objects.filter(actor_id__in=ids))
delete_qs(models.Activity.objects.filter(actor_id__in=ids))
@celery.app.task(name="federation.purge_actors")
def purge_actors(ids=[], domains=[], only=[]):
actors = models.Actor.objects.filter(
Q(id__in=ids) | Q(domain_id__in=domains)
).order_by("id")
found_ids = list(actors.values_list("id", flat=True))
logger.info("Starting purging %s accounts", len(found_ids))
handle_purge_actors(ids=found_ids, only=only)
@celery.app.task(name="federation.rotate_actor_key")
@celery.require_instance(models.Actor.objects.local(), "actor")
def rotate_actor_key(actor):
pair = keys.get_key_pair()
actor.private_key = pair[0].decode()
actor.public_key = pair[1].decode()
actor.save(update_fields=["private_key", "public_key"])

21
api/funkwhale_api/federation/utils.py
View File

@ -3,7 +3,9 @@ import re
from django.conf import settings
from funkwhale_api.common import session
from funkwhale_api.moderation import models as moderation_models
from . import exceptions
from . import signing
@ -58,7 +60,14 @@ def slugify_username(username):
return re.sub(r"[-\s]+", "_", value)
def retrieve(fid, actor=None, serializer_class=None, queryset=None):
def retrieve_ap_object(
fid, actor=None, serializer_class=None, queryset=None, apply_instance_policies=True
):
from . import activity
policies = moderation_models.InstancePolicy.objects.active().filter(block_all=True)
if apply_instance_policies and policies.matching_url(fid):
raise exceptions.BlockedActorOrDomain()
if queryset:
try:
# queryset can also be a Model class
@ -83,6 +92,16 @@ def retrieve(fid, actor=None, serializer_class=None, queryset=None):
)
response.raise_for_status()
data = response.json()
# we match against moderation policies here again, because the FID of the returned
# object may not be the same as the URL used to access it
try:
id = data["id"]
except KeyError:
pass
else:
if apply_instance_policies and activity.should_reject(id=id, payload=data):
raise exceptions.BlockedActorOrDomain()
if not serializer_class:
return data
serializer = serializer_class(data=data)

18
api/funkwhale_api/federation/views.py
View File

@ -3,7 +3,7 @@ from django.core import paginator
from django.http import HttpResponse
from django.urls import reverse
from rest_framework import exceptions, mixins, response, viewsets
from rest_framework.decorators import detail_route, list_route
from rest_framework.decorators import action
from funkwhale_api.common import preferences
from funkwhale_api.music import models as music_models
@ -23,7 +23,7 @@ class SharedViewSet(FederationMixin, viewsets.GenericViewSet):
authentication_classes = [authentication.SignatureAuthentication]
renderer_classes = [renderers.ActivityPubRenderer]
@list_route(methods=["post"])
@action(methods=["post"], detail=False)
def inbox(self, request, *args, **kwargs):
if request.method.lower() == "post" and request.actor is None:
raise exceptions.AuthenticationFailed(
@ -42,7 +42,7 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
queryset = models.Actor.objects.local().select_related("user")
serializer_class = serializers.ActorSerializer
@detail_route(methods=["get", "post"])
@action(methods=["get", "post"], detail=True)
def inbox(self, request, *args, **kwargs):
if request.method.lower() == "post" and request.actor is None:
raise exceptions.AuthenticationFailed(
@ -52,17 +52,17 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
activity.receive(activity=request.data, on_behalf_of=request.actor)
return response.Response({}, status=200)
@detail_route(methods=["get", "post"])
@action(methods=["get", "post"], detail=True)
def outbox(self, request, *args, **kwargs):
return response.Response({}, status=200)
@detail_route(methods=["get"])
@action(methods=["get"], detail=True)
def followers(self, request, *args, **kwargs):
self.get_object()
# XXX to implement
return response.Response({})
@detail_route(methods=["get"])
@action(methods=["get"], detail=True)
def following(self, request, *args, **kwargs):
self.get_object()
# XXX to implement
@ -74,7 +74,7 @@ class WellKnownViewSet(viewsets.GenericViewSet):
permission_classes = []
renderer_classes = [renderers.JSONRenderer, renderers.WebfingerRenderer]
@list_route(methods=["get"])
@action(methods=["get"], detail=False)
def nodeinfo(self, request, *args, **kwargs):
if not preferences.get("instance__nodeinfo_enabled"):
return HttpResponse(status=404)
@ -88,7 +88,7 @@ class WellKnownViewSet(viewsets.GenericViewSet):
}
return response.Response(data)
@list_route(methods=["get"])
@action(methods=["get"], detail=False)
def webfinger(self, request, *args, **kwargs):
if not preferences.get("federation__enabled"):
return HttpResponse(status=405)
@ -180,7 +180,7 @@ class MusicLibraryViewSet(
return response.Response(data)
@detail_route(methods=["get"])
@action(methods=["get"], detail=True)
def followers(self, request, *args, **kwargs):
self.get_object()
# XXX Implement this

4
api/funkwhale_api/history/factories.py
View File

@ -1,12 +1,12 @@
import factory
from funkwhale_api.factories import registry
from funkwhale_api.factories import registry, NoUpdateOnCreate
from funkwhale_api.music import factories
from funkwhale_api.users.factories import UserFactory
@registry.register
class ListeningFactory(factory.django.DjangoModelFactory):
class ListeningFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
user = factory.SubFactory(UserFactory)
track = factory.SubFactory(factories.TrackFactory)

2
api/funkwhale_api/history/views.py
View File

@ -41,7 +41,7 @@ class ListeningViewSet(
queryset = queryset.filter(
fields.privacy_level_query(self.request.user, "user__privacy_level")
)
tracks = Track.objects.annotate_playable_by_actor(
tracks = Track.objects.with_playable_uploads(
music_utils.get_actor_from_request(self.request)
).select_related("artist", "album__artist")
return queryset.prefetch_related(Prefetch("track", queryset=tracks))

9
api/funkwhale_api/instance/dynamic_preferences_registry.py
View File

@ -58,15 +58,6 @@ class RavenDSN(types.StringPreference):
field_kwargs = {"required": False}
@global_preferences_registry.register
class RavenEnabled(types.BooleanPreference):
show_in_api = True
section = raven
name = "front_enabled"
default = False
verbose_name = "Report front-end errors with Raven"
@global_preferences_registry.register
class InstanceNodeinfoEnabled(types.BooleanPreference):
show_in_api = False

10
api/funkwhale_api/instance/nodeinfo.py
View File

@ -17,7 +17,7 @@ def get():
"protocols": ["activitypub"],
"services": {"inbound": [], "outbound": []},
"openRegistrations": preferences.get("users__registration_enabled"),
"usage": {"users": {"total": 0}},
"usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}},
"metadata": {
"private": preferences.get("instance__nodeinfo_private"),
"shortDescription": preferences.get("instance__short_description"),
@ -28,7 +28,7 @@ def get():
"federationNeedsApproval": preferences.get(
"federation__music_needs_approval"
),
"anonymousCanListen": preferences.get(
"anonymousCanListen": not preferences.get(
"common__api_authentication_required"
),
},
@ -37,7 +37,11 @@ def get():
if share_stats:
getter = memo(lambda: stats.get(), max_age=600)
statistics = getter()
data["usage"]["users"]["total"] = statistics["users"]
data["usage"]["users"]["total"] = statistics["users"]["total"]
data["usage"]["users"]["activeHalfyear"] = statistics["users"][
"active_halfyear"
]
data["usage"]["users"]["activeMonth"] = statistics["users"]["active_month"]
data["metadata"]["library"]["tracks"] = {"total": statistics["tracks"]}
data["metadata"]["library"]["artists"] = {"total": statistics["artists"]}
data["metadata"]["library"]["albums"] = {"total": statistics["albums"]}

12
api/funkwhale_api/instance/stats.py
View File

@ -1,4 +1,7 @@
import datetime
from django.db.models import Sum
from django.utils import timezone
from funkwhale_api.favorites.models import TrackFavorite
from funkwhale_api.history.models import Listening
@ -19,6 +22,15 @@ def get():
def get_users():
qs = User.objects.filter(is_active=True)
now = timezone.now()
active_month = now - datetime.timedelta(days=30)
active_halfyear = now - datetime.timedelta(days=30 * 6)
return {
"total": qs.count(),
"active_month": qs.filter(last_activity__gte=active_month).count(),
"active_halfyear": qs.filter(last_activity__gte=active_halfyear).count(),
}
return User.objects.count()

63
api/funkwhale_api/manage/filters.py
View File

@ -1,6 +1,10 @@
from django_filters import rest_framework as filters
from funkwhale_api.common import fields
from funkwhale_api.common import search
from funkwhale_api.federation import models as federation_models
from funkwhale_api.moderation import models as moderation_models
from funkwhale_api.music import models as music_models
from funkwhale_api.users import models as users_models
@ -20,6 +24,41 @@ class ManageUploadFilterSet(filters.FilterSet):
fields = ["q", "track__album", "track__artist", "track"]
class ManageDomainFilterSet(filters.FilterSet):
q = fields.SearchFilter(search_fields=["name"])
class Meta:
model = federation_models.Domain
fields = ["name"]
class ManageActorFilterSet(filters.FilterSet):
q = fields.SmartSearchFilter(
config=search.SearchConfig(
search_fields={
"name": {"to": "name"},
"username": {"to": "preferred_username"},
"email": {"to": "user__email"},
"bio": {"to": "summary"},
"type": {"to": "type"},
},
filter_fields={
"domain": {"to": "domain__name__iexact"},
"username": {"to": "preferred_username__iexact"},
"email": {"to": "user__email__iexact"},
},
)
)
local = filters.BooleanFilter(field_name="_", method="filter_local")
class Meta:
model = federation_models.Actor
fields = ["q", "domain", "type", "manually_approves_followers", "local"]
def filter_local(self, queryset, name, value):
return queryset.local(value)
class ManageUserFilterSet(filters.FilterSet):
q = fields.SearchFilter(search_fields=["username", "email", "name"])
@ -31,10 +70,9 @@ class ManageUserFilterSet(filters.FilterSet):
"privacy_level",
"is_staff",
"is_superuser",
"permission_upload",
"permission_library",
"permission_settings",
"permission_federation",
"permission_moderation",
]
@ -50,3 +88,24 @@ class ManageInvitationFilterSet(filters.FilterSet):
if value is None:
return queryset
return queryset.open(value)
class ManageInstancePolicyFilterSet(filters.FilterSet):
q = fields.SearchFilter(
search_fields=[
"summary",
"target_domain__name",
"target_actor__username",
"target_actor__domain__name",
]
)
class Meta:
model = moderation_models.InstancePolicy
fields = [
"q",
"block_all",
"silence_activity",
"silence_notifications",
"reject_media",
]

171
api/funkwhale_api/manage/serializers.py
View File

@ -3,6 +3,11 @@ from django.db import transaction
from rest_framework import serializers
from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import fields as federation_fields
from funkwhale_api.federation import tasks as federation_tasks
from funkwhale_api.moderation import models as moderation_models
from funkwhale_api.music import models as music_models
from funkwhale_api.users import models as users_models
@ -115,6 +120,7 @@ class ManageUserSerializer(serializers.ModelSerializer):
"permissions",
"privacy_level",
"upload_quota",
"full_username",
)
read_only_fields = [
"id",
@ -168,3 +174,168 @@ class ManageInvitationActionSerializer(common_serializers.ActionSerializer):
@transaction.atomic
def handle_delete(self, objects):
return objects.delete()
class ManageDomainSerializer(serializers.ModelSerializer):
actors_count = serializers.SerializerMethodField()
outbox_activities_count = serializers.SerializerMethodField()
class Meta:
model = federation_models.Domain
fields = [
"name",
"creation_date",
"actors_count",
"outbox_activities_count",
"nodeinfo",
"nodeinfo_fetch_date",
"instance_policy",
]
read_only_fields = [
"creation_date",
"instance_policy",
"nodeinfo",
"nodeinfo_fetch_date",
]
def get_actors_count(self, o):
return getattr(o, "actors_count", 0)
def get_outbox_activities_count(self, o):
return getattr(o, "outbox_activities_count", 0)
class ManageDomainActionSerializer(common_serializers.ActionSerializer):
actions = [common_serializers.Action("purge", allow_all=False)]
filterset_class = filters.ManageDomainFilterSet
pk_field = "name"
@transaction.atomic
def handle_purge(self, objects):
ids = objects.values_list("pk", flat=True)
common_utils.on_commit(federation_tasks.purge_actors.delay, domains=list(ids))
class ManageActorSerializer(serializers.ModelSerializer):
uploads_count = serializers.SerializerMethodField()
user = ManageUserSerializer()
class Meta:
model = federation_models.Actor
fields = [
"id",
"url",
"fid",
"preferred_username",
"full_username",
"domain",
"name",
"summary",
"type",
"creation_date",
"last_fetch_date",
"inbox_url",
"outbox_url",
"shared_inbox_url",
"manually_approves_followers",
"uploads_count",
"user",
"instance_policy",
]
read_only_fields = ["creation_date", "instance_policy"]
def get_uploads_count(self, o):
return getattr(o, "uploads_count", 0)
class ManageActorActionSerializer(common_serializers.ActionSerializer):
actions = [common_serializers.Action("purge", allow_all=False)]
filterset_class = filters.ManageActorFilterSet
@transaction.atomic
def handle_purge(self, objects):
ids = objects.values_list("id", flat=True)
common_utils.on_commit(federation_tasks.purge_actors.delay, ids=list(ids))
class TargetSerializer(serializers.Serializer):
type = serializers.ChoiceField(choices=["domain", "actor"])
id = serializers.CharField()
def to_representation(self, value):
if value["type"] == "domain":
return {"type": "domain", "id": value["obj"].name}
if value["type"] == "actor":
return {"type": "actor", "id": value["obj"].full_username}
def to_internal_value(self, value):
if value["type"] == "domain":
field = serializers.PrimaryKeyRelatedField(
queryset=federation_models.Domain.objects.external()
)
if value["type"] == "actor":
field = federation_fields.ActorRelatedField()
value["obj"] = field.to_internal_value(value["id"])
return value
class ManageInstancePolicySerializer(serializers.ModelSerializer):
target = TargetSerializer()
actor = federation_fields.ActorRelatedField(read_only=True)
class Meta:
model = moderation_models.InstancePolicy
fields = [
"id",
"uuid",
"target",
"creation_date",
"actor",
"summary",
"is_active",
"block_all",
"silence_activity",
"silence_notifications",
"reject_media",
]
read_only_fields = ["uuid", "id", "creation_date", "actor", "target"]
def validate(self, data):
try:
target = data.pop("target")
except KeyError:
# partial update
return data
if target["type"] == "domain":
data["target_domain"] = target["obj"]
if target["type"] == "actor":
data["target_actor"] = target["obj"]
return data
@transaction.atomic
def save(self, *args, **kwargs):
instance = super().save(*args, **kwargs)
need_purge = self.instance.is_active and (
self.instance.block_all or self.instance.reject_media
)
if need_purge:
only = []
if self.instance.reject_media:
only.append("media")
target = instance.target
if target["type"] == "domain":
common_utils.on_commit(
federation_tasks.purge_actors.delay,
domains=[target["obj"].pk],
only=only,
)
if target["type"] == "actor":
common_utils.on_commit(
federation_tasks.purge_actors.delay,
ids=[target["obj"].pk],
only=only,
)
return instance

22
api/funkwhale_api/manage/urls.py
View File

@ -3,13 +3,33 @@ from rest_framework import routers
from . import views
federation_router = routers.SimpleRouter()
federation_router.register(r"domains", views.ManageDomainViewSet, "domains")
library_router = routers.SimpleRouter()
library_router.register(r"uploads", views.ManageUploadViewSet, "uploads")
moderation_router = routers.SimpleRouter()
moderation_router.register(
r"instance-policies", views.ManageInstancePolicyViewSet, "instance-policies"
)
users_router = routers.SimpleRouter()
users_router.register(r"users", views.ManageUserViewSet, "users")
users_router.register(r"invitations", views.ManageInvitationViewSet, "invitations")
other_router = routers.SimpleRouter()
other_router.register(r"accounts", views.ManageActorViewSet, "accounts")
urlpatterns = [
url(
r"^federation/",
include((federation_router.urls, "federation"), namespace="federation"),
),
url(r"^library/", include((library_router.urls, "instance"), namespace="library")),
url(
r"^moderation/",
include((moderation_router.urls, "moderation"), namespace="moderation"),
),
url(r"^users/", include((users_router.urls, "instance"), namespace="users")),
]
] + other_router.urls

127
api/funkwhale_api/manage/views.py
View File

@ -1,8 +1,12 @@
from rest_framework import mixins, response, viewsets
from rest_framework.decorators import list_route
from rest_framework import decorators as rest_decorators
from django.shortcuts import get_object_or_404
from funkwhale_api.common import preferences
from funkwhale_api.common import preferences, decorators
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import tasks as federation_tasks
from funkwhale_api.music import models as music_models
from funkwhale_api.moderation import models as moderation_models
from funkwhale_api.users import models as users_models
from funkwhale_api.users.permissions import HasUserPermission
@ -18,7 +22,7 @@ class ManageUploadViewSet(
.order_by("-id")
)
serializer_class = serializers.ManageUploadSerializer
filter_class = filters.ManageUploadFilterSet
filterset_class = filters.ManageUploadFilterSet
permission_classes = (HasUserPermission,)
required_permissions = ["library"]
ordering_fields = [
@ -31,7 +35,7 @@ class ManageUploadViewSet(
"duration",
]
@list_route(methods=["post"])
@rest_decorators.action(methods=["post"], detail=False)
def action(self, request, *args, **kwargs):
queryset = self.get_queryset()
serializer = serializers.ManageUploadActionSerializer(
@ -50,7 +54,7 @@ class ManageUserViewSet(
):
queryset = users_models.User.objects.all().order_by("-id")
serializer_class = serializers.ManageUserSerializer
filter_class = filters.ManageUserFilterSet
filterset_class = filters.ManageUserFilterSet
permission_classes = (HasUserPermission,)
required_permissions = ["settings"]
ordering_fields = ["date_joined", "last_activity", "username"]
@ -75,7 +79,7 @@ class ManageInvitationViewSet(
.select_related("owner")
)
serializer_class = serializers.ManageInvitationSerializer
filter_class = filters.ManageInvitationFilterSet
filterset_class = filters.ManageInvitationFilterSet
permission_classes = (HasUserPermission,)
required_permissions = ["settings"]
ordering_fields = ["creation_date", "expiration_date"]
@ -83,7 +87,7 @@ class ManageInvitationViewSet(
def perform_create(self, serializer):
serializer.save(owner=self.request.user)
@list_route(methods=["post"])
@rest_decorators.action(methods=["post"], detail=False)
def action(self, request, *args, **kwargs):
queryset = self.get_queryset()
serializer = serializers.ManageInvitationActionSerializer(
@ -92,3 +96,112 @@ class ManageInvitationViewSet(
serializer.is_valid(raise_exception=True)
result = serializer.save()
return response.Response(result, status=200)
class ManageDomainViewSet(
mixins.CreateModelMixin,
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
viewsets.GenericViewSet,
):
lookup_value_regex = r"[a-zA-Z0-9\-\.]+"
queryset = (
federation_models.Domain.objects.external()
.with_actors_count()
.with_outbox_activities_count()
.prefetch_related("instance_policy")
.order_by("name")
)
serializer_class = serializers.ManageDomainSerializer
filterset_class = filters.ManageDomainFilterSet
permission_classes = (HasUserPermission,)
required_permissions = ["moderation"]
ordering_fields = [
"name",
"creation_date",
"nodeinfo_fetch_date",
"actors_count",
"outbox_activities_count",
"instance_policy",
]
@rest_decorators.action(methods=["get"], detail=True)
def nodeinfo(self, request, *args, **kwargs):
domain = self.get_object()
federation_tasks.update_domain_nodeinfo(domain_name=domain.name)
domain.refresh_from_db()
return response.Response(domain.nodeinfo, status=200)
@rest_decorators.action(methods=["get"], detail=True)
def stats(self, request, *args, **kwargs):
domain = self.get_object()
return response.Response(domain.get_stats(), status=200)
action = decorators.action_route(serializers.ManageDomainActionSerializer)
class ManageActorViewSet(
mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
):
lookup_value_regex = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
queryset = (
federation_models.Actor.objects.all()
.with_uploads_count()
.order_by("-creation_date")
.select_related("user")
.prefetch_related("instance_policy")
)
serializer_class = serializers.ManageActorSerializer
filterset_class = filters.ManageActorFilterSet
permission_classes = (HasUserPermission,)
required_permissions = ["moderation"]
ordering_fields = [
"name",
"preferred_username",
"domain",
"fid",
"creation_date",
"last_fetch_date",
"uploads_count",
"outbox_activities_count",
"instance_policy",
]
def get_object(self):
queryset = self.filter_queryset(self.get_queryset())
username, domain = self.kwargs["pk"].split("@")
filter_kwargs = {"domain_id": domain, "preferred_username": username}
obj = get_object_or_404(queryset, **filter_kwargs)
self.check_object_permissions(self.request, obj)
return obj
@rest_decorators.action(methods=["get"], detail=True)
def stats(self, request, *args, **kwargs):
domain = self.get_object()
return response.Response(domain.get_stats(), status=200)
action = decorators.action_route(serializers.ManageActorActionSerializer)
class ManageInstancePolicyViewSet(
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
mixins.DestroyModelMixin,
mixins.CreateModelMixin,
mixins.UpdateModelMixin,
viewsets.GenericViewSet,
):
queryset = (
moderation_models.InstancePolicy.objects.all()
.order_by("-creation_date")
.select_related()
)
serializer_class = serializers.ManageInstancePolicySerializer
filterset_class = filters.ManageInstancePolicyFilterSet
permission_classes = (HasUserPermission,)
required_permissions = ["moderation"]
ordering_fields = ["id", "creation_date"]
def perform_create(self, serializer):
serializer.save(actor=self.request.user.actor)

0
api/funkwhale_api/moderation/__init__.py Normal file
View File

30
api/funkwhale_api/moderation/admin.py Normal file
View File

@ -0,0 +1,30 @@
from funkwhale_api.common import admin
from . import models
@admin.register(models.InstancePolicy)
class InstancePolicyAdmin(admin.ModelAdmin):
list_display = [
"actor",
"target_domain",
"target_actor",
"creation_date",
"block_all",
"reject_media",
"silence_activity",
"silence_notifications",
]
list_filter = [
"block_all",
"reject_media",
"silence_activity",
"silence_notifications",
]
search_fields = [
"actor__fid",
"target_domain__name",
"target_domain__actor__fid",
"summary",
]
list_select_related = True

23
api/funkwhale_api/moderation/factories.py Normal file
View File

@ -0,0 +1,23 @@
import factory
from funkwhale_api.factories import registry, NoUpdateOnCreate
from funkwhale_api.federation import factories as federation_factories
@registry.register
class InstancePolicyFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
summary = factory.Faker("paragraph")
actor = factory.SubFactory(federation_factories.ActorFactory)
block_all = True
is_active = True
class Meta:
model = "moderation.InstancePolicy"
class Params:
for_domain = factory.Trait(
target_domain=factory.SubFactory(federation_factories.DomainFactory)
)
for_actor = factory.Trait(
target_actor=factory.SubFactory(federation_factories.ActorFactory)
)

35
api/funkwhale_api/moderation/migrations/0001_initial.py Normal file
View File

@ -0,0 +1,35 @@
# Generated by Django 2.0.9 on 2019-01-07 06:06
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import uuid
class Migration(migrations.Migration):
initial = True
dependencies = [
('federation', '0016_auto_20181227_1605'),
]
operations = [
migrations.CreateModel(
name='InstancePolicy',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('is_active', models.BooleanField(default=True)),
('summary', models.TextField(blank=True, max_length=10000, null=True)),
('block_all', models.BooleanField(default=False)),
('silence_activity', models.BooleanField(default=False)),
('silence_notifications', models.BooleanField(default=False)),
('reject_media', models.BooleanField(default=False)),
('actor', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='created_instance_policies', to='federation.Actor')),
('target_actor', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='instance_policy', to='federation.Actor')),
('target_domain', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='instance_policy', to='federation.Domain')),
],
),
]

0
api/funkwhale_api/moderation/migrations/__init__.py Normal file
View File

75
api/funkwhale_api/moderation/models.py Normal file
View File

@ -0,0 +1,75 @@
import urllib.parse
import uuid
from django.db import models
from django.utils import timezone
class InstancePolicyQuerySet(models.QuerySet):
def active(self):
return self.filter(is_active=True)
def matching_url(self, *urls):
if not urls:
return self.none()
query = None
for url in urls:
new_query = self.matching_url_query(url)
if query:
query = query | new_query
else:
query = new_query
return self.filter(query)
def matching_url_query(self, url):
parsed = urllib.parse.urlparse(url)
return models.Q(target_domain_id=parsed.hostname) | models.Q(
target_actor__fid=url
)
class InstancePolicy(models.Model):
uuid = models.UUIDField(default=uuid.uuid4, unique=True)
actor = models.ForeignKey(
"federation.Actor",
related_name="created_instance_policies",
on_delete=models.SET_NULL,
null=True,
blank=True,
)
target_domain = models.OneToOneField(
"federation.Domain",
related_name="instance_policy",
on_delete=models.CASCADE,
null=True,
blank=True,
)
target_actor = models.OneToOneField(
"federation.Actor",
related_name="instance_policy",
on_delete=models.CASCADE,
null=True,
blank=True,
)
creation_date = models.DateTimeField(default=timezone.now)
is_active = models.BooleanField(default=True)
# a summary explaining why the policy is in place
summary = models.TextField(max_length=10000, null=True, blank=True)
# either block everything (simpler, but less granularity)
block_all = models.BooleanField(default=False)
# or pick individual restrictions below
# do not show in timelines/notifications, except for actual followers
silence_activity = models.BooleanField(default=False)
silence_notifications = models.BooleanField(default=False)
# do not download any media from the target
reject_media = models.BooleanField(default=False)
objects = InstancePolicyQuerySet.as_manager()
@property
def target(self):
if self.target_actor:
return {"type": "actor", "obj": self.target_actor}
if self.target_domain_id:
return {"type": "domain", "obj": self.target_domain}

22
api/funkwhale_api/music/admin.py
View File

@ -78,6 +78,28 @@ class UploadAdmin(admin.ModelAdmin):
list_filter = ["mimetype", "import_status", "library__privacy_level"]
@admin.register(models.UploadVersion)
class UploadVersionAdmin(admin.ModelAdmin):
list_display = [
"upload",
"audio_file",
"mimetype",
"size",
"bitrate",
"creation_date",
"accessed_date",
]
list_select_related = ["upload"]
search_fields = [
"upload__source",
"upload__acoustid_track_id",
"upload__track__title",
"upload__track__album__title",
"upload__track__artist__name",
]
list_filter = ["mimetype"]
def launch_scan(modeladmin, request, queryset):
for library in queryset:
library.schedule_scan(actor=request.user.actor, force=True)

34
api/funkwhale_api/music/dynamic_preferences_registry.py Normal file
View File

@ -0,0 +1,34 @@
from dynamic_preferences import types
from dynamic_preferences.registries import global_preferences_registry
music = types.Section("music")
@global_preferences_registry.register
class MaxTracks(types.BooleanPreference):
show_in_api = True
section = music
name = "transcoding_enabled"
verbose_name = "Transcoding enabled"
help_text = (
"Enable transcoding of audio files in formats requested by the client. "
"This is especially useful for devices that do not support formats "
"such as Flac or Ogg, but the transcoding process will increase the "
"load on the server."
)
default = True
@global_preferences_registry.register
class MusicCacheDuration(types.IntPreference):
show_in_api = True
section = music
name = "transcoding_cache_duration"
default = 60 * 24 * 7
verbose_name = "Transcoding cache duration"
help_text = (
"How much minutes do you want to keep a copy of transcoded tracks "
"on the server? Transcoded files that were not listened in this interval "
"will be erased and retranscoded on the next listening."
)
field_kwargs = {"required": False}

71
api/funkwhale_api/music/factories.py
View File

@ -2,10 +2,11 @@ import os
import factory
from funkwhale_api.factories import ManyToManyFromList, registry
from funkwhale_api.federation import factories as federation_factories
from funkwhale_api.users import factories as users_factories
from funkwhale_api.factories import ManyToManyFromList, registry, NoUpdateOnCreate
from funkwhale_api.federation import factories as federation_factories
from funkwhale_api.music import licenses
from funkwhale_api.users import factories as users_factories
SAMPLES_PATH = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
@ -30,8 +31,31 @@ def playable_factory(field):
return inner
def deduce_from_conf(field):
@factory.lazy_attribute
def inner(self):
return licenses.LICENSES_BY_ID[self.code][field]
return inner
@registry.register
class ArtistFactory(factory.django.DjangoModelFactory):
class LicenseFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
code = "cc-by-4.0"
url = deduce_from_conf("url")
commercial = deduce_from_conf("commercial")
redistribute = deduce_from_conf("redistribute")
copyleft = deduce_from_conf("copyleft")
attribution = deduce_from_conf("attribution")
derivative = deduce_from_conf("derivative")
class Meta:
model = "music.License"
django_get_or_create = ("code",)
@registry.register
class ArtistFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
name = factory.Faker("name")
mbid = factory.Faker("uuid4")
fid = factory.Faker("federation_url")
@ -42,7 +66,7 @@ class ArtistFactory(factory.django.DjangoModelFactory):
@registry.register
class AlbumFactory(factory.django.DjangoModelFactory):
class AlbumFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
title = factory.Faker("sentence", nb_words=3)
mbid = factory.Faker("uuid4")
release_date = factory.Faker("date_object")
@ -57,7 +81,7 @@ class AlbumFactory(factory.django.DjangoModelFactory):
@registry.register
class TrackFactory(factory.django.DjangoModelFactory):
class TrackFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
fid = factory.Faker("federation_url")
title = factory.Faker("sentence", nb_words=3)
mbid = factory.Faker("uuid4")
@ -70,9 +94,18 @@ class TrackFactory(factory.django.DjangoModelFactory):
class Meta:
model = "music.Track"
@factory.post_generation
def license(self, created, extracted, **kwargs):
if not created:
return
if extracted:
self.license = LicenseFactory(code=extracted)
self.save()
@registry.register
class UploadFactory(factory.django.DjangoModelFactory):
class UploadFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
fid = factory.Faker("federation_url")
track = factory.SubFactory(TrackFactory)
library = factory.SubFactory(federation_factories.MusicLibraryFactory)
@ -89,14 +122,26 @@ class UploadFactory(factory.django.DjangoModelFactory):
model = "music.Upload"
class Params:
in_place = factory.Trait(audio_file=None)
in_place = factory.Trait(audio_file=None, mimetype=None)
playable = factory.Trait(
import_status="finished", library__privacy_level="everyone"
)
@registry.register
class WorkFactory(factory.django.DjangoModelFactory):
class UploadVersionFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
upload = factory.SubFactory(UploadFactory, bitrate=200000)
bitrate = factory.SelfAttribute("upload.bitrate")
mimetype = "audio/mpeg"
audio_file = factory.django.FileField()
size = 2000000
class Meta:
model = "music.UploadVersion"
@registry.register
class WorkFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
mbid = factory.Faker("uuid4")
language = "eng"
nature = "song"
@ -107,7 +152,7 @@ class WorkFactory(factory.django.DjangoModelFactory):
@registry.register
class LyricsFactory(factory.django.DjangoModelFactory):
class LyricsFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
work = factory.SubFactory(WorkFactory)
url = factory.Faker("url")
content = factory.Faker("paragraphs", nb=4)
@ -117,7 +162,7 @@ class LyricsFactory(factory.django.DjangoModelFactory):
@registry.register
class TagFactory(factory.django.DjangoModelFactory):
class TagFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
name = factory.SelfAttribute("slug")
slug = factory.Faker("slug")
@ -128,7 +173,7 @@ class TagFactory(factory.django.DjangoModelFactory):
# XXX To remove
class ImportBatchFactory(factory.django.DjangoModelFactory):
class ImportBatchFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
submitted_by = factory.SubFactory(users_factories.UserFactory)
class Meta:
@ -136,7 +181,7 @@ class ImportBatchFactory(factory.django.DjangoModelFactory):
@registry.register
class ImportJobFactory(factory.django.DjangoModelFactory):
class ImportJobFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
batch = factory.SubFactory(ImportBatchFactory)
source = factory.Faker("url")
mbid = factory.Faker("uuid4")

9
api/funkwhale_api/music/filters.py
View File

@ -9,7 +9,7 @@ from . import utils
class ArtistFilter(filters.FilterSet):
q = fields.SearchFilter(search_fields=["name"])
playable = filters.BooleanFilter(name="_", method="filter_playable")
playable = filters.BooleanFilter(field_name="_", method="filter_playable")
class Meta:
model = models.Artist
@ -25,7 +25,7 @@ class ArtistFilter(filters.FilterSet):
class TrackFilter(filters.FilterSet):
q = fields.SearchFilter(search_fields=["title", "album__title", "artist__name"])
playable = filters.BooleanFilter(name="_", method="filter_playable")
playable = filters.BooleanFilter(field_name="_", method="filter_playable")
class Meta:
model = models.Track
@ -34,6 +34,7 @@ class TrackFilter(filters.FilterSet):
"playable": ["exact"],
"artist": ["exact"],
"album": ["exact"],
"license": ["exact"],
}
def filter_playable(self, queryset, name, value):
@ -47,7 +48,7 @@ class UploadFilter(filters.FilterSet):
track_artist = filters.UUIDFilter("track__artist__uuid")
album_artist = filters.UUIDFilter("track__album__artist__uuid")
library = filters.UUIDFilter("library__uuid")
playable = filters.BooleanFilter(name="_", method="filter_playable")
playable = filters.BooleanFilter(field_name="_", method="filter_playable")
q = fields.SmartSearchFilter(
config=search.SearchConfig(
search_fields={
@ -85,7 +86,7 @@ class UploadFilter(filters.FilterSet):
class AlbumFilter(filters.FilterSet):
playable = filters.BooleanFilter(name="_", method="filter_playable")
playable = filters.BooleanFilter(field_name="_", method="filter_playable")
q = fields.SearchFilter(search_fields=["title", "artist__name" "source"])
class Meta:

363
api/funkwhale_api/music/licenses.py Normal file
View File

@ -0,0 +1,363 @@
import logging
import re
from django.db import transaction
from . import models
logger = logging.getLogger(__name__)
MODEL_FIELDS = [
"redistribute",
"derivative",
"attribution",
"copyleft",
"commercial",
"url",
]
@transaction.atomic
def load(data):
"""
Load/update database objects with our hardcoded data
"""
existing = models.License.objects.all()
existing_by_code = {e.code: e for e in existing}
to_create = []
for row in data:
try:
license = existing_by_code[row["code"]]
except KeyError:
logger.info("Loading new license: {}".format(row["code"]))
to_create.append(
models.License(code=row["code"], **{f: row[f] for f in MODEL_FIELDS})
)
else:
logger.info("Updating license: {}".format(row["code"]))
stored = [getattr(license, f) for f in MODEL_FIELDS]
wanted = [row[f] for f in MODEL_FIELDS]
if wanted == stored:
continue
# the object in database needs an update
for f in MODEL_FIELDS:
setattr(license, f, row[f])
license.save()
models.License.objects.bulk_create(to_create)
return sorted(models.License.objects.all(), key=lambda o: o.code)
_cache = None
def match(*values):
"""
Given a string, extracted from music file tags, return corresponding License
instance, if found
"""
global _cache
for value in values:
if not value:
continue
# we are looking for the first url in our value
# This regex is not perfect, but it's good enough for now
urls = re.findall(
r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+",
value,
)
if not urls:
logger.debug('Impossible to guess license from string "{}"'.format(value))
continue
url = urls[0]
if _cache:
existing = _cache
else:
existing = load(LICENSES)
_cache = existing
for license in existing:
if license.conf is None:
continue
for i in license.conf["identifiers"]:
if match_urls(url, i):
return license
def match_urls(*urls):
"""
We want to ensure the two url match but don't care for protocol
or trailing slashes
"""
urls = [u.rstrip("/") for u in urls]
urls = [u.lstrip("http://") for u in urls]
urls = [u.lstrip("https://") for u in urls]
return len(set(urls)) == 1
def get_cc_license(version, perks, country=None, country_name=None):
if len(perks) == 0:
raise ValueError("No perks!")
url_template = "//creativecommons.org/licenses/{type}/{version}/"
code_parts = []
name_parts = []
perks_data = [
("by", "Attribution"),
("nc", "NonCommercial"),
("sa", "ShareAlike"),
("nd", "NoDerivatives"),
]
for perk, name in perks_data:
if perk in perks:
code_parts.append(perk)
name_parts.append(name)
url = url_template.format(version=version, type="-".join(code_parts))
code_parts.append(version)
name = "Creative commons - {perks} {version}".format(
perks="-".join(name_parts), version=version
)
if country:
code_parts.append(country)
name += " {}".format(country_name)
url += country + "/"
data = {
"name": name,
"code": "cc-{}".format("-".join(code_parts)),
"redistribute": True,
"commercial": "nc" not in perks,
"derivative": "nd" not in perks,
"copyleft": "sa" in perks,
"attribution": "by" in perks,
"url": "https:" + url,
"identifiers": ["http:" + url],
}
return data
COUNTRIES = {
"ar": "Argentina",
"au": "Australia",
"at": "Austria",
"be": "Belgium",
"br": "Brazil",
"bg": "Bulgaria",
"ca": "Canada",
"cl": "Chile",
"cn": "China Mainland",
"co": "Colombia",
"cr": "Costa Rica",
"hr": "Croatia",
"cz": "Czech Republic",
"dk": "Denmark",
"ec": "Ecuador",
"eg": "Egypt",
"ee": "Estonia",
"fi": "Finland",
"fr": "France",
"de": "Germany",
"gr": "Greece",
"gt": "Guatemala",
"hk": "Hong Kong",
"hu": "Hungary",
"igo": "IGO",
"in": "India",
"ie": "Ireland",
"il": "Israel",
"it": "Italy",
"jp": "Japan",
"lu": "Luxembourg",
"mk": "Macedonia",
"my": "Malaysia",
"mt": "Malta",
"mx": "Mexico",
"nl": "Netherlands",
"nz": "New Zealand",
"no": "Norway",
"pe": "Peru",
"ph": "Philippines",
"pl": "Poland",
"pt": "Portugal",
"pr": "Puerto Rico",
"ro": "Romania",
"rs": "Serbia",
"sg": "Singapore",
"si": "Slovenia",
"za": "South Africa",
"kr": "South Korea",
"es": "Spain",
"se": "Sweden",
"ch": "Switzerland",
"tw": "Taiwan",
"th": "Thailand",
"uk": "UK: England &amp; Wales",
"scotland": "UK: Scotland",
"ug": "Uganda",
"us": "United States",
"ve": "Venezuela",
"vn": "Vietnam",
}
CC_30_COUNTRIES = [
"at",
"au",
"br",
"ch",
"cl",
"cn",
"cr",
"cz",
"de",
"ec",
"ee",
"eg",
"es",
"fr",
"gr",
"gt",
"hk",
"hr",
"ie",
"igo",
"it",
"lu",
"nl",
"no",
"nz",
"ph",
"pl",
"pr",
"pt",
"ro",
"rs",
"sg",
"th",
"tw",
"ug",
"us",
"ve",
"vn",
"za",
]
CC_25_COUNTRIES = [
"ar",
"bg",
"ca",
"co",
"dk",
"hu",
"il",
"in",
"mk",
"mt",
"mx",
"my",
"pe",
"scotland",
]
LICENSES = [
# a non-exhaustive list: http://musique-libre.org/doc/le-tableau-des-licences-libres-et-ouvertes-de-dogmazic/
{
"code": "cc0-1.0",
"name": "CC0 - Public domain",
"redistribute": True,
"derivative": True,
"commercial": True,
"attribution": False,
"copyleft": False,
"url": "https://creativecommons.org/publicdomain/zero/1.0/",
"identifiers": [
# note the http here.
# This is the kind of URL that is embedded in music files metadata
"http://creativecommons.org/publicdomain/zero/1.0/"
],
},
# Creative commons version 4.0
get_cc_license(version="4.0", perks=["by"]),
get_cc_license(version="4.0", perks=["by", "sa"]),
get_cc_license(version="4.0", perks=["by", "nc"]),
get_cc_license(version="4.0", perks=["by", "nc", "sa"]),
get_cc_license(version="4.0", perks=["by", "nc", "nd"]),
get_cc_license(version="4.0", perks=["by", "nd"]),
# Creative commons version 3.0
get_cc_license(version="3.0", perks=["by"]),
get_cc_license(version="3.0", perks=["by", "sa"]),
get_cc_license(version="3.0", perks=["by", "nc"]),
get_cc_license(version="3.0", perks=["by", "nc", "sa"]),
get_cc_license(version="3.0", perks=["by", "nc", "nd"]),
get_cc_license(version="3.0", perks=["by", "nd"]),
# Creative commons version 2.5
get_cc_license(version="2.5", perks=["by"]),
get_cc_license(version="2.5", perks=["by", "sa"]),
get_cc_license(version="2.5", perks=["by", "nc"]),
get_cc_license(version="2.5", perks=["by", "nc", "sa"]),
get_cc_license(version="2.5", perks=["by", "nc", "nd"]),
get_cc_license(version="2.5", perks=["by", "nd"]),
# Creative commons version 2.0
get_cc_license(version="2.0", perks=["by"]),
get_cc_license(version="2.0", perks=["by", "sa"]),
get_cc_license(version="2.0", perks=["by", "nc"]),
get_cc_license(version="2.0", perks=["by", "nc", "sa"]),
get_cc_license(version="2.0", perks=["by", "nc", "nd"]),
get_cc_license(version="2.0", perks=["by", "nd"]),
# Creative commons version 1.0
get_cc_license(version="1.0", perks=["by"]),
get_cc_license(version="1.0", perks=["by", "sa"]),
get_cc_license(version="1.0", perks=["by", "nc"]),
get_cc_license(version="1.0", perks=["by", "nc", "sa"]),
get_cc_license(version="1.0", perks=["by", "nc", "nd"]),
get_cc_license(version="1.0", perks=["by", "nd"]),
]
# generate ported (by country) CC licenses:
for country in CC_30_COUNTRIES:
name = COUNTRIES[country]
LICENSES += [
get_cc_license(version="3.0", perks=["by"], country=country, country_name=name),
get_cc_license(
version="3.0", perks=["by", "sa"], country=country, country_name=name
),
get_cc_license(
version="3.0", perks=["by", "nc"], country=country, country_name=name
),
get_cc_license(
version="3.0", perks=["by", "nc", "sa"], country=country, country_name=name
),
get_cc_license(
version="3.0", perks=["by", "nc", "nd"], country=country, country_name=name
),
get_cc_license(
version="3.0", perks=["by", "nd"], country=country, country_name=name
),
]
for country in CC_25_COUNTRIES:
name = COUNTRIES[country]
LICENSES += [
get_cc_license(version="2.5", perks=["by"], country=country, country_name=name),
get_cc_license(
version="2.5", perks=["by", "sa"], country=country, country_name=name
),
get_cc_license(
version="2.5", perks=["by", "nc"], country=country, country_name=name
),
get_cc_license(
version="2.5", perks=["by", "nc", "sa"], country=country, country_name=name
),
get_cc_license(
version="2.5", perks=["by", "nc", "nd"], country=country, country_name=name
),
get_cc_license(
version="2.5", perks=["by", "nd"], country=country, country_name=name
),
]
LICENSES = sorted(LICENSES, key=lambda l: l["code"])
LICENSES_BY_ID = {l["code"]: l for l in LICENSES}

34
api/funkwhale_api/music/management/commands/check_licenses.py Normal file
View File

@ -0,0 +1,34 @@
from django.core.management.base import BaseCommand, CommandError
import requests.exceptions
from funkwhale_api.music import licenses
class Command(BaseCommand):
help = "Check that specified licenses URLs are actually reachable"
def handle(self, *args, **options):
errored = []
objs = licenses.LICENSES
total = len(objs)
for i, data in enumerate(objs):
self.stderr.write("{}/{} Checking {}...".format(i + 1, total, data["code"]))
response = requests.get(data["url"])
try:
response.raise_for_status()
except requests.exceptions.RequestException:
self.stderr.write("!!! Error while fetching {}!".format(data["code"]))
errored.append((data, response))
if errored:
self.stdout.write("{} licenses were not reachable!".format(len(errored)))
for row, response in errored:
self.stdout.write(
"- {}: error {} at url {}".format(
row["code"], response.status_code, row["url"]
)
)
raise CommandError()
else:
self.stdout.write("All licenses are valid and reachable :)")

173
api/funkwhale_api/music/metadata.py
View File

@ -1,8 +1,16 @@
import base64
import datetime
import mutagen
import logging
import pendulum
import mutagen._util
import mutagen.oggtheora
import mutagen.oggvorbis
import mutagen.flac
from django import forms
logger = logging.getLogger(__name__)
NODEFAULT = object()
@ -14,14 +22,26 @@ class UnsupportedTag(KeyError):
pass
class ParseError(ValueError):
pass
def get_id3_tag(f, k):
if k == "pictures":
return f.tags.getall("APIC")
# First we try to grab the standard key
try:
return f.tags[k].text[0]
except KeyError:
pass
possible_attributes = [("text", True), ("url", False)]
for attr, select_first in possible_attributes:
try:
v = getattr(f.tags[k], attr)
if select_first:
v = v[0]
return v
except KeyError:
break
except AttributeError:
continue
# then we fallback on parsing non standard tags
all_tags = f.tags.getall("TXXX")
try:
@ -68,6 +88,31 @@ def clean_flac_pictures(apic):
return pictures
def clean_ogg_pictures(metadata_block_picture):
pictures = []
for b64_data in [metadata_block_picture]:
try:
data = base64.b64decode(b64_data)
except (TypeError, ValueError):
continue
try:
picture = mutagen.flac.Picture(data)
except mutagen.flac.FLACError:
continue
pictures.append(
{
"mimetype": picture.mime,
"content": picture.data,
"description": "",
"type": picture.type.real,
}
)
return pictures
def get_mp3_recording_id(f, k):
try:
return [t for t in f.tags.getall("UFID") if "musicbrainz.org" in t.owner][
@ -77,7 +122,7 @@ def get_mp3_recording_id(f, k):
raise TagNotFound(k)
def convert_track_number(v):
def convert_position(v):
try:
return int(v)
except ValueError:
@ -103,8 +148,22 @@ class FirstUUIDField(forms.UUIDField):
def get_date(value):
parsed = pendulum.parse(str(value))
return datetime.date(parsed.year, parsed.month, parsed.day)
ADDITIONAL_FORMATS = ["%Y-%d-%m %H:%M"] # deezer date format
try:
parsed = pendulum.parse(str(value))
return datetime.date(parsed.year, parsed.month, parsed.day)
except pendulum.exceptions.ParserError:
pass
for date_format in ADDITIONAL_FORMATS:
try:
parsed = datetime.datetime.strptime(value, date_format)
except ValueError:
continue
else:
return datetime.date(parsed.year, parsed.month, parsed.day)
raise ParseError("{} cannot be parsed as a date".format(value))
def split_and_return_first(separator):
@ -127,8 +186,9 @@ CONF = {
"fields": {
"track_number": {
"field": "TRACKNUMBER",
"to_application": convert_track_number,
"to_application": convert_position,
},
"disc_number": {"field": "DISCNUMBER", "to_application": convert_position},
"title": {},
"artist": {},
"album_artist": {
@ -141,6 +201,8 @@ CONF = {
"musicbrainz_artistid": {},
"musicbrainz_albumartistid": {},
"musicbrainz_recordingid": {"field": "musicbrainz_trackid"},
"license": {},
"copyright": {},
},
},
"OggVorbis": {
@ -148,8 +210,9 @@ CONF = {
"fields": {
"track_number": {
"field": "TRACKNUMBER",
"to_application": convert_track_number,
"to_application": convert_position,
},
"disc_number": {"field": "DISCNUMBER", "to_application": convert_position},
"title": {},
"artist": {},
"album_artist": {
@ -162,6 +225,12 @@ CONF = {
"musicbrainz_artistid": {},
"musicbrainz_albumartistid": {},
"musicbrainz_recordingid": {"field": "musicbrainz_trackid"},
"license": {},
"copyright": {},
"pictures": {
"field": "metadata_block_picture",
"to_application": clean_ogg_pictures,
},
},
},
"OggTheora": {
@ -169,8 +238,9 @@ CONF = {
"fields": {
"track_number": {
"field": "TRACKNUMBER",
"to_application": convert_track_number,
"to_application": convert_position,
},
"disc_number": {"field": "DISCNUMBER", "to_application": convert_position},
"title": {},
"artist": {},
"album_artist": {"field": "albumartist"},
@ -180,13 +250,16 @@ CONF = {
"musicbrainz_artistid": {"field": "MusicBrainz Artist Id"},
"musicbrainz_albumartistid": {"field": "MusicBrainz Album Artist Id"},
"musicbrainz_recordingid": {"field": "MusicBrainz Track Id"},
"license": {},
"copyright": {},
},
},
"MP3": {
"getter": get_id3_tag,
"clean_pictures": clean_id3_pictures,
"fields": {
"track_number": {"field": "TRCK", "to_application": convert_track_number},
"track_number": {"field": "TRCK", "to_application": convert_position},
"disc_number": {"field": "TPOS", "to_application": convert_position},
"title": {"field": "TIT2"},
"artist": {"field": "TPE1"},
"album_artist": {"field": "TPE2"},
@ -200,6 +273,8 @@ CONF = {
"getter": get_mp3_recording_id,
},
"pictures": {},
"license": {"field": "WCOP"},
"copyright": {"field": "TCOP"},
},
},
"FLAC": {
@ -208,8 +283,9 @@ CONF = {
"fields": {
"track_number": {
"field": "tracknumber",
"to_application": convert_track_number,
"to_application": convert_position,
},
"disc_number": {"field": "discnumber", "to_application": convert_position},
"title": {},
"artist": {},
"album_artist": {"field": "albumartist"},
@ -221,12 +297,15 @@ CONF = {
"musicbrainz_recordingid": {"field": "musicbrainz_trackid"},
"test": {},
"pictures": {},
"license": {},
"copyright": {},
},
},
}
ALL_FIELDS = [
"track_number",
"disc_number",
"title",
"artist",
"album_artist",
@ -236,14 +315,17 @@ ALL_FIELDS = [
"musicbrainz_artistid",
"musicbrainz_albumartistid",
"musicbrainz_recordingid",
"license",
"copyright",
]
class Metadata(object):
def __init__(self, path):
self._file = mutagen.File(path)
def __init__(self, filething, kind=mutagen.File):
self._file = kind(filething)
if self._file is None:
raise ValueError("Cannot parse metadata from {}".format(path))
raise ValueError("Cannot parse metadata from {}".format(filething))
self.fallback = self.load_fallback(filething, self._file)
ft = self.get_file_type(self._file)
try:
self._conf = CONF[ft]
@ -253,7 +335,40 @@ class Metadata(object):
def get_file_type(self, f):
return f.__class__.__name__
def load_fallback(self, filething, parent):
"""
In some situations, such as Ogg Theora files tagged with MusicBrainz Picard,
part of the tags are only available in the ogg vorbis comments
"""
try:
filething.seek(0)
except AttributeError:
pass
if isinstance(parent, mutagen.oggtheora.OggTheora):
try:
return Metadata(filething, kind=mutagen.oggvorbis.OggVorbis)
except (ValueError, mutagen._util.MutagenError):
raise
pass
def get(self, key, default=NODEFAULT):
try:
return self._get_from_self(key)
except TagNotFound:
if not self.fallback:
if default != NODEFAULT:
return default
else:
raise
else:
return self.fallback.get(key, default=default)
except UnsupportedTag:
if not self.fallback:
raise
else:
return self.fallback.get(key, default=default)
def _get_from_self(self, key, default=NODEFAULT):
try:
field_conf = self._conf["fields"][key]
except KeyError:
@ -275,7 +390,7 @@ class Metadata(object):
v = field.to_python(v)
return v
def all(self):
def all(self, ignore_parse_errors=True):
"""
Return a dict containing all metadata of the file
"""
@ -286,11 +401,22 @@ class Metadata(object):
data[field] = self.get(field, None)
except (TagNotFound, forms.ValidationError):
data[field] = None
except ParseError as e:
if not ignore_parse_errors:
raise
logger.warning("Unparsable field {}: {}".format(field, str(e)))
data[field] = None
return data
def get_picture(self, picture_type="cover_front"):
ptype = getattr(mutagen.id3.PictureType, picture_type.upper())
def get_picture(self, *picture_types):
if not picture_types:
raise ValueError("You need to request at least one picture type")
ptypes = [
getattr(mutagen.id3.PictureType, picture_type.upper())
for picture_type in picture_types
]
try:
pictures = self.get("pictures")
except (UnsupportedTag, TagNotFound):
@ -298,6 +424,9 @@ class Metadata(object):
cleaner = self._conf.get("clean_pictures", lambda v: v)
pictures = cleaner(pictures)
for p in pictures:
if p["type"] == ptype:
return p
if not pictures:
return
for ptype in ptypes:
for p in pictures:
if p["type"] == ptype:
return p

53
api/funkwhale_api/music/migrations/0033_auto_20181023_1837.py Normal file
View File

@ -0,0 +1,53 @@
# Generated by Django 2.0.9 on 2018-10-23 18:37
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import funkwhale_api.music.models
class Migration(migrations.Migration):
dependencies = [
('music', '0032_track_file_to_upload'),
]
operations = [
migrations.CreateModel(
name='UploadVersion',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('mimetype', models.CharField(choices=[('audio/ogg', 'ogg'), ('audio/mpeg', 'mp3'), ('audio/x-flac', 'flac')], max_length=50)),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('accessed_date', models.DateTimeField(blank=True, null=True)),
('audio_file', models.FileField(max_length=255, upload_to=funkwhale_api.music.models.get_file_path)),
('bitrate', models.PositiveIntegerField()),
('size', models.IntegerField()),
('upload', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='music.Upload')),
],
),
migrations.AlterField(
model_name='album',
name='from_activity',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='federation.Activity'),
),
migrations.AlterField(
model_name='artist',
name='from_activity',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='federation.Activity'),
),
migrations.AlterField(
model_name='track',
name='from_activity',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='federation.Activity'),
),
migrations.AlterField(
model_name='work',
name='from_activity',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='federation.Activity'),
),
migrations.AlterUniqueTogether(
name='uploadversion',
unique_together={('upload', 'mimetype', 'bitrate')},
),
]

36
api/funkwhale_api/music/migrations/0034_auto_20181127_0325.py Normal file
View File

@ -0,0 +1,36 @@
# Generated by Django 2.0.9 on 2018-11-27 03:25
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('music', '0033_auto_20181023_1837'),
]
operations = [
migrations.CreateModel(
name='License',
fields=[
('code', models.CharField(max_length=100, primary_key=True, serialize=False)),
('url', models.URLField(max_length=500)),
('copyleft', models.BooleanField()),
('commercial', models.BooleanField()),
('attribution', models.BooleanField()),
('derivative', models.BooleanField()),
('redistribute', models.BooleanField()),
],
),
migrations.AlterField(
model_name='uploadversion',
name='mimetype',
field=models.CharField(choices=[('audio/ogg', 'ogg'), ('audio/mpeg', 'mp3'), ('audio/x-flac', 'flac'), ('audio/flac', 'flac')], max_length=50),
),
migrations.AddField(
model_name='track',
name='license',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tracks', to='music.License'),
),
]

24
api/funkwhale_api/music/migrations/0035_auto_20181203_1515.py Normal file
View File

@ -0,0 +1,24 @@
# Generated by Django 2.0.9 on 2018-12-03 15:15
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('music', '0034_auto_20181127_0325'),
]
operations = [
migrations.AddField(
model_name='track',
name='copyright',
field=models.CharField(blank=True, max_length=500, null=True),
),
migrations.AlterField(
model_name='track',
name='license',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='tracks', to='music.License'),
),
]

18
api/funkwhale_api/music/migrations/0036_track_disc_number.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.0.9 on 2018-12-04 15:10
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('music', '0035_auto_20181203_1515'),
]
operations = [
migrations.AddField(
model_name='track',
name='disc_number',
field=models.PositiveIntegerField(blank=True, null=True),
),
]

43
api/funkwhale_api/music/migrations/0037_auto_20190103_1757.py Normal file
View File

@ -0,0 +1,43 @@
# Generated by Django 2.0.9 on 2019-01-03 17:57
from django.db import migrations, models
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
('music', '0036_track_disc_number'),
]
operations = [
migrations.AlterModelOptions(
name='track',
options={'ordering': ['album', 'disc_number', 'position']},
),
migrations.AlterField(
model_name='album',
name='creation_date',
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
),
migrations.AlterField(
model_name='artist',
name='creation_date',
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
),
migrations.AlterField(
model_name='track',
name='creation_date',
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
),
migrations.AlterField(
model_name='upload',
name='creation_date',
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
),
migrations.AlterField(
model_name='work',
name='creation_date',
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
),
]

184
api/funkwhale_api/music/models.py
View File

@ -11,7 +11,7 @@ from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.core.files.base import ContentFile
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models
from django.db import models, transaction
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.urls import reverse
@ -29,7 +29,7 @@ from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import utils as federation_utils
from . import importers, metadata, utils
logger = logging.getLogger(__file__)
logger = logging.getLogger(__name__)
def empty_dict():
@ -44,7 +44,7 @@ class APIModelMixin(models.Model):
"federation.Activity", null=True, blank=True, on_delete=models.SET_NULL
)
api_includes = []
creation_date = models.DateTimeField(default=timezone.now)
creation_date = models.DateTimeField(default=timezone.now, db_index=True)
import_hooks = []
class Meta:
@ -113,6 +113,33 @@ class APIModelMixin(models.Model):
return super().save(**kwargs)
class License(models.Model):
code = models.CharField(primary_key=True, max_length=100)
url = models.URLField(max_length=500)
# if true, license is a copyleft license, meaning that derivative
# work must be shared under the same license
copyleft = models.BooleanField()
# if true, commercial use of the work is allowed
commercial = models.BooleanField()
# if true, attribution to the original author is required when reusing
# the work
attribution = models.BooleanField()
# if true, derivative work are allowed
derivative = models.BooleanField()
# if true, redistribution of the wor is allowed
redistribute = models.BooleanField()
@property
def conf(self):
from . import licenses
for row in licenses.LICENSES:
if self.code == row["code"]:
return row
logger.warning("%s do not match any registered license", self.code)
class ArtistQuerySet(models.QuerySet):
def with_albums_count(self):
return self.annotate(_albums_count=models.Count("albums"))
@ -124,8 +151,8 @@ class ArtistQuerySet(models.QuerySet):
def annotate_playable_by_actor(self, actor):
tracks = (
Track.objects.playable_by(actor)
.filter(artist=models.OuterRef("id"))
Upload.objects.playable_by(actor)
.filter(track__artist=models.OuterRef("id"))
.order_by("id")
.values("id")[:1]
)
@ -134,10 +161,11 @@ class ArtistQuerySet(models.QuerySet):
def playable_by(self, actor, include=True):
tracks = Track.objects.playable_by(actor, include)
matches = self.filter(tracks__in=tracks).values_list("pk")
if include:
return self.filter(tracks__in=tracks)
return self.filter(pk__in=matches)
else:
return self.exclude(tracks__in=tracks)
return self.exclude(pk__in=matches)
class Artist(APIModelMixin):
@ -192,8 +220,8 @@ class AlbumQuerySet(models.QuerySet):
def annotate_playable_by_actor(self, actor):
tracks = (
Track.objects.playable_by(actor)
.filter(album=models.OuterRef("id"))
Upload.objects.playable_by(actor)
.filter(track__album=models.OuterRef("id"))
.order_by("id")
.values("id")[:1]
)
@ -202,10 +230,15 @@ class AlbumQuerySet(models.QuerySet):
def playable_by(self, actor, include=True):
tracks = Track.objects.playable_by(actor, include)
matches = self.filter(tracks__in=tracks).values_list("pk")
if include:
return self.filter(tracks__in=tracks)
return self.filter(pk__in=matches)
else:
return self.exclude(tracks__in=tracks)
return self.exclude(pk__in=matches)
def with_prefetched_tracks_and_playable_uploads(self, actor):
tracks = Track.objects.with_playable_uploads(actor)
return self.prefetch_related(models.Prefetch("tracks", queryset=tracks))
class Album(APIModelMixin):
@ -398,24 +431,23 @@ class TrackQuerySet(models.QuerySet):
def playable_by(self, actor, include=True):
files = Upload.objects.playable_by(actor, include)
matches = self.filter(uploads__in=files).values_list("pk")
if include:
return self.filter(uploads__in=files)
return self.filter(pk__in=matches)
else:
return self.exclude(uploads__in=files)
return self.exclude(pk__in=matches)
def annotate_duration(self):
first_upload = Upload.objects.filter(track=models.OuterRef("pk")).order_by("pk")
return self.annotate(
duration=models.Subquery(first_upload.values("duration")[:1])
def with_playable_uploads(self, actor):
uploads = Upload.objects.playable_by(actor).select_related("track")
return self.prefetch_related(
models.Prefetch("uploads", queryset=uploads, to_attr="playable_uploads")
)
def annotate_file_data(self):
first_upload = Upload.objects.filter(track=models.OuterRef("pk")).order_by("pk")
return self.annotate(
bitrate=models.Subquery(first_upload.values("bitrate")[:1]),
size=models.Subquery(first_upload.values("size")[:1]),
mimetype=models.Subquery(first_upload.values("mimetype")[:1]),
)
def order_for_album(self):
"""
Order by disc number then position
"""
return self.order_by("disc_number", "position", "title")
def get_artist(release_list):
@ -427,6 +459,7 @@ def get_artist(release_list):
class Track(APIModelMixin):
title = models.CharField(max_length=255)
artist = models.ForeignKey(Artist, related_name="tracks", on_delete=models.CASCADE)
disc_number = models.PositiveIntegerField(null=True, blank=True)
position = models.PositiveIntegerField(null=True, blank=True)
album = models.ForeignKey(
Album, related_name="tracks", null=True, blank=True, on_delete=models.CASCADE
@ -434,6 +467,14 @@ class Track(APIModelMixin):
work = models.ForeignKey(
Work, related_name="tracks", null=True, blank=True, on_delete=models.CASCADE
)
license = models.ForeignKey(
License,
null=True,
blank=True,
on_delete=models.DO_NOTHING,
related_name="tracks",
)
copyright = models.CharField(max_length=500, null=True, blank=True)
federation_namespace = "tracks"
musicbrainz_model = "recording"
api = musicbrainz.api.recordings
@ -454,7 +495,7 @@ class Track(APIModelMixin):
tags = TaggableManager(blank=True)
class Meta:
ordering = ["album", "position"]
ordering = ["album", "disc_number", "position"]
def __str__(self):
return self.title
@ -551,6 +592,17 @@ class Track(APIModelMixin):
def listen_url(self):
return reverse("api:v1:listen-detail", kwargs={"uuid": self.uuid})
@property
def local_license(self):
"""
Since license primary keys are strings, and we can get the data
from our hardcoded licenses.LICENSES list, there is no need
for extra SQL joins / queries.
"""
from . import licenses
return licenses.LICENSES_BY_ID.get(self.license_id)
class UploadQuerySet(models.QuerySet):
def playable_by(self, actor, include=True):
@ -566,6 +618,9 @@ class UploadQuerySet(models.QuerySet):
def for_federation(self):
return self.filter(import_status="finished", mimetype__startswith="audio/")
def with_file(self):
return self.exclude(audio_file=None).exclude(audio_file="")
TRACK_FILE_IMPORT_STATUS_CHOICES = (
("pending", "Pending"),
@ -576,6 +631,9 @@ TRACK_FILE_IMPORT_STATUS_CHOICES = (
def get_file_path(instance, filename):
if isinstance(instance, UploadVersion):
return common_utils.ChunkedPath("transcoded")(instance, filename)
if instance.library.actor.get_user():
return common_utils.ChunkedPath("tracks")(instance, filename)
else:
@ -600,7 +658,7 @@ class Upload(models.Model):
blank=True,
max_length=500,
)
creation_date = models.DateTimeField(default=timezone.now)
creation_date = models.DateTimeField(default=timezone.now, db_index=True)
modification_date = models.DateTimeField(default=timezone.now, null=True)
accessed_date = models.DateTimeField(null=True, blank=True)
duration = models.IntegerField(null=True, blank=True)
@ -687,9 +745,14 @@ class Upload(models.Model):
@property
def extension(self):
if not self.audio_file:
return
return os.path.splitext(self.audio_file.name)[-1].replace(".", "", 1)
try:
return utils.MIMETYPE_TO_EXTENSION[self.mimetype]
except KeyError:
pass
if self.audio_file:
return os.path.splitext(self.audio_file.name)[-1].replace(".", "", 1)
if self.in_place_path:
return os.path.splitext(self.in_place_path)[-1].replace(".", "", 1)
def get_file_size(self):
if self.audio_file:
@ -739,6 +802,67 @@ class Upload(models.Model):
def listen_url(self):
return self.track.listen_url + "?upload={}".format(self.uuid)
def get_transcoded_version(self, format):
mimetype = utils.EXTENSION_TO_MIMETYPE[format]
existing_versions = list(self.versions.filter(mimetype=mimetype))
if existing_versions:
# we found an existing version, no need to transcode again
return existing_versions[0]
return self.create_transcoded_version(mimetype, format)
@transaction.atomic
def create_transcoded_version(self, mimetype, format):
# we create the version with an empty file, then
# we'll write to it
f = ContentFile(b"")
version = self.versions.create(
mimetype=mimetype, bitrate=self.bitrate or 128000, size=0
)
# we keep the same name, but we update the extension
new_name = os.path.splitext(os.path.basename(self.audio_file.name))[
0
] + ".{}".format(format)
version.audio_file.save(new_name, f)
utils.transcode_file(
input=self.audio_file,
output=version.audio_file,
input_format=utils.MIMETYPE_TO_EXTENSION[self.mimetype],
output_format=utils.MIMETYPE_TO_EXTENSION[mimetype],
)
version.size = version.audio_file.size
version.save(update_fields=["size"])
return version
@property
def in_place_path(self):
if not self.source or not self.source.startswith("file://"):
return
return self.source.lstrip("file://")
MIMETYPE_CHOICES = [(mt, ext) for ext, mt in utils.AUDIO_EXTENSIONS_AND_MIMETYPE]
class UploadVersion(models.Model):
upload = models.ForeignKey(
Upload, related_name="versions", on_delete=models.CASCADE
)
mimetype = models.CharField(max_length=50, choices=MIMETYPE_CHOICES)
creation_date = models.DateTimeField(default=timezone.now)
accessed_date = models.DateTimeField(null=True, blank=True)
audio_file = models.FileField(upload_to=get_file_path, max_length=255)
bitrate = models.PositiveIntegerField()
size = models.IntegerField()
class Meta:
unique_together = ("upload", "mimetype", "bitrate")
@property
def filename(self):
return self.upload.filename
IMPORT_STATUS_CHOICES = (
("pending", "Pending"),
@ -983,7 +1107,7 @@ def update_request_status(sender, instance, created, **kwargs):
@receiver(models.signals.post_save, sender=Album)
def warm_album_covers(sender, instance, **kwargs):
if not instance.cover:
if not instance.cover or not settings.CREATE_IMAGE_THUMBNAILS:
return
album_covers_warmer = VersatileImageFieldWarmer(
instance_or_queryset=instance, rendition_key_set="square", image_attr="cover"

208
api/funkwhale_api/music/serializers.py
View File

@ -1,4 +1,8 @@
import urllib.parse
from django.db import transaction
from django import urls
from django.conf import settings
from rest_framework import serializers
from taggit.models import Tag
from versatileimagefield.serializers import VersatileImageFieldSerializer
@ -7,6 +11,7 @@ from funkwhale_api.activity import serializers as activity_serializers
from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import routes
from funkwhale_api.federation import utils as federation_utils
from . import filters, models, tasks
@ -14,6 +19,21 @@ from . import filters, models, tasks
cover_field = VersatileImageFieldSerializer(allow_null=True, sizes="square")
class LicenseSerializer(serializers.Serializer):
id = serializers.SerializerMethodField()
url = serializers.URLField()
code = serializers.CharField()
name = serializers.CharField()
redistribute = serializers.BooleanField()
derivative = serializers.BooleanField()
commercial = serializers.BooleanField()
attribution = serializers.BooleanField()
copyleft = serializers.BooleanField()
def get_id(self, obj):
return obj["identifiers"][0]
class ArtistAlbumSerializer(serializers.ModelSerializer):
tracks_count = serializers.SerializerMethodField()
cover = cover_field
@ -59,7 +79,7 @@ class ArtistSimpleSerializer(serializers.ModelSerializer):
class AlbumTrackSerializer(serializers.ModelSerializer):
artist = ArtistSimpleSerializer(read_only=True)
is_playable = serializers.SerializerMethodField()
uploads = serializers.SerializerMethodField()
listen_url = serializers.SerializerMethodField()
duration = serializers.SerializerMethodField()
@ -73,16 +93,17 @@ class AlbumTrackSerializer(serializers.ModelSerializer):
"artist",
"creation_date",
"position",
"is_playable",
"disc_number",
"uploads",
"listen_url",
"duration",
"copyright",
"license",
)
def get_is_playable(self, obj):
try:
return bool(obj.is_playable_by_actor)
except AttributeError:
return None
def get_uploads(self, obj):
uploads = getattr(obj, "playable_uploads", [])
return TrackUploadSerializer(uploads, many=True).data
def get_listen_url(self, obj):
return obj.listen_url
@ -115,15 +136,14 @@ class AlbumSerializer(serializers.ModelSerializer):
)
def get_tracks(self, o):
ordered_tracks = sorted(
o.tracks.all(),
key=lambda v: (v.position, v.title) if v.position else (99999, v.title),
)
ordered_tracks = o.tracks.all()
return AlbumTrackSerializer(ordered_tracks, many=True).data
def get_is_playable(self, obj):
try:
return any([bool(t.is_playable_by_actor) for t in obj.tracks.all()])
return any(
[bool(getattr(t, "playable_uploads", [])) for t in obj.tracks.all()]
)
except AttributeError:
return None
@ -145,16 +165,26 @@ class TrackAlbumSerializer(serializers.ModelSerializer):
)
class TrackUploadSerializer(serializers.ModelSerializer):
class Meta:
model = models.Upload
fields = (
"uuid",
"listen_url",
"size",
"duration",
"bitrate",
"mimetype",
"extension",
)
class TrackSerializer(serializers.ModelSerializer):
artist = ArtistSimpleSerializer(read_only=True)
album = TrackAlbumSerializer(read_only=True)
lyrics = serializers.SerializerMethodField()
is_playable = serializers.SerializerMethodField()
uploads = serializers.SerializerMethodField()
listen_url = serializers.SerializerMethodField()
duration = serializers.SerializerMethodField()
bitrate = serializers.SerializerMethodField()
size = serializers.SerializerMethodField()
mimetype = serializers.SerializerMethodField()
class Meta:
model = models.Track
@ -166,13 +196,12 @@ class TrackSerializer(serializers.ModelSerializer):
"artist",
"creation_date",
"position",
"disc_number",
"lyrics",
"is_playable",
"uploads",
"listen_url",
"duration",
"bitrate",
"size",
"mimetype",
"copyright",
"license",
)
def get_lyrics(self, obj):
@ -181,37 +210,12 @@ class TrackSerializer(serializers.ModelSerializer):
def get_listen_url(self, obj):
return obj.listen_url
def get_is_playable(self, obj):
try:
return bool(obj.is_playable_by_actor)
except AttributeError:
return None
def get_duration(self, obj):
try:
return obj.duration
except AttributeError:
return None
def get_bitrate(self, obj):
try:
return obj.bitrate
except AttributeError:
return None
def get_size(self, obj):
try:
return obj.size
except AttributeError:
return None
def get_mimetype(self, obj):
try:
return obj.mimetype
except AttributeError:
return None
def get_uploads(self, obj):
uploads = getattr(obj, "playable_uploads", [])
return TrackUploadSerializer(uploads, many=True).data
@common_serializers.track_fields_for_update("name", "description", "privacy_level")
class LibraryForOwnerSerializer(serializers.ModelSerializer):
uploads_count = serializers.SerializerMethodField()
size = serializers.SerializerMethodField()
@ -236,6 +240,11 @@ class LibraryForOwnerSerializer(serializers.ModelSerializer):
def get_size(self, o):
return getattr(o, "_size", 0)
def on_updated_fields(self, obj, before, after):
routes.outbox.dispatch(
{"type": "Update", "object": {"type": "Library"}}, context={"library": obj}
)
class UploadSerializer(serializers.ModelSerializer):
track = TrackSerializer(required=False, allow_null=True)
@ -376,3 +385,100 @@ class TrackActivitySerializer(activity_serializers.ModelSerializer):
def get_type(self, obj):
return "Audio"
class OembedSerializer(serializers.Serializer):
format = serializers.ChoiceField(choices=["json"])
url = serializers.URLField()
maxheight = serializers.IntegerField(required=False)
maxwidth = serializers.IntegerField(required=False)
def validate(self, validated_data):
try:
match = common_utils.spa_resolve(
urllib.parse.urlparse(validated_data["url"]).path
)
except urls.exceptions.Resolver404:
raise serializers.ValidationError(
"Invalid URL {}".format(validated_data["url"])
)
data = {
"version": "1.0",
"type": "rich",
"provider_name": settings.APP_NAME,
"provider_url": settings.FUNKWHALE_URL,
"height": validated_data.get("maxheight") or 400,
"width": validated_data.get("maxwidth") or 600,
}
embed_id = None
embed_type = None
if match.url_name == "library_track":
qs = models.Track.objects.select_related("artist", "album__artist").filter(
pk=int(match.kwargs["pk"])
)
try:
track = qs.get()
except models.Track.DoesNotExist:
raise serializers.ValidationError(
"No track matching id {}".format(match.kwargs["pk"])
)
embed_type = "track"
embed_id = track.pk
data["title"] = "{} by {}".format(track.title, track.artist.name)
if track.album.cover:
data["thumbnail_url"] = federation_utils.full_url(
track.album.cover.crop["400x400"].url
)
data["thumbnail_width"] = 400
data["thumbnail_height"] = 400
data["description"] = track.full_name
data["author_name"] = track.artist.name
data["height"] = 150
data["author_url"] = federation_utils.full_url(
common_utils.spa_reverse(
"library_artist", kwargs={"pk": track.artist.pk}
)
)
elif match.url_name == "library_album":
qs = models.Album.objects.select_related("artist").filter(
pk=int(match.kwargs["pk"])
)
try:
album = qs.get()
except models.Album.DoesNotExist:
raise serializers.ValidationError(
"No album matching id {}".format(match.kwargs["pk"])
)
embed_type = "album"
embed_id = album.pk
if album.cover:
data["thumbnail_url"] = federation_utils.full_url(
album.cover.crop["400x400"].url
)
data["thumbnail_width"] = 400
data["thumbnail_height"] = 400
data["title"] = "{} by {}".format(album.title, album.artist.name)
data["description"] = "{} by {}".format(album.title, album.artist.name)
data["author_name"] = album.artist.name
data["height"] = 400
data["author_url"] = federation_utils.full_url(
common_utils.spa_reverse(
"library_artist", kwargs={"pk": album.artist.pk}
)
)
else:
raise serializers.ValidationError(
"Unsupported url: {}".format(validated_data["url"])
)
data[
"html"
] = '<iframe width="{}" height="{}" scrolling="no" frameborder="no" src="{}"></iframe>'.format(
data["width"],
data["height"],
settings.FUNKWHALE_EMBED_URL
+ "?type={}&id={}".format(embed_type, embed_id),
)
return data
def create(self, data):
return data

168
api/funkwhale_api/music/spa_views.py Normal file
View File

@ -0,0 +1,168 @@
import urllib.parse
from django.conf import settings
from django.urls import reverse
from funkwhale_api.common import utils
from . import models
def library_track(request, pk):
queryset = models.Track.objects.filter(pk=pk).select_related("album", "artist")
try:
obj = queryset.get()
except models.Track.DoesNotExist:
return []
track_url = utils.join_url(
settings.FUNKWHALE_URL,
utils.spa_reverse("library_track", kwargs={"pk": obj.pk}),
)
metas = [
{"tag": "meta", "property": "og:url", "content": track_url},
{"tag": "meta", "property": "og:title", "content": obj.title},
{"tag": "meta", "property": "og:type", "content": "music.song"},
{"tag": "meta", "property": "music:album:disc", "content": obj.disc_number},
{"tag": "meta", "property": "music:album:track", "content": obj.position},
{
"tag": "meta",
"property": "music:musician",
"content": utils.join_url(
settings.FUNKWHALE_URL,
utils.spa_reverse("library_artist", kwargs={"pk": obj.artist.pk}),
),
},
{
"tag": "meta",
"property": "music:album",
"content": utils.join_url(
settings.FUNKWHALE_URL,
utils.spa_reverse("library_album", kwargs={"pk": obj.album.pk}),
),
},
]
if obj.album.cover:
metas.append(
{
"tag": "meta",
"property": "og:image",
"content": utils.join_url(
settings.FUNKWHALE_URL, obj.album.cover.crop["400x400"].url
),
}
)
if obj.uploads.playable_by(None).exists():
metas.append(
{
"tag": "meta",
"property": "og:audio",
"content": utils.join_url(settings.FUNKWHALE_URL, obj.listen_url),
}
)
metas.append(
{
"tag": "link",
"rel": "alternate",
"type": "application/json+oembed",
"href": (
utils.join_url(settings.FUNKWHALE_URL, reverse("api:v1:oembed"))
+ "?format=json&url={}".format(urllib.parse.quote_plus(track_url))
),
}
)
return metas
def library_album(request, pk):
queryset = models.Album.objects.filter(pk=pk).select_related("artist")
try:
obj = queryset.get()
except models.Album.DoesNotExist:
return []
album_url = utils.join_url(
settings.FUNKWHALE_URL,
utils.spa_reverse("library_album", kwargs={"pk": obj.pk}),
)
metas = [
{"tag": "meta", "property": "og:url", "content": album_url},
{"tag": "meta", "property": "og:title", "content": obj.title},
{"tag": "meta", "property": "og:type", "content": "music.album"},
{
"tag": "meta",
"property": "music:musician",
"content": utils.join_url(
settings.FUNKWHALE_URL,
utils.spa_reverse("library_artist", kwargs={"pk": obj.artist.pk}),
),
},
]
if obj.release_date:
metas.append(
{
"tag": "meta",
"property": "music:release_date",
"content": str(obj.release_date),
}
)
if obj.cover:
metas.append(
{
"tag": "meta",
"property": "og:image",
"content": utils.join_url(
settings.FUNKWHALE_URL, obj.cover.crop["400x400"].url
),
}
)
if models.Upload.objects.filter(track__album=obj).playable_by(None).exists():
metas.append(
{
"tag": "link",
"rel": "alternate",
"type": "application/json+oembed",
"href": (
utils.join_url(settings.FUNKWHALE_URL, reverse("api:v1:oembed"))
+ "?format=json&url={}".format(urllib.parse.quote_plus(album_url))
),
}
)
return metas
def library_artist(request, pk):
queryset = models.Artist.objects.filter(pk=pk)
try:
obj = queryset.get()
except models.Artist.DoesNotExist:
return []
artist_url = utils.join_url(
settings.FUNKWHALE_URL,
utils.spa_reverse("library_artist", kwargs={"pk": obj.pk}),
)
# we use latest album's cover as artist image
latest_album = (
obj.albums.exclude(cover="").exclude(cover=None).order_by("release_date").last()
)
metas = [
{"tag": "meta", "property": "og:url", "content": artist_url},
{"tag": "meta", "property": "og:title", "content": obj.name},
{"tag": "meta", "property": "og:type", "content": "profile"},
]
if latest_album and latest_album.cover:
metas.append(
{
"tag": "meta",
"property": "og:image",
"content": utils.join_url(
settings.FUNKWHALE_URL, latest_album.cover.crop["400x400"].url
),
}
)
return metas

28
api/funkwhale_api/music/tasks.py
View File

@ -1,4 +1,5 @@
import collections
import datetime
import logging
import os
@ -10,11 +11,12 @@ from django.dispatch import receiver
from musicbrainzngs import ResponseError
from requests.exceptions import RequestException
from funkwhale_api.common import channels
from funkwhale_api.common import channels, preferences
from funkwhale_api.federation import routes
from funkwhale_api.federation import library as lb
from funkwhale_api.taskapp import celery
from . import licenses
from . import lyrics as lyrics_utils
from . import models
from . import metadata
@ -189,7 +191,7 @@ def process_upload(upload):
final_metadata = collections.ChainMap(
additional_data, import_metadata, file_metadata
)
additional_data["cover_data"] = m.get_picture("cover_front")
additional_data["cover_data"] = m.get_picture("cover_front", "other")
additional_data["upload_source"] = upload.source
track = get_track_from_import_metadata(final_metadata)
except UploadImportError as e:
@ -272,9 +274,12 @@ def federation_audio_track_to_metadata(payload):
"title": payload["name"],
"album": payload["album"]["name"],
"track_number": payload["position"],
"disc_number": payload.get("disc"),
"artist": payload["artists"][0]["name"],
"album_artist": payload["album"]["artists"][0]["name"],
"date": payload["album"].get("released"),
"license": payload.get("license"),
"copyright": payload.get("copyright"),
# musicbrainz
"musicbrainz_recordingid": str(musicbrainz_recordingid)
if musicbrainz_recordingid
@ -493,8 +498,11 @@ def get_track_from_import_metadata(data):
"mbid": track_mbid,
"artist": artist,
"position": track_number,
"disc_number": data.get("disc_number"),
"fid": track_fid,
"from_activity_id": from_activity_id,
"license": licenses.match(data.get("license"), data.get("copyright")),
"copyright": data.get("copyright"),
}
if data.get("fdate"):
defaults["creation_date"] = data.get("fdate")
@ -526,3 +534,19 @@ def broadcast_import_status_update_to_owner(old_status, new_status, upload, **kw
},
},
)
@celery.app.task(name="music.clean_transcoding_cache")
def clean_transcoding_cache():
delay = preferences.get("music__transcoding_cache_duration")
if delay < 1:
return # cache clearing disabled
limit = timezone.now() - datetime.timedelta(minutes=delay)
candidates = (
models.UploadVersion.objects.filter(
(Q(accessed_date__lt=limit) | Q(accessed_date=None))
)
.only("audio_file", "id")
.order_by("id")
)
return candidates.delete()

9
api/funkwhale_api/music/utils.py
View File

@ -2,6 +2,7 @@ import mimetypes
import magic
import mutagen
import pydub
from funkwhale_api.common.search import normalize_query, get_query # noqa
@ -32,6 +33,7 @@ AUDIO_EXTENSIONS_AND_MIMETYPE = [
("ogg", "audio/ogg"),
("mp3", "audio/mpeg"),
("flac", "audio/x-flac"),
("flac", "audio/flac"),
]
EXTENSION_TO_MIMETYPE = {ext: mt for ext, mt in AUDIO_EXTENSIONS_AND_MIMETYPE}
@ -68,3 +70,10 @@ def get_actor_from_request(request):
actor = request.user.actor
return actor
def transcode_file(input, output, input_format, output_format, **kwargs):
with input.open("rb"):
audio = pydub.AudioSegment.from_file(input, format=input_format)
with output.open("wb"):
return audio.export(output, format=output_format, **kwargs)

167
api/funkwhale_api/music/views.py
View File

@ -11,41 +11,42 @@ from rest_framework import mixins
from rest_framework import permissions
from rest_framework import settings as rest_settings
from rest_framework import views, viewsets
from rest_framework.decorators import detail_route, list_route
from rest_framework.decorators import action
from rest_framework.response import Response
from taggit.models import Tag
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common import permissions as common_permissions
from funkwhale_api.common import preferences
from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation.authentication import SignatureAuthentication
from funkwhale_api.federation import api_serializers as federation_api_serializers
from funkwhale_api.federation import routes
from . import filters, models, serializers, tasks, utils
from . import filters, licenses, models, serializers, tasks, utils
logger = logging.getLogger(__name__)
def get_libraries(filter_uploads):
def view(self, request, *args, **kwargs):
def libraries(self, request, *args, **kwargs):
obj = self.get_object()
actor = utils.get_actor_from_request(request)
uploads = models.Upload.objects.all()
uploads = filter_uploads(obj, uploads)
uploads = uploads.playable_by(actor)
libraries = models.Library.objects.filter(
qs = models.Library.objects.filter(
pk__in=uploads.values_list("library", flat=True)
)
libraries = libraries.select_related("actor")
page = self.paginate_queryset(libraries)
).annotate(_uploads_count=Count("uploads"))
qs = qs.select_related("actor")
page = self.paginate_queryset(qs)
if page is not None:
serializer = federation_api_serializers.LibrarySerializer(page, many=True)
return self.get_paginated_response(serializer.data)
serializer = federation_api_serializers.LibrarySerializer(libraries, many=True)
serializer = federation_api_serializers.LibrarySerializer(qs, many=True)
return Response(serializer.data)
return view
return libraries
class TagViewSetMixin(object):
@ -61,7 +62,7 @@ class ArtistViewSet(viewsets.ReadOnlyModelViewSet):
queryset = models.Artist.objects.all()
serializer_class = serializers.ArtistWithAlbumsSerializer
permission_classes = [common_permissions.ConditionalAuthentication]
filter_class = filters.ArtistFilter
filterset_class = filters.ArtistFilter
ordering_fields = ("id", "name", "creation_date")
def get_queryset(self):
@ -70,9 +71,9 @@ class ArtistViewSet(viewsets.ReadOnlyModelViewSet):
albums = albums.annotate_playable_by_actor(
utils.get_actor_from_request(self.request)
)
return queryset.prefetch_related(Prefetch("albums", queryset=albums)).distinct()
return queryset.prefetch_related(Prefetch("albums", queryset=albums))
libraries = detail_route(methods=["get"])(
libraries = action(methods=["get"], detail=True)(
get_libraries(
filter_uploads=lambda o, uploads: uploads.filter(
Q(track__artist=o) | Q(track__album__artist=o)
@ -88,25 +89,19 @@ class AlbumViewSet(viewsets.ReadOnlyModelViewSet):
serializer_class = serializers.AlbumSerializer
permission_classes = [common_permissions.ConditionalAuthentication]
ordering_fields = ("creation_date", "release_date", "title")
filter_class = filters.AlbumFilter
filterset_class = filters.AlbumFilter
def get_queryset(self):
queryset = super().get_queryset()
tracks = models.Track.objects.annotate_playable_by_actor(
utils.get_actor_from_request(self.request)
).select_related("artist")
if (
hasattr(self, "kwargs")
and self.kwargs
and self.request.method.lower() == "get"
):
# we are detailing a single album, so we can add the overhead
# to fetch additional data
tracks = tracks.annotate_duration()
tracks = (
models.Track.objects.select_related("artist")
.with_playable_uploads(utils.get_actor_from_request(self.request))
.order_for_album()
)
qs = queryset.prefetch_related(Prefetch("tracks", queryset=tracks))
return qs.distinct()
return qs
libraries = detail_route(methods=["get"])(
libraries = action(methods=["get"], detail=True)(
get_libraries(filter_uploads=lambda o, uploads: uploads.filter(track__album=o))
)
@ -149,7 +144,9 @@ class LibraryViewSet(
)
instance.delete()
@detail_route(methods=["get"])
follows = action
@action(methods=["get"], detail=True)
@transaction.non_atomic_requests
def follows(self, request, *args, **kwargs):
library = self.get_object()
@ -177,7 +174,7 @@ class TrackViewSet(TagViewSetMixin, viewsets.ReadOnlyModelViewSet):
queryset = models.Track.objects.all().for_nested_serialization()
serializer_class = serializers.TrackSerializer
permission_classes = [common_permissions.ConditionalAuthentication]
filter_class = filters.TrackFilter
filterset_class = filters.TrackFilter
ordering_fields = (
"creation_date",
"title",
@ -193,20 +190,12 @@ class TrackViewSet(TagViewSetMixin, viewsets.ReadOnlyModelViewSet):
if user.is_authenticated and filter_favorites == "true":
queryset = queryset.filter(track_favorites__user=user)
queryset = queryset.annotate_playable_by_actor(
queryset = queryset.with_playable_uploads(
utils.get_actor_from_request(self.request)
).annotate_duration()
if (
hasattr(self, "kwargs")
and self.kwargs
and self.request.method.lower() == "get"
):
# we are detailing a single track, so we can add the overhead
# to fetch additional data
queryset = queryset.annotate_file_data()
return queryset.distinct()
)
return queryset
@detail_route(methods=["get"])
@action(methods=["get"], detail=True)
@transaction.non_atomic_requests
def lyrics(self, request, *args, **kwargs):
try:
@ -231,7 +220,7 @@ class TrackViewSet(TagViewSetMixin, viewsets.ReadOnlyModelViewSet):
serializer = serializers.LyricsSerializer(lyrics)
return Response(serializer.data)
libraries = detail_route(methods=["get"])(
libraries = action(methods=["get"], detail=True)(
get_libraries(filter_uploads=lambda o, uploads: uploads.filter(track=o))
)
@ -267,12 +256,31 @@ def get_file_path(audio_file):
return path.encode("utf-8")
def handle_serve(upload, user):
def should_transcode(upload, format):
if not preferences.get("music__transcoding_enabled"):
return False
if format is None:
return False
if format not in utils.EXTENSION_TO_MIMETYPE:
# format should match supported formats
return False
if upload.mimetype is None:
# upload should have a mimetype, otherwise we cannot transcode
return False
if upload.mimetype == utils.EXTENSION_TO_MIMETYPE[format]:
# requested format sould be different than upload mimetype, otherwise
# there is no need to transcode
return False
return True
def handle_serve(upload, user, format=None):
f = upload
# we update the accessed_date
f.accessed_date = timezone.now()
f.save(update_fields=["accessed_date"])
now = timezone.now()
upload.accessed_date = now
upload.save(update_fields=["accessed_date"])
f = upload
if f.audio_file:
file_path = get_file_path(f.audio_file)
@ -298,6 +306,14 @@ def handle_serve(upload, user):
elif f.source and f.source.startswith("file://"):
file_path = get_file_path(f.source.replace("file://", "", 1))
mt = f.mimetype
if should_transcode(f, format):
transcoded_version = upload.get_transcoded_version(format)
transcoded_version.accessed_date = now
transcoded_version.save(update_fields=["accessed_date"])
f = transcoded_version
file_path = get_file_path(f.audio_file)
mt = f.mimetype
if mt:
response = Response(content_type=mt)
else:
@ -337,7 +353,8 @@ class ListenViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
if not upload:
return Response(status=404)
return handle_serve(upload, user=request.user)
format = request.GET.get("to")
return handle_serve(upload, user=request.user, format=format)
class UploadViewSet(
@ -360,7 +377,7 @@ class UploadViewSet(
]
owner_field = "library.actor.user"
owner_checks = ["read", "write"]
filter_class = filters.UploadFilter
filterset_class = filters.UploadFilter
ordering_fields = (
"creation_date",
"import_date",
@ -373,7 +390,7 @@ class UploadViewSet(
qs = super().get_queryset()
return qs.filter(library__actor=self.request.user.actor)
@list_route(methods=["post"])
@action(methods=["post"], detail=False)
def action(self, request, *args, **kwargs):
queryset = self.get_queryset()
serializer = serializers.UploadActionSerializer(request.data, queryset=queryset)
@ -433,28 +450,29 @@ class Search(views.APIView):
"artist__name__unaccent",
]
query_obj = utils.get_query(query, search_fields)
return (
qs = (
models.Track.objects.all()
.filter(query_obj)
.select_related("artist", "album__artist")
)[: self.max_results]
)
return common_utils.order_for_search(qs, "title")[: self.max_results]
def get_albums(self, query):
search_fields = ["mbid", "title__unaccent", "artist__name__unaccent"]
query_obj = utils.get_query(query, search_fields)
return (
qs = (
models.Album.objects.all()
.filter(query_obj)
.select_related()
.prefetch_related("tracks")
)[: self.max_results]
.prefetch_related("tracks__artist")
)
return common_utils.order_for_search(qs, "title")[: self.max_results]
def get_artists(self, query):
search_fields = ["mbid", "name__unaccent"]
query_obj = utils.get_query(query, search_fields)
return (models.Artist.objects.all().filter(query_obj).with_albums())[
: self.max_results
]
qs = models.Artist.objects.all().filter(query_obj).with_albums()
return common_utils.order_for_search(qs, "name")[: self.max_results]
def get_tags(self, query):
search_fields = ["slug", "name__unaccent"]
@ -468,3 +486,38 @@ class Search(views.APIView):
)
return qs.filter(query_obj)[: self.max_results]
class LicenseViewSet(viewsets.ReadOnlyModelViewSet):
permission_classes = [common_permissions.ConditionalAuthentication]
serializer_class = serializers.LicenseSerializer
queryset = models.License.objects.all().order_by("code")
lookup_value_regex = ".*"
def get_queryset(self):
# ensure our licenses are up to date in DB
licenses.load(licenses.LICENSES)
return super().get_queryset()
def get_serializer(self, *args, **kwargs):
if len(args) == 0:
return super().get_serializer(*args, **kwargs)
# our serializer works with license dict, not License instances
# so we pass those instead
instance_or_qs = args[0]
try:
first_arg = instance_or_qs.conf
except AttributeError:
first_arg = [i.conf for i in instance_or_qs if i.conf]
return super().get_serializer(*((first_arg,) + args[1:]), **kwargs)
class OembedView(views.APIView):
permission_classes = [common_permissions.ConditionalAuthentication]
def get(self, request, *args, **kwargs):
serializer = serializers.OembedSerializer(data=request.GET)
serializer.is_valid(raise_exception=True)
embed_data = serializer.save()
return Response(embed_data)

8
api/funkwhale_api/musicbrainz/views.py
View File

@ -1,5 +1,5 @@
from rest_framework import viewsets
from rest_framework.decorators import list_route
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.views import APIView
@ -47,19 +47,19 @@ class ReleaseBrowse(APIView):
class SearchViewSet(viewsets.ViewSet):
permission_classes = [ConditionalAuthentication]
@list_route(methods=["get"])
@action(methods=["get"], detail=False)
def recordings(self, request, *args, **kwargs):
query = request.GET["query"]
results = api.recordings.search(query)
return Response(results)
@list_route(methods=["get"])
@action(methods=["get"], detail=False)
def releases(self, request, *args, **kwargs):
query = request.GET["query"]
results = api.releases.search(query)
return Response(results)
@list_route(methods=["get"])
@action(methods=["get"], detail=False)
def artists(self, request, *args, **kwargs):
query = request.GET["query"]
results = api.artists.search(query)

6
api/funkwhale_api/playlists/factories.py
View File

@ -1,12 +1,12 @@
import factory
from funkwhale_api.factories import registry
from funkwhale_api.factories import registry, NoUpdateOnCreate
from funkwhale_api.music.factories import TrackFactory
from funkwhale_api.users.factories import UserFactory
@registry.register
class PlaylistFactory(factory.django.DjangoModelFactory):
class PlaylistFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
name = factory.Faker("name")
user = factory.SubFactory(UserFactory)
@ -15,7 +15,7 @@ class PlaylistFactory(factory.django.DjangoModelFactory):
@registry.register
class PlaylistTrackFactory(factory.django.DjangoModelFactory):
class PlaylistTrackFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
playlist = factory.SubFactory(PlaylistFactory)
track = factory.SubFactory(TrackFactory)

4
api/funkwhale_api/playlists/filters.py
View File

@ -7,8 +7,8 @@ from . import models
class PlaylistFilter(filters.FilterSet):
q = filters.CharFilter(name="_", method="filter_q")
playable = filters.BooleanFilter(name="_", method="filter_playable")
q = filters.CharFilter(field_name="_", method="filter_q")
playable = filters.BooleanFilter(field_name="_", method="filter_playable")
class Meta:
model = models.Playlist

29
api/funkwhale_api/playlists/models.py
View File

@ -38,22 +38,21 @@ class PlaylistQuerySet(models.QuerySet):
)
return self.prefetch_related(plt_prefetch)
def annotate_playable_by_actor(self, actor):
plts = (
PlaylistTrack.objects.playable_by(actor)
.filter(playlist=models.OuterRef("id"))
.order_by("id")
.values("id")[:1]
def with_playable_plts(self, actor):
return self.prefetch_related(
models.Prefetch(
"playlist_tracks",
queryset=PlaylistTrack.objects.playable_by(actor),
to_attr="playable_plts",
)
)
subquery = models.Subquery(plts)
return self.annotate(is_playable_by_actor=subquery)
def playable_by(self, actor, include=True):
plts = PlaylistTrack.objects.playable_by(actor, include)
if include:
return self.filter(playlist_tracks__in=plts)
return self.filter(playlist_tracks__in=plts).distinct()
else:
return self.exclude(playlist_tracks__in=plts)
return self.exclude(playlist_tracks__in=plts).distinct()
class Playlist(models.Model):
@ -148,7 +147,7 @@ class Playlist(models.Model):
class PlaylistTrackQuerySet(models.QuerySet):
def for_nested_serialization(self, actor=None):
tracks = music_models.Track.objects.annotate_playable_by_actor(actor)
tracks = music_models.Track.objects.with_playable_uploads(actor)
tracks = tracks.select_related("artist", "album__artist")
return self.prefetch_related(
models.Prefetch("track", queryset=tracks, to_attr="_prefetched_track")
@ -156,8 +155,8 @@ class PlaylistTrackQuerySet(models.QuerySet):
def annotate_playable_by_actor(self, actor):
tracks = (
music_models.Track.objects.playable_by(actor)
.filter(pk=models.OuterRef("track"))
music_models.Upload.objects.playable_by(actor)
.filter(track__pk=models.OuterRef("track"))
.order_by("id")
.values("id")[:1]
)
@ -167,9 +166,9 @@ class PlaylistTrackQuerySet(models.QuerySet):
def playable_by(self, actor, include=True):
tracks = music_models.Track.objects.playable_by(actor, include)
if include:
return self.filter(track__pk__in=tracks)
return self.filter(track__pk__in=tracks).distinct()
else:
return self.exclude(track__pk__in=tracks)
return self.exclude(track__pk__in=tracks).distinct()
class PlaylistTrack(models.Model):

2
api/funkwhale_api/playlists/serializers.py
View File

@ -93,7 +93,7 @@ class PlaylistSerializer(serializers.ModelSerializer):
def get_is_playable(self, obj):
try:
return bool(obj.is_playable_by_actor)
return bool(obj.playable_plts)
except AttributeError:
return None

12
api/funkwhale_api/playlists/views.py
View File

@ -1,7 +1,7 @@
from django.db import transaction
from django.db.models import Count
from rest_framework import exceptions, mixins, viewsets
from rest_framework.decorators import detail_route
from rest_framework.decorators import action
from rest_framework.permissions import IsAuthenticatedOrReadOnly
from rest_framework.response import Response
@ -33,10 +33,10 @@ class PlaylistViewSet(
IsAuthenticatedOrReadOnly,
]
owner_checks = ["write"]
filter_class = filters.PlaylistFilter
filterset_class = filters.PlaylistFilter
ordering_fields = ("id", "name", "creation_date", "modification_date")
@detail_route(methods=["get"])
@action(methods=["get"], detail=True)
def tracks(self, request, *args, **kwargs):
playlist = self.get_object()
plts = playlist.playlist_tracks.all().for_nested_serialization(
@ -46,7 +46,7 @@ class PlaylistViewSet(
data = {"count": len(plts), "results": serializer.data}
return Response(data, status=200)
@detail_route(methods=["post"])
@action(methods=["post"], detail=True)
@transaction.atomic
def add(self, request, *args, **kwargs):
playlist = self.get_object()
@ -67,7 +67,7 @@ class PlaylistViewSet(
data = {"count": len(plts), "results": serializer.data}
return Response(data, status=201)
@detail_route(methods=["delete"])
@action(methods=["delete"], detail=True)
@transaction.atomic
def clear(self, request, *args, **kwargs):
playlist = self.get_object()
@ -78,7 +78,7 @@ class PlaylistViewSet(
def get_queryset(self):
return self.queryset.filter(
fields.privacy_level_query(self.request.user)
).annotate_playable_by_actor(music_utils.get_actor_from_request(self.request))
).with_playable_plts(music_utils.get_actor_from_request(self.request))
def perform_create(self, serializer):
return serializer.save(

27
api/funkwhale_api/providers/acoustid/__init__.py
View File

@ -1,27 +0,0 @@
import acoustid
from dynamic_preferences.registries import global_preferences_registry
class Client(object):
def __init__(self, api_key):
self.api_key = api_key
def match(self, file_path):
return acoustid.match(self.api_key, file_path, parse=False)
def get_best_match(self, file_path):
results = self.match(file_path=file_path)
MIN_SCORE_FOR_MATCH = 0.8
try:
rows = results["results"]
except KeyError:
return
for row in rows:
if row["score"] >= MIN_SCORE_FOR_MATCH:
return row
def get_acoustid_client():
manager = global_preferences_registry.manager()
return Client(api_key=manager["providers_acoustid__api_key"])

16
api/funkwhale_api/providers/acoustid/dynamic_preferences_registry.py
View File

@ -1,16 +0,0 @@
from django import forms
from dynamic_preferences.registries import global_preferences_registry
from dynamic_preferences.types import Section, StringPreference
acoustid = Section("providers_acoustid")
@global_preferences_registry.register
class APIKey(StringPreference):
section = acoustid
name = "api_key"
default = ""
verbose_name = "Acoustid API key"
help_text = "The API key used to query AcoustID. Get one at https://acoustid.org/new-application."
widget = forms.PasswordInput
field_kwargs = {"required": False}

8
api/funkwhale_api/radios/factories.py
View File

@ -1,11 +1,11 @@
import factory
from funkwhale_api.factories import registry
from funkwhale_api.factories import registry, NoUpdateOnCreate
from funkwhale_api.users.factories import UserFactory
@registry.register
class RadioFactory(factory.django.DjangoModelFactory):
class RadioFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
name = factory.Faker("name")
description = factory.Faker("paragraphs")
user = factory.SubFactory(UserFactory)
@ -16,7 +16,7 @@ class RadioFactory(factory.django.DjangoModelFactory):
@registry.register
class RadioSessionFactory(factory.django.DjangoModelFactory):
class RadioSessionFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
user = factory.SubFactory(UserFactory)
class Meta:
@ -24,7 +24,7 @@ class RadioSessionFactory(factory.django.DjangoModelFactory):
@registry.register(name="radios.CustomRadioSession")
class CustomRadioSessionFactory(factory.django.DjangoModelFactory):
class CustomRadioSessionFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
user = factory.SubFactory(UserFactory)
radio_type = "custom"
custom_radio = factory.SubFactory(

10
api/funkwhale_api/radios/views.py
View File

@ -1,6 +1,6 @@
from django.db.models import Q
from rest_framework import mixins, permissions, status, viewsets
from rest_framework.decorators import detail_route, list_route
from rest_framework.decorators import action
from rest_framework.response import Response
from funkwhale_api.common import permissions as common_permissions
@ -23,7 +23,7 @@ class RadioViewSet(
permissions.IsAuthenticated,
common_permissions.OwnerPermission,
]
filter_class = filtersets.RadioFilter
filterset_class = filtersets.RadioFilter
owner_field = "user"
owner_checks = ["write"]
@ -40,7 +40,7 @@ class RadioViewSet(
def perform_update(self, serializer):
return serializer.save(user=self.request.user)
@detail_route(methods=["get"])
@action(methods=["get"], detail=True)
def tracks(self, request, *args, **kwargs):
radio = self.get_object()
tracks = radio.get_candidates().for_nested_serialization()
@ -50,14 +50,14 @@ class RadioViewSet(
serializer = TrackSerializer(page, many=True)
return self.get_paginated_response(serializer.data)
@list_route(methods=["get"])
@action(methods=["get"], detail=False)
def filters(self, request, *args, **kwargs):
serializer = serializers.FilterSerializer(
filters.registry.exposed_filters, many=True
)
return Response(serializer.data)
@list_route(methods=["post"])
@action(methods=["post"], detail=False)
def validate(self, request, *args, **kwargs):
try:
f_list = request.data["filters"]

2
api/funkwhale_api/subsonic/filters.py
View File

@ -4,7 +4,7 @@ from funkwhale_api.music import models as music_models
class AlbumList2FilterSet(filters.FilterSet):
type = filters.CharFilter(name="_", method="filter_type")
type = filters.CharFilter(field_name="_", method="filter_type")
class Meta:
model = music_models.Album

34
api/funkwhale_api/subsonic/renderers.py
View File

@ -2,17 +2,30 @@ import xml.etree.ElementTree as ET
from rest_framework import renderers
import funkwhale_api
def structure_payload(data):
payload = {
"status": "ok",
"version": "1.16.0",
"type": "funkwhale",
"funkwhaleVersion": funkwhale_api.__version__,
}
payload.update(data)
if "detail" in payload:
payload["error"] = {"code": 0, "message": payload.pop("detail")}
if "error" in payload:
payload["status"] = "failed"
return payload
class SubsonicJSONRenderer(renderers.JSONRenderer):
def render(self, data, accepted_media_type=None, renderer_context=None):
if not data:
# when stream view is called, we don't have any data
return super().render(data, accepted_media_type, renderer_context)
final = {"subsonic-response": {"status": "ok", "version": "1.16.0"}}
final["subsonic-response"].update(data)
if "error" in final:
# an error was returned
final["subsonic-response"]["status"] = "failed"
final = {"subsonic-response": structure_payload(data)}
return super().render(final, accepted_media_type, renderer_context)
@ -23,15 +36,8 @@ class SubsonicXMLRenderer(renderers.JSONRenderer):
if not data:
# when stream view is called, we don't have any data
return super().render(data, accepted_media_type, renderer_context)
final = {
"xmlns": "http://subsonic.org/restapi",
"status": "ok",
"version": "1.16.0",
}
final.update(data)
if "error" in final:
# an error was returned
final["status"] = "failed"
final = structure_payload(data)
final["xmlns"] = "http://subsonic.org/restapi"
tree = dict_to_xml_tree("subsonic-response", final)
return b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(
tree, encoding="utf-8"

24
api/funkwhale_api/subsonic/serializers.py
View File

@ -226,6 +226,30 @@ def get_music_directory_data(artist):
return data
def get_folders(user):
return []
def get_user_detail_data(user):
return {
"username": user.username,
"email": user.email,
"scrobblingEnabled": "true",
"adminRole": "false",
"settingsRole": "false",
"commentRole": "false",
"podcastRole": "false",
"coverArtRole": "false",
"shareRole": "false",
"uploadRole": "true",
"downloadRole": "true",
"playlistRole": "true",
"streamRole": "true",
"jukeboxRole": "true",
"folder": [f["id"] for f in get_folders(user)],
}
class ScrobbleSerializer(serializers.Serializer):
submission = serializers.BooleanField(default=True, required=False)
id = serializers.PrimaryKeyRelatedField(

207
api/funkwhale_api/subsonic/views.py
View File

@ -1,20 +1,23 @@
import datetime
import functools
from django.conf import settings
from django.utils import timezone
from rest_framework import exceptions
from rest_framework import permissions as rest_permissions
from rest_framework import renderers, response, viewsets
from rest_framework.decorators import list_route
from rest_framework.decorators import action
from rest_framework.serializers import ValidationError
import funkwhale_api
from funkwhale_api.activity import record
from funkwhale_api.common import preferences
from funkwhale_api.common import preferences, utils as common_utils
from funkwhale_api.favorites.models import TrackFavorite
from funkwhale_api.music import models as music_models
from funkwhale_api.music import utils
from funkwhale_api.music import views as music_views
from funkwhale_api.playlists import models as playlists_models
from funkwhale_api.users import models as users_models
from . import authentication, filters, negotiation, serializers
@ -23,6 +26,7 @@ def find_object(
queryset, model_field="pk", field="id", cast=int, filter_playable=False
):
def decorator(func):
@functools.wraps(func)
def inner(self, request, *args, **kwargs):
data = request.GET or request.POST
try:
@ -55,7 +59,7 @@ def find_object(
if filter_playable:
actor = utils.get_actor_from_request(request)
qs = qs.playable_by(actor).distinct()
qs = qs.playable_by(actor)
try:
obj = qs.get(**{model_field: value})
@ -95,7 +99,10 @@ class SubsonicViewSet(viewsets.GenericViewSet):
def handle_exception(self, exc):
# subsonic API sends 200 status code with custom error
# codes in the payload
mapping = {exceptions.AuthenticationFailed: (40, "Wrong username or password.")}
mapping = {
exceptions.AuthenticationFailed: (40, "Wrong username or password."),
exceptions.NotAuthenticated: (10, "Required parameter is missing."),
}
payload = {"status": "failed"}
if exc.__class__ in mapping:
code, message = mapping[exc.__class__]
@ -105,12 +112,13 @@ class SubsonicViewSet(viewsets.GenericViewSet):
return response.Response(payload, status=200)
@list_route(methods=["get", "post"], permission_classes=[])
@action(detail=False, methods=["get", "post"], permission_classes=[])
def ping(self, request, *args, **kwargs):
data = {"status": "ok", "version": "1.16.0"}
return response.Response(data, status=200)
@list_route(
@action(
detail=False,
methods=["get", "post"],
url_name="get_license",
permissions_classes=[],
@ -121,6 +129,8 @@ class SubsonicViewSet(viewsets.GenericViewSet):
data = {
"status": "ok",
"version": "1.16.0",
"type": "funkwhale",
"funkwhaleVersion": funkwhale_api.__version__,
"license": {
"valid": "true",
"email": "valid@valid.license",
@ -129,7 +139,12 @@ class SubsonicViewSet(viewsets.GenericViewSet):
}
return response.Response(data, status=200)
@list_route(methods=["get", "post"], url_name="get_artists", url_path="getArtists")
@action(
detail=False,
methods=["get", "post"],
url_name="get_artists",
url_path="getArtists",
)
def get_artists(self, request, *args, **kwargs):
artists = music_models.Artist.objects.all().playable_by(
utils.get_actor_from_request(request)
@ -139,7 +154,12 @@ class SubsonicViewSet(viewsets.GenericViewSet):
return response.Response(payload, status=200)
@list_route(methods=["get", "post"], url_name="get_indexes", url_path="getIndexes")
@action(
detail=False,
methods=["get", "post"],
url_name="get_indexes",
url_path="getIndexes",
)
def get_indexes(self, request, *args, **kwargs):
artists = music_models.Artist.objects.all().playable_by(
utils.get_actor_from_request(request)
@ -149,7 +169,12 @@ class SubsonicViewSet(viewsets.GenericViewSet):
return response.Response(payload, status=200)
@list_route(methods=["get", "post"], url_name="get_artist", url_path="getArtist")
@action(
detail=False,
methods=["get", "post"],
url_name="get_artist",
url_path="getArtist",
)
@find_object(music_models.Artist.objects.all(), filter_playable=True)
def get_artist(self, request, *args, **kwargs):
artist = kwargs.pop("obj")
@ -158,7 +183,9 @@ class SubsonicViewSet(viewsets.GenericViewSet):
return response.Response(payload, status=200)
@list_route(methods=["get", "post"], url_name="get_song", url_path="getSong")
@action(
detail=False, methods=["get", "post"], url_name="get_song", url_path="getSong"
)
@find_object(music_models.Track.objects.all(), filter_playable=True)
def get_song(self, request, *args, **kwargs):
track = kwargs.pop("obj")
@ -167,8 +194,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
return response.Response(payload, status=200)
@list_route(
methods=["get", "post"], url_name="get_artist_info2", url_path="getArtistInfo2"
@action(
detail=False,
methods=["get", "post"],
url_name="get_artist_info2",
url_path="getArtistInfo2",
)
@find_object(music_models.Artist.objects.all(), filter_playable=True)
def get_artist_info2(self, request, *args, **kwargs):
@ -176,7 +206,9 @@ class SubsonicViewSet(viewsets.GenericViewSet):
return response.Response(payload, status=200)
@list_route(methods=["get", "post"], url_name="get_album", url_path="getAlbum")
@action(
detail=False, methods=["get", "post"], url_name="get_album", url_path="getAlbum"
)
@find_object(
music_models.Album.objects.select_related("artist"), filter_playable=True
)
@ -186,46 +218,88 @@ class SubsonicViewSet(viewsets.GenericViewSet):
payload = {"album": data}
return response.Response(payload, status=200)
@list_route(methods=["get", "post"], url_name="stream", url_path="stream")
@action(detail=False, methods=["get", "post"], url_name="stream", url_path="stream")
@find_object(music_models.Track.objects.all(), filter_playable=True)
def stream(self, request, *args, **kwargs):
data = request.GET or request.POST
track = kwargs.pop("obj")
queryset = track.uploads.select_related("track__album__artist", "track__artist")
upload = queryset.first()
if not upload:
return response.Response(status=404)
return music_views.handle_serve(upload=upload, user=request.user)
@list_route(methods=["get", "post"], url_name="star", url_path="star")
format = data.get("format", "raw")
if format == "raw":
format = None
return music_views.handle_serve(upload=upload, user=request.user, format=format)
@action(detail=False, methods=["get", "post"], url_name="star", url_path="star")
@find_object(music_models.Track.objects.all())
def star(self, request, *args, **kwargs):
track = kwargs.pop("obj")
TrackFavorite.add(user=request.user, track=track)
return response.Response({"status": "ok"})
@list_route(methods=["get", "post"], url_name="unstar", url_path="unstar")
@action(detail=False, methods=["get", "post"], url_name="unstar", url_path="unstar")
@find_object(music_models.Track.objects.all())
def unstar(self, request, *args, **kwargs):
track = kwargs.pop("obj")
request.user.track_favorites.filter(track=track).delete()
return response.Response({"status": "ok"})
@list_route(
methods=["get", "post"], url_name="get_starred2", url_path="getStarred2"
@action(
detail=False,
methods=["get", "post"],
url_name="get_starred2",
url_path="getStarred2",
)
def get_starred2(self, request, *args, **kwargs):
favorites = request.user.track_favorites.all()
data = {"starred2": {"song": serializers.get_starred_tracks_data(favorites)}}
return response.Response(data)
@list_route(methods=["get", "post"], url_name="get_starred", url_path="getStarred")
@action(
detail=False,
methods=["get", "post"],
url_name="get_random_songs",
url_path="getRandomSongs",
)
def get_random_songs(self, request, *args, **kwargs):
data = request.GET or request.POST
actor = utils.get_actor_from_request(request)
queryset = music_models.Track.objects.all()
queryset = queryset.playable_by(actor)
try:
size = int(data["size"])
except (TypeError, KeyError, ValueError):
size = 50
queryset = (
queryset.playable_by(actor).prefetch_related("uploads").order_by("?")[:size]
)
data = {
"randomSongs": {
"song": serializers.GetSongSerializer(queryset, many=True).data
}
}
return response.Response(data)
@action(
detail=False,
methods=["get", "post"],
url_name="get_starred",
url_path="getStarred",
)
def get_starred(self, request, *args, **kwargs):
favorites = request.user.track_favorites.all()
data = {"starred": {"song": serializers.get_starred_tracks_data(favorites)}}
return response.Response(data)
@list_route(
methods=["get", "post"], url_name="get_album_list2", url_path="getAlbumList2"
@action(
detail=False,
methods=["get", "post"],
url_name="get_album_list2",
url_path="getAlbumList2",
)
def get_album_list2(self, request, *args, **kwargs):
queryset = music_models.Album.objects.with_tracks_count().order_by(
@ -252,7 +326,9 @@ class SubsonicViewSet(viewsets.GenericViewSet):
data = {"albumList2": {"album": serializers.get_album_list2_data(queryset)}}
return response.Response(data)
@list_route(methods=["get", "post"], url_name="search3", url_path="search3")
@action(
detail=False, methods=["get", "post"], url_name="search3", url_path="search3"
)
def search3(self, request, *args, **kwargs):
data = request.GET or request.POST
query = str(data.get("query", "")).replace("*", "")
@ -310,12 +386,16 @@ class SubsonicViewSet(viewsets.GenericViewSet):
utils.get_query(query, c["search_fields"])
)
queryset = queryset.playable_by(actor)
queryset = common_utils.order_for_search(queryset, c["search_fields"][0])
queryset = queryset[offset : offset + size]
payload["searchResult3"][c["subsonic"]] = c["serializer"](queryset)
return response.Response(payload)
@list_route(
methods=["get", "post"], url_name="get_playlists", url_path="getPlaylists"
@action(
detail=False,
methods=["get", "post"],
url_name="get_playlists",
url_path="getPlaylists",
)
def get_playlists(self, request, *args, **kwargs):
playlists = request.user.playlists.with_tracks_count().select_related("user")
@ -326,8 +406,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
}
return response.Response(data)
@list_route(
methods=["get", "post"], url_name="get_playlist", url_path="getPlaylist"
@action(
detail=False,
methods=["get", "post"],
url_name="get_playlist",
url_path="getPlaylist",
)
@find_object(playlists_models.Playlist.objects.with_tracks_count())
def get_playlist(self, request, *args, **kwargs):
@ -335,8 +418,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
data = {"playlist": serializers.get_playlist_detail_data(playlist)}
return response.Response(data)
@list_route(
methods=["get", "post"], url_name="update_playlist", url_path="updatePlaylist"
@action(
detail=False,
methods=["get", "post"],
url_name="update_playlist",
url_path="updatePlaylist",
)
@find_object(lambda request: request.user.playlists.all(), field="playlistId")
def update_playlist(self, request, *args, **kwargs):
@ -377,8 +463,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
data = {"status": "ok"}
return response.Response(data)
@list_route(
methods=["get", "post"], url_name="delete_playlist", url_path="deletePlaylist"
@action(
detail=False,
methods=["get", "post"],
url_name="delete_playlist",
url_path="deletePlaylist",
)
@find_object(lambda request: request.user.playlists.all())
def delete_playlist(self, request, *args, **kwargs):
@ -387,8 +476,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
data = {"status": "ok"}
return response.Response(data)
@list_route(
methods=["get", "post"], url_name="create_playlist", url_path="createPlaylist"
@action(
detail=False,
methods=["get", "post"],
url_name="create_playlist",
url_path="createPlaylist",
)
def create_playlist(self, request, *args, **kwargs):
data = request.GET or request.POST
@ -426,7 +518,43 @@ class SubsonicViewSet(viewsets.GenericViewSet):
data = {"playlist": serializers.get_playlist_detail_data(playlist)}
return response.Response(data)
@list_route(
@action(
detail=False,
methods=["get", "post"],
url_name="get_avatar",
url_path="getAvatar",
)
@find_object(
queryset=users_models.User.objects.exclude(avatar=None).exclude(avatar=""),
model_field="username__iexact",
field="username",
cast=str,
)
def get_avatar(self, request, *args, **kwargs):
user = kwargs.pop("obj")
mapping = {"nginx": "X-Accel-Redirect", "apache2": "X-Sendfile"}
path = music_views.get_file_path(user.avatar)
file_header = mapping[settings.REVERSE_PROXY_TYPE]
# let the proxy set the content-type
r = response.Response({}, content_type="")
r[file_header] = path
return r
@action(
detail=False, methods=["get", "post"], url_name="get_user", url_path="getUser"
)
@find_object(
queryset=lambda request: users_models.User.objects.filter(pk=request.user.pk),
model_field="username__iexact",
field="username",
cast=str,
)
def get_user(self, request, *args, **kwargs):
data = {"user": serializers.get_user_detail_data(request.user)}
return response.Response(data)
@action(
detail=False,
methods=["get", "post"],
url_name="get_music_folders",
url_path="getMusicFolders",
@ -435,8 +563,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
data = {"musicFolders": {"musicFolder": [{"id": 1, "name": "Music"}]}}
return response.Response(data)
@list_route(
methods=["get", "post"], url_name="get_cover_art", url_path="getCoverArt"
@action(
detail=False,
methods=["get", "post"],
url_name="get_cover_art",
url_path="getCoverArt",
)
def get_cover_art(self, request, *args, **kwargs):
data = request.GET or request.POST
@ -472,7 +603,9 @@ class SubsonicViewSet(viewsets.GenericViewSet):
r[file_header] = path
return r
@list_route(methods=["get", "post"], url_name="scrobble", url_path="scrobble")
@action(
detail=False, methods=["get", "post"], url_name="scrobble", url_path="scrobble"
)
def scrobble(self, request, *args, **kwargs):
data = request.GET or request.POST
serializer = serializers.ScrobbleSerializer(

5
api/funkwhale_api/users/admin.py
View File

@ -51,7 +51,7 @@ class UserAdmin(AuthUserAdmin):
"privacy_level",
"permission_settings",
"permission_library",
"permission_federation",
"permission_moderation",
]
fieldsets = (
@ -67,10 +67,9 @@ class UserAdmin(AuthUserAdmin):
"is_active",
"is_staff",
"is_superuser",
"permission_upload",
"permission_library",
"permission_settings",
"permission_federation",
"permission_moderation",
)
},
),

Some files were not shown because too many files have changed in this diff Show More