Merge branch 'release/0.16'

This commit is contained in:
Eliot Berriot 2018-07-22 22:44:14 +02:00
commit f3ce4f443d
No known key found for this signature in database
GPG Key ID: DD6965E2476E5C27
228 changed files with 27586 additions and 2507 deletions

View File

@ -10,5 +10,4 @@ PYTHONDONTWRITEBYTECODE=true
WEBPACK_DEVSERVER_PORT=8080
MUSIC_DIRECTORY_PATH=/music
BROWSABLE_API_ENABLED=True
CACHEOPS_ENABLED=False
FORWARDED_PROTO=http

2
.gitignore vendored
View File

@ -91,3 +91,5 @@ data/
po/*.po
docs/swagger
_build
front/src/translations.json
front/locales/en_US/LC_MESSAGES/app.po

View File

@ -4,7 +4,8 @@ variables:
IMAGE_LATEST: $IMAGE_NAME:latest
PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
PYTHONDONTWRITEBYTECODE: "true"
REVIEW_DOMAIN: preview.funkwhale.audio
REVIEW_INSTANCE_URL: https://demo.funkwhale.audio
stages:
- review
@ -19,37 +20,42 @@ review_front:
when: manual
allow_failure: true
before_script:
- curl -L -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
- chmod +x /usr/local/bin/jq
- cd front
script:
- yarn install
- yarn run i18n-compile
# this is to ensure we don't have any errors in the output,
# cf https://code.eliotberriot.com/funkwhale/funkwhale/issues/169
- INSTANCE_URL=$REVIEW_INSTANCE_URL yarn run build | tee /dev/stderr | (! grep -i 'ERROR in')
- mkdir -p /static/front/$CI_BUILD_REF_SLUG
- cp -r dist/* /static/front/$CI_BUILD_REF_SLUG
- mkdir -p /static/front/$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG
- cp -r dist/* /static/front/$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG
cache:
key: "$CI_PROJECT_ID__front_dependencies"
key: "funkwhale__front_dependencies"
paths:
- front/node_modules
- front/yarn.lock
environment:
name: review/front-$CI_BUILD_REF_NAME
url: http://front-$CI_BUILD_REF_SLUG.$REVIEW_DOMAIN
name: review/front/$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG
url: http://front-$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG.$REVIEW_DOMAIN
on_stop: stop_front_review
only:
- branches@funkwhale/funkwhale
- branches
tags:
- funkwhale-review
stop_front_review:
stage: review
script:
- rm -rf /static/front/$CI_BUILD_REF_SLUG/
- rm -rf /static/front/$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG/
variables:
GIT_STRATEGY: none
when: manual
only:
- branches
environment:
name: review/front-$CI_BUILD_REF_NAME
name: review/front/$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG
action: stop
tags:
- funkwhale-review
@ -63,33 +69,38 @@ review_docs:
BUILD_PATH: "../public"
before_script:
- cd docs
- apt-get update
- apt-get install -y graphviz
- pip install sphinx
cache:
key: "$CI_PROJECT_ID__sphinx"
paths:
- "$PIP_CACHE_DIR"
script:
- pip install sphinx
- ./build_docs.sh
- mkdir -p /static/docs/$CI_BUILD_REF_SLUG
- cp -r $CI_PROJECT_DIR/public/* /static/docs/$CI_BUILD_REF_SLUG
- mkdir -p /static/docs/$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG
- cp -r $CI_PROJECT_DIR/public/* /static/docs/$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG
environment:
name: review/docs-$CI_BUILD_REF_NAME
url: http://docs-$CI_BUILD_REF_SLUG.$REVIEW_DOMAIN
name: review/docs/$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG
url: http://docs-$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG.$REVIEW_DOMAIN
on_stop: stop_docs_review
only:
- branches@funkwhale/funkwhale
- branches
tags:
- funkwhale-review
stop_docs_review:
stage: review
script:
- rm -rf /static/docs/$CI_BUILD_REF_SLUG/
- rm -rf /static/docs/$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG/
variables:
GIT_STRATEGY: none
when: manual
only:
- branches
environment:
name: review/docs-$CI_BUILD_REF_NAME
name: review/docs/$CI_PROJECT_PATH_SLUG-$CI_BUILD_REF_SLUG
action: stop
tags:
- funkwhale-review
@ -132,9 +143,9 @@ test_api:
DJANGO_ALLOWED_HOSTS: "localhost"
DATABASE_URL: "postgresql://postgres@postgres/postgres"
FUNKWHALE_URL: "https://funkwhale.ci"
CACHEOPS_ENABLED: "false"
DJANGO_SETTINGS_MODULE: config.settings.local
only:
- branches
before_script:
- cd api
- pip install -r requirements/base.txt
@ -151,12 +162,13 @@ test_front:
image: node:9
before_script:
- cd front
only:
- branches
script:
- yarn install
- yarn run unit
cache:
key: "$CI_PROJECT_ID__front_dependencies"
key: "funkwhale__front_dependencies"
paths:
- front/node_modules
- front/yarn.lock
@ -172,17 +184,18 @@ build_front:
stage: build
image: node:9
before_script:
- curl -L -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
- chmod +x /usr/local/bin/jq
- cd front
script:
- yarn install
- yarn run i18n-extract
- yarn run i18n-compile
# this is to ensure we don't have any errors in the output,
# cf https://code.eliotberriot.com/funkwhale/funkwhale/issues/169
- yarn run build | tee /dev/stderr | (! grep -i 'ERROR in')
- chmod -R 750 dist
cache:
key: "$CI_PROJECT_ID__front_dependencies"
key: "funkwhale__front_dependencies"
paths:
- front/node_modules
- front/yarn.lock
@ -205,8 +218,10 @@ pages:
BUILD_PATH: "../public"
before_script:
- cd docs
script:
- apt-get update
- apt-get install -y graphviz
- pip install sphinx
script:
- ./build_docs.sh
cache:
key: "$CI_PROJECT_ID__sphinx"
@ -243,7 +258,9 @@ build_api:
name: "api_${CI_COMMIT_REF_NAME}"
paths:
- api
script: echo Done!
script:
- chmod -R 750 api
- echo Done!
only:
- tags@funkwhale/funkwhale
- master@funkwhale/funkwhale

237
CHANGELOG
View File

@ -4,12 +4,245 @@ Changelog
You can subscribe to release announcements by:
- Following `funkwhale@mastodon.eliotberriot.com <https://mastodon.eliotberriot.com/@funkwhale>`_ on Mastodon
- Subscribing to the following Atom feed: https://code.eliotberriot.com/funkwhale/funkwhale/commits/develop?format=atom&search=tag
- Subscribing to the following Atom feed: https://code.eliotberriot.com/funkwhale/funkwhale/commits/develop?format=atom&search=Merge+tag
This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.html.
.. towncrier
0.16 (unreleased)
-----------------
Upgrade instructions are available at
https://docs.funkwhale.audio/upgrading.html
Features:
- Complete redesign of the library home and playlist pages (#284)
- Expose ActivityPub actors for users (#317)
- Implemented a basic but functionnal Github-like search on federated tracks
list (#344)
- Internationalized interface as well as translations for Arabic, French,
Esperanto, Italian, Occitan, Polish, Portuguese and Swedish (#161, #167)
- Users can now upload an avatar in their settings page (#257)
Enhancements:
- Added feedback when creating/updating radio (#302)
- Apply restrictions to username characters during signup
- Autoselect best language based on browser configuration (#386)
- Can now order tracks on federated track list (#326)
- Can now relaunch pending import jobs from the web interface (#323)
- Ensure we do not display pagination on single pages (#334)
- Ensure we have sane defaults for MEDIA_ROOT, STATIC_ROOT and
MUSIC_DIRECTORY_PATH in the deployment .env file (#350)
- Make some space for the volume slider to allow precise control (#318)
- Removed django-cacheops dependency
- Store track artist and album artist separately (#237) Better handling of
tracks with a different artist than the album artist
- The navigation bar of Library is now fixed (#375)
- Use thumbnails for avatars and covers to reduce bandwidth
Bugfixes:
- Ensure 750 permissions on CI artifacts (#332)
- Ensure images are not cropped in queue (#337)
- Ensure we do not import artists with empty names (#351)
- Fix notifications not closing when clicking on the cross (#366)
- Fix the most annoying offset in the whole fediverse (#369)
- Fixed persistent message in playlist modal (#304)
- Fixed unfiltered results in favorites API (#384)
- Raise a warning instead of crashing when getting a broken path in file import
(#138)
- Remove parallelization of uploads during import to avoid crashing small
servers (#382)
- Subsonic API login is now case insensitive (#339)
- Validate Date header in HTTP Signatures (#328)
Documentation:
- Added troubleshotting and technical overview documentation (#256)
- Arch Linux installation steps
- Document that users can use Ultrasonic on Android (#316)
- Fixed a couple of typos
- Some cosmetic improvements to the doc
i18n:
- Arabic translation (!302)
- Polish translation (!304)
Library home and playlist page overhaul
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The library home page have been completely redesigned to include:
- other users activity (listenings, playlists and favorites)
- recently imported albums
We think this new version showcases more music in a more useful way, let us know
what you think about it!
The playlist page have been updated as well.
Internationalized interface
^^^^^^^^^^^^^^^^^^^^^^^^^^^
After months of work, we're proud to announce our interface is now ready
for internationalization.
Translators have already started the work of translating Funkwhale in 8 different languages,
and we're ready to add more as needed.
You can easily get involved at https://translate.funkwhale.audio/engage/funkwhale/
Better handling of tracks with a different artist than the album artist
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Some tracks involve a different artist than the album artist (e.g. a featuring)
and Funkwhale has been known to do weird things when importing such tracks, resulting
in albums that contained a single track, for instance.
The situation should be improved with this release, as Funkwhale is now able to
store separately the track and album artist, and display it properly in the interface.
Users now have an ActivityPub Actor [Manual action required]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
In the process of implementing federation for user activity such as listening
history, we are now making user profiles (a.k.a. ActivityPub actors) available through federation.
This does not means the federation is working, but this is a needed step to implement it.
Those profiles will be created automatically for new users, but you have to run a command
to create them for existing users.
On docker setups::
docker-compose run --rm api python manage.py script create_actors --no-input
On non-docker setups::
python manage.py script create_actors --no-input
This should only take a few seconds to run. It is safe to interrupt the process or rerun it multiple times.
Image thumbnails [Manual action required]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
To reduce bandwidth usage on slow or limited connexions and improve performance
in general, we now use smaller images in the front-end. For instance, if you have
an album cover with a 1000x1000 pixel size, we will create smaller
versions of this image (50x50, 200x200, 400x400) and reference those resized version
when we don't actually need the original image.
Thumbnail will be created automatically for new objects, however, you have
to launch a manual command to deal with existing ones.
On docker setups::
docker-compose run --rm api python manage.py script create_image_variations --no-input
On non-docker setups::
python manage.py script create_image_variations --no-input
This should be quite fast but may take up to a few minutes depending on the number
of albums you have in database. It is safe to interrupt the process or rerun it multiple times.
Improved search on federated tracks list
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Having a powerful but easy-to-use search is important but difficult to achieve, especially
if you do not want to have a real complex search interface.
Github does a pretty good job with that, using a structured but simple query system
(See https://help.github.com/articles/searching-issues-and-pull-requests/#search-only-issues-or-pull-requests).
This release implements a limited but working subset of this query system. You can use it only on the federated
tracks list (/manage/federation/tracks) at the moment, but depending on feedback it will be rolled-out on other pages as well.
This is the type of query you can run:
- ``hello world``: search for "hello" and "world" in all the available fields
- ``hello in:artist`` search for results where artist name is "hello"
- ``spring in:artist,album`` search for results where artist name or album title contain "spring"
- ``artist:hello`` search for results where artist name equals "hello"
- ``artist:"System of a Down" domain:instance.funkwhale`` search for results where artist name equals "System of a Down" and inside "instance.funkwhale" library
Ensure MEDIA_ROOT, STATIC_ROOT and MUSIC_DIRECTORY_* are set explicitely [Manual action required]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
In our default .env file, MEDIA_ROOT and STATIC_ROOT were commented by default, causing
some deployment issues on non-docker setups when people forgot to uncomment them.
From now on, those variables are uncommented, and will also be used on docker setups
to mount the volumes automatically in the docker-compose.yml file. This has been a source
of headache as well in some deployments, where you had to update both the .env file and
the compose file.
This also applies to in-place paths (MUSIC_DIRECTORY_PATH and MUSIC_DIRECTORY_SERVE_PATH),
whose values are now used directly to set up the proper Docker volumes.
This will only affect new deployments though. If you want to benefit from this on an
existing instance, do a backup of your ``.env`` and ``docker-compose.yml`` files and apply the following changes:
- Ensure ``MEDIA_ROOT`` is uncommented in your .env file and match the absolute path where media files are stored
on your host (``/srv/funkwhale/data/media`` by default)
- Ensure ``STATIC_ROOT`` is uncommented in your .env file and match the absolute path where static files are stored
on your host (``/srv/funkwhale/data/static`` by default)
- If you use in-place import:
- Ensure MUSIC_DIRECTORY_PATH is uncommented and set to ``/music``
- Ensure MUSIC_DIRECTORY_SERVE_PATH is uncommented and set to the absolute path on your host were your music files
are stored (``/srv/funkwhale/data/music`` by default)
- Edit your docker-compose.yml file to reflect the changes:
- Search for volumes (there should be two occurences) that contains ``/app/funkwhale_api/media`` on the right side, and
replace the whole line with ``- "${MEDIA_ROOT}:${MEDIA_ROOT}"``
- Search for a volume that contains ``/app/staticfiles`` on the right side, and
replace the whole line with ``- "${STATIC_ROOT}:${STATIC_ROOT}"``
- If you use in-place import, search for volumes (there should be two occurences) that contains ``/music:ro`` on the right side, and
replace the whole line with ``- "${MUSIC_DIRECTORY_SERVE_PATH}:${MUSIC_DIRECTORY_PATH}:ro"``
In the end, the ``volumes`` directives of your containers should look like that::
...
celeryworker
volumes:
- "${MUSIC_DIRECTORY_SERVE_PATH}:${MUSIC_DIRECTORY_PATH}:ro"
- "${MEDIA_ROOT}:${MEDIA_ROOT}"
...
api:
volumes:
- "${MUSIC_DIRECTORY_SERVE_PATH}:${MUSIC_DIRECTORY_PATH}:ro"
- "${MEDIA_ROOT}:${MEDIA_ROOT}"
- "${STATIC_ROOT}:${STATIC_ROOT}"
- ./front/dist:/frontend
...
Removed Cacheops dependency
---------------------------
We removed one of our dependency named django-cacheops. It was unly used in a few places,
and not playing nice with other dependencies.
You can safely remove this dependency in your environment with ``pip uninstall django-cacheops`` if you're
not using docker.
You can also safely remove any ``CACHEOPS_ENABLED`` setting from your environment file.
0.15 (2018-06-24)
-----------------
@ -1346,7 +1579,7 @@ Basic transcoding is now available to/from the following formats : ogg and mp3.
This relies internally on FFMPEG and can put some load on your server.
It's definitely recommended you setup some caching for the transcoded files
at your webserver level. Check the the exemple nginx file at deploy/nginx.conf
at your webserver level. Check the the exemple nginx file at deploy/nginx.conf
for an implementation.
On the frontend, usage of transcoding should be transparent in the player.

View File

@ -1,5 +1,5 @@
Contribute to Funkwhale development
==================================
===================================
First of all, thank you for your interest in the project! We really
appreciate the fact that you're about to take some time to read this
@ -82,7 +82,7 @@ Visit https://code.eliotberriot.com/funkwhale/funkwhale and clone the repository
A note about branches
^^^^^^^^^^^^^^^^^^^^^
Next release development occurs on the "develop" branch, and releases are made on the "master" branch. Therefor, when submitting Merge Requests, ensure you are merging on the develop branch.
Next release development occurs on the "develop" branch, and releases are made on the "master" branch. Therefore, when submitting Merge Requests, ensure you are merging on the develop branch.
Working with docker
@ -111,7 +111,7 @@ Create it like this::
Create docker network
^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^
Create the federation network::
@ -280,7 +280,7 @@ Typical workflow for a contribution
3. Create a dedicated branch for your work ``42-awesome-fix``. It is good practice to prefix your branch name with the ID of the issue you are solving.
4. Work on your stuff
5. Commit small, atomic changes to make it easier to review your contribution
6. Add a changelog fragment to summarize your changes: ``echo "Implemented awesome stuff (#42)" > changes/changelog.d/42.feature"``
6. Add a changelog fragment to summarize your changes: ``echo "Implemented awesome stuff (#42)" > changes/changelog.d/42.feature``
7. Push your branch
8. Create your merge request
9. Take a step back and enjoy, we're really grateful you did all of this and took the time to contribute!
@ -289,8 +289,9 @@ Typical workflow for a contribution
Internationalization
--------------------
We're using https://github.com/Polyconseil/vue-gettext to manage i18n in the project.
When working on the front-end, any end-user string should be translated
using either ``<i18next path="yourstring">`` or the ``$t('yourstring')``
using either ``<translate>yourstring</translate>`` or ``$gettext('yourstring')``
function.
Extraction is done by calling ``yarn run i18n-extract``, which

View File

@ -14,8 +14,8 @@ Getting help
We offer various Matrix.org rooms to discuss about Funkwhale:
- `#funkwhale:matrix.org <https://riot.im/app/#/room/#funkwhale:matrix.org>`_ for general questions about funkwhale
- `#funkwhale-dev:matrix.org <https://riot.im/app/#/room/#funkwhale-dev:matrix.org>`_ for development-focused discussion
- `#funkwhale:matrix.org <https://matrix.to/#/#funkwhale:matrix.org>`_ for general questions about funkwhale
- `#funkwhale-dev:matrix.org <https://matrix.to/#/#funkwhale-dev:matrix.org>`_ for development-focused discussion
Please join those rooms if you have any questions!
@ -26,4 +26,9 @@ Contribute
----------
Contribution guidelines as well as development installation instructions
are outlined in `CONTRIBUTING <CONTRIBUTING>`_
are outlined in `CONTRIBUTING <CONTRIBUTING>`_.
Translate
^^^^^^^^^
Translators willing to help can refer to `TRANSLATORS <TRANSLATORS>`_ for instructions.

28
TRANSLATORS.rst Normal file
View File

@ -0,0 +1,28 @@
Translating Funkwhale
=====================
Thank you for reading this! If you want to help translate Funkwhale,
you found the proper place :)
Translation is done via our own Weblate instance at https://translate.funkwhale.audio/projects/funkwhale/front/.
You can signup/login using your Gitlab account (from https://code.eliotberriot.com).
Translation workflow
--------------------
Once you're logged-in on the Weblate instance, you can suggest translations. Your suggestions will then be reviewer
by the project maintainer or other translators to ensure consistency.
Guidelines
----------
Respecting those guidelines is mandatory if you want your translation to be included:
- Use gender-neutral language and wording
Requesting a new language
-------------------------
If you'd like to see a new language in Funkwhale, please open an issue here:
https://code.eliotberriot.com/funkwhale/funkwhale/issues

View File

@ -92,8 +92,8 @@ THIRD_PARTY_APPS = (
"rest_auth.registration",
"dynamic_preferences",
"django_filters",
"cacheops",
"django_cleanup",
"versatileimagefield",
)
@ -302,6 +302,7 @@ SESSION_COOKIE_HTTPONLY = False
ACCOUNT_AUTHENTICATION_METHOD = "username_email"
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_VERIFICATION = "mandatory"
ACCOUNT_USERNAME_VALIDATORS = "funkwhale_api.users.serializers.username_validators"
# Custom user app defaults
# Select the correct user model
@ -420,15 +421,6 @@ PROTECT_FILES_PATH = env("PROTECT_FILES_PATH", default="/_protected")
# use this setting to tweak for how long you want to cache
# musicbrainz results. (value is in seconds)
MUSICBRAINZ_CACHE_DURATION = env.int("MUSICBRAINZ_CACHE_DURATION", default=300)
CACHEOPS_REDIS = env("CACHE_URL", default=CACHE_DEFAULT)
CACHEOPS_ENABLED = env.bool("CACHEOPS_ENABLED", default=True)
CACHEOPS = {
"music.artist": {"ops": "all", "timeout": 60 * 60},
"music.album": {"ops": "all", "timeout": 60 * 60},
"music.track": {"ops": "all", "timeout": 60 * 60},
"music.trackfile": {"ops": "all", "timeout": 60 * 60},
"taggit.tag": {"ops": "all", "timeout": 60 * 60},
}
# Custom Admin URL, use {% url 'admin:index' %}
ADMIN_URL = env("DJANGO_ADMIN_URL", default="^api/admin/")
@ -441,6 +433,7 @@ PLAYLISTS_MAX_TRACKS = env.int("PLAYLISTS_MAX_TRACKS", default=250)
ACCOUNT_USERNAME_BLACKLIST = [
"funkwhale",
"library",
"instance",
"test",
"status",
"root",
@ -449,6 +442,11 @@ ACCOUNT_USERNAME_BLACKLIST = [
"superuser",
"staff",
"service",
"me",
"ghost",
"_",
"hello",
"contact",
] + env.list("ACCOUNT_USERNAME_BLACKLIST", default=[])
EXTERNAL_REQUESTS_VERIFY_SSL = env.bool("EXTERNAL_REQUESTS_VERIFY_SSL", default=True)
@ -465,3 +463,13 @@ MUSIC_DIRECTORY_SERVE_PATH = env(
USERS_INVITATION_EXPIRATION_DAYS = env.int(
"USERS_INVITATION_EXPIRATION_DAYS", default=14
)
VERSATILEIMAGEFIELD_RENDITION_KEY_SETS = {
"square": [
("original", "url"),
("square_crop", "crop__400x400"),
("medium_square_crop", "crop__200x200"),
("small_square_crop", "crop__50x50"),
]
}
VERSATILEIMAGEFIELD_SETTINGS = {"create_images_on_demand": False}

View File

@ -39,6 +39,7 @@ DEBUG_TOOLBAR_CONFIG = {
"DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
"SHOW_TEMPLATE_CONTEXT": True,
"SHOW_TOOLBAR_CALLBACK": lambda request: True,
"JQUERY_URL": "",
}
# django-extensions

View File

@ -51,12 +51,6 @@ CSRF_TRUSTED_ORIGINS = ALLOWED_HOSTS
# END SITE CONFIGURATION
# STORAGE CONFIGURATION
# ------------------------------------------------------------------------------
# Uploaded Media Files
# ------------------------
DEFAULT_FILE_STORAGE = "django.core.files.storage.FileSystemStorage"
# Static Assets
# ------------------------
STATICFILES_STORAGE = "django.contrib.staticfiles.storage.StaticFilesStorage"

View File

@ -1,7 +0,0 @@
from funkwhale_api.users.models import User
u = User.objects.create(email="demo@demo.com", username="demo", is_staff=True)
u.set_password("demo")
u.subsonic_api_token = "demo"
u.save()

View File

@ -1,13 +0,0 @@
#! /bin/bash
echo "Loading demo data..."
python manage.py migrate --noinput
echo "Creating demo user..."
cat demo/demo-user.py | python manage.py shell -i python
echo "Importing demo tracks..."
python manage.py import_files "/music/**/*.ogg" --recursive --noinput --username demo

View File

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
__version__ = "0.15"
__version__ = "0.16"
__version_info__ = tuple(
[
int(num) if num.isdigit() else num

View File

@ -1,7 +1,7 @@
import django_filters
from django.db import models
from funkwhale_api.music import utils
from . import search
PRIVACY_LEVEL_CHOICES = [
("me", "Only me"),
@ -34,5 +34,17 @@ class SearchFilter(django_filters.CharFilter):
def filter(self, qs, value):
if not value:
return qs
query = utils.get_query(value, self.search_fields)
query = search.get_query(value, self.search_fields)
return qs.filter(query)
class SmartSearchFilter(django_filters.CharFilter):
def __init__(self, *args, **kwargs):
self.config = kwargs.pop("config")
super().__init__(*args, **kwargs)
def filter(self, qs, value):
if not value:
return qs
cleaned = self.config.clean(value)
return search.apply(qs, cleaned)

View File

@ -19,7 +19,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
name = options["script_name"]
if not name:
self.show_help()
return self.show_help()
available_scripts = self.get_scripts()
try:
@ -50,7 +50,7 @@ class Command(BaseCommand):
self.stdout.write(self.style.SUCCESS(name))
self.stdout.write("")
for line in script["help"].splitlines():
self.stdout.write("     {}".format(line))
self.stdout.write(" {}".format(line))
self.stdout.write("")
def get_scripts(self):

View File

@ -14,6 +14,11 @@ def get(pref):
return manager[pref]
def set(pref, value):
manager = global_preferences_registry.manager()
manager[pref] = value
class StringListSerializer(serializers.BaseSerializer):
separator = ","
sort = True

View File

@ -0,0 +1,12 @@
from . import create_actors
from . import create_image_variations
from . import django_permissions_to_user_permissions
from . import test
__all__ = [
"create_actors",
"create_image_variations",
"django_permissions_to_user_permissions",
"test",
]

View File

@ -0,0 +1,23 @@
"""
Compute different sizes of image used for Album covers and User avatars
"""
from django.db.utils import IntegrityError
from funkwhale_api.users.models import User, create_actor
def main(command, **kwargs):
qs = User.objects.filter(actor__isnull=True).order_by("username")
total = len(qs)
command.stdout.write("{} users found without actors".format(total))
for i, user in enumerate(qs):
command.stdout.write(
"{}/{} creating actor for {}".format(i + 1, total, user.username)
)
try:
user.actor = create_actor(user)
except IntegrityError as e:
# somehow, an actor with the the url exists in the database
command.stderr.write("Error while creating actor: {}".format(str(e)))
continue
user.save(update_fields=["actor"])

View File

@ -0,0 +1,30 @@
"""
Compute different sizes of image used for Album covers and User avatars
"""
from versatileimagefield.image_warmer import VersatileImageFieldWarmer
from funkwhale_api.music.models import Album
from funkwhale_api.users.models import User
MODELS = [(Album, "cover", "square"), (User, "avatar", "square")]
def main(command, **kwargs):
for model, attribute, key_set in MODELS:
qs = model.objects.exclude(**{"{}__isnull".format(attribute): True})
qs = qs.exclude(**{attribute: ""})
warmer = VersatileImageFieldWarmer(
instance_or_queryset=qs,
rendition_key_set=key_set,
image_attr=attribute,
verbose=True,
)
command.stdout.write(
"Creating images for {} / {}".format(model.__name__, attribute)
)
num_created, failed_to_create = warmer.warm()
command.stdout.write(
" {} created, {} in error".format(num_created, len(failed_to_create))
)

View File

@ -0,0 +1,130 @@
import re
from django.db.models import Q
QUERY_REGEX = re.compile('(((?P<key>\w+):)?(?P<value>"[^"]+"|[\S]+))')
def parse_query(query):
"""
Given a search query such as "hello is:issue status:opened",
returns a list of dictionnaries discribing each query token
"""
matches = [m.groupdict() for m in QUERY_REGEX.finditer(query.lower())]
for m in matches:
if m["value"].startswith('"') and m["value"].endswith('"'):
m["value"] = m["value"][1:-1]
return matches
def normalize_query(
query_string,
findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
normspace=re.compile(r"\s{2,}").sub,
):
""" Splits the query string in invidual keywords, getting rid of unecessary spaces
and grouping quoted words together.
Example:
>>> normalize_query(' some random words "with quotes " and spaces')
['some', 'random', 'words', 'with quotes', 'and', 'spaces']
"""
return [normspace(" ", (t[0] or t[1]).strip()) for t in findterms(query_string)]
def get_query(query_string, search_fields):
""" Returns a query, that is a combination of Q objects. That combination
aims to search keywords within a model by testing the given search fields.
"""
query = None # Query to search for every search term
terms = normalize_query(query_string)
for term in terms:
or_query = None # Query to search for a given term in each field
for field_name in search_fields:
q = Q(**{"%s__icontains" % field_name: term})
if or_query is None:
or_query = q
else:
or_query = or_query | q
if query is None:
query = or_query
else:
query = query & or_query
return query
def filter_tokens(tokens, valid):
return [t for t in tokens if t["key"] in valid]
def apply(qs, config_data):
for k in ["filter_query", "search_query"]:
q = config_data.get(k)
if q:
qs = qs.filter(q)
return qs
class SearchConfig:
def __init__(self, search_fields={}, filter_fields={}, types=[]):
self.filter_fields = filter_fields
self.search_fields = search_fields
self.types = types
def clean(self, query):
tokens = parse_query(query)
cleaned_data = {}
cleaned_data["types"] = self.clean_types(filter_tokens(tokens, ["is"]))
cleaned_data["search_query"] = self.clean_search_query(
filter_tokens(tokens, [None, "in"])
)
unhandled_tokens = [t for t in tokens if t["key"] not in [None, "is", "in"]]
cleaned_data["filter_query"] = self.clean_filter_query(unhandled_tokens)
return cleaned_data
def clean_search_query(self, tokens):
if not self.search_fields or not tokens:
return
fields_subset = {
f for t in filter_tokens(tokens, ["in"]) for f in t["value"].split(",")
} or set(self.search_fields.keys())
fields_subset = set(self.search_fields.keys()) & fields_subset
to_fields = [self.search_fields[k]["to"] for k in fields_subset]
query_string = " ".join([t["value"] for t in filter_tokens(tokens, [None])])
return get_query(query_string, sorted(to_fields))
def clean_filter_query(self, tokens):
if not self.filter_fields or not tokens:
return
matching = [t for t in tokens if t["key"] in self.filter_fields]
queries = [
Q(**{self.filter_fields[t["key"]]["to"]: t["value"]}) for t in matching
]
query = None
for q in queries:
if not query:
query = q
else:
query = query & q
return query
def clean_types(self, tokens):
if not self.types:
return []
if not tokens:
# no filtering on type, we return all types
return [t for key, t in self.types]
types = []
for token in tokens:
for key, t in self.types:
if key.lower() == token["value"]:
types.append(t)
return types

View File

@ -1,5 +1,9 @@
from django.utils.deconstruct import deconstructible
import os
import shutil
import uuid
from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
from django.db import transaction
@ -41,3 +45,22 @@ def set_query_parameter(url, **kwargs):
new_query_string = urlencode(query_params, doseq=True)
return urlunsplit((scheme, netloc, path, new_query_string, fragment))
@deconstructible
class ChunkedPath(object):
def __init__(self, root, preserve_file_name=True):
self.root = root
self.preserve_file_name = preserve_file_name
def __call__(self, instance, filename):
uid = str(uuid.uuid4())
chunk_size = 2
chunks = [uid[i : i + chunk_size] for i in range(0, len(uid), chunk_size)]
if self.preserve_file_name:
parts = chunks[:3] + [filename]
else:
ext = os.path.splitext(filename)[1][1:].lower()
new_filename = "".join(chunks[3:]) + ".{}".format(ext)
parts = chunks[:3] + [new_filename]
return os.path.join(self.root, *parts)

View File

@ -0,0 +1,152 @@
import mimetypes
from os.path import splitext
from django.core.exceptions import ValidationError
from django.core.files.images import get_image_dimensions
from django.template.defaultfilters import filesizeformat
from django.utils.deconstruct import deconstructible
from django.utils.translation import ugettext_lazy as _
@deconstructible
class ImageDimensionsValidator:
"""
ImageField dimensions validator.
from https://gist.github.com/emilio-rst/4f81ea2718736a6aaf9bdb64d5f2ea6c
"""
def __init__(
self,
width=None,
height=None,
min_width=None,
max_width=None,
min_height=None,
max_height=None,
):
"""
Constructor
Args:
width (int): exact width
height (int): exact height
min_width (int): minimum width
min_height (int): minimum height
max_width (int): maximum width
max_height (int): maximum height
"""
self.width = width
self.height = height
self.min_width = min_width
self.max_width = max_width
self.min_height = min_height
self.max_height = max_height
def __call__(self, image):
w, h = get_image_dimensions(image)
if self.width is not None and w != self.width:
raise ValidationError(_("Width must be %dpx.") % (self.width,))
if self.height is not None and h != self.height:
raise ValidationError(_("Height must be %dpx.") % (self.height,))
if self.min_width is not None and w < self.min_width:
raise ValidationError(_("Minimum width must be %dpx.") % (self.min_width,))
if self.min_height is not None and h < self.min_height:
raise ValidationError(
_("Minimum height must be %dpx.") % (self.min_height,)
)
if self.max_width is not None and w > self.max_width:
raise ValidationError(_("Maximum width must be %dpx.") % (self.max_width,))
if self.max_height is not None and h > self.max_height:
raise ValidationError(
_("Maximum height must be %dpx.") % (self.max_height,)
)
@deconstructible
class FileValidator(object):
"""
Taken from https://gist.github.com/jrosebr1/2140738
Validator for files, checking the size, extension and mimetype.
Initialization parameters:
allowed_extensions: iterable with allowed file extensions
ie. ('txt', 'doc')
allowd_mimetypes: iterable with allowed mimetypes
ie. ('image/png', )
min_size: minimum number of bytes allowed
ie. 100
max_size: maximum number of bytes allowed
ie. 24*1024*1024 for 24 MB
Usage example::
MyModel(models.Model):
myfile = FileField(validators=FileValidator(max_size=24*1024*1024), ...)
"""
extension_message = _(
"Extension '%(extension)s' not allowed. Allowed extensions are: '%(allowed_extensions)s.'"
)
mime_message = _(
"MIME type '%(mimetype)s' is not valid. Allowed types are: %(allowed_mimetypes)s."
)
min_size_message = _(
"The current file %(size)s, which is too small. The minumum file size is %(allowed_size)s."
)
max_size_message = _(
"The current file %(size)s, which is too large. The maximum file size is %(allowed_size)s."
)
def __init__(self, *args, **kwargs):
self.allowed_extensions = kwargs.pop("allowed_extensions", None)
self.allowed_mimetypes = kwargs.pop("allowed_mimetypes", None)
self.min_size = kwargs.pop("min_size", 0)
self.max_size = kwargs.pop("max_size", None)
def __call__(self, value):
"""
Check the extension, content type and file size.
"""
# Check the extension
ext = splitext(value.name)[1][1:].lower()
if self.allowed_extensions and ext not in self.allowed_extensions:
message = self.extension_message % {
"extension": ext,
"allowed_extensions": ", ".join(self.allowed_extensions),
}
raise ValidationError(message)
# Check the content type
mimetype = mimetypes.guess_type(value.name)[0]
if self.allowed_mimetypes and mimetype not in self.allowed_mimetypes:
message = self.mime_message % {
"mimetype": mimetype,
"allowed_mimetypes": ", ".join(self.allowed_mimetypes),
}
raise ValidationError(message)
# Check the file size
filesize = len(value)
if self.max_size and filesize > self.max_size:
message = self.max_size_message % {
"size": filesizeformat(filesize),
"allowed_size": filesizeformat(self.max_size),
}
raise ValidationError(message)
elif filesize < self.min_size:
message = self.min_size_message % {
"size": filesizeformat(filesize),
"allowed_size": filesizeformat(self.min_size),
}
raise ValidationError(message)

View File

@ -0,0 +1,15 @@
from django_filters import rest_framework as filters
from funkwhale_api.common import fields
from . import models
class TrackFavoriteFilter(filters.FilterSet):
q = fields.SearchFilter(
search_fields=["track__title", "track__artist__name", "track__album__title"]
)
class Meta:
model = models.TrackFavorite
fields = ["user", "q"]

View File

@ -2,8 +2,8 @@
from rest_framework import serializers
from funkwhale_api.activity import serializers as activity_serializers
from funkwhale_api.music.serializers import TrackActivitySerializer
from funkwhale_api.users.serializers import UserActivitySerializer
from funkwhale_api.music.serializers import TrackActivitySerializer, TrackSerializer
from funkwhale_api.users.serializers import UserActivitySerializer, UserBasicSerializer
from . import models
@ -26,6 +26,15 @@ class TrackFavoriteActivitySerializer(activity_serializers.ModelSerializer):
class UserTrackFavoriteSerializer(serializers.ModelSerializer):
track = TrackSerializer(read_only=True)
user = UserBasicSerializer(read_only=True)
class Meta:
model = models.TrackFavorite
fields = ("id", "user", "track", "creation_date")
class UserTrackFavoriteWriteSerializer(serializers.ModelSerializer):
class Meta:
model = models.TrackFavorite
fields = ("id", "track", "creation_date")

View File

@ -1,12 +1,13 @@
from rest_framework import mixins, status, viewsets
from rest_framework.decorators import list_route
from rest_framework.permissions import IsAuthenticatedOrReadOnly
from rest_framework.response import Response
from funkwhale_api.activity import record
from funkwhale_api.common.permissions import ConditionalAuthentication
from funkwhale_api.common import fields, permissions
from funkwhale_api.music.models import Track
from . import models, serializers
from . import filters, models, serializers
class TrackFavoriteViewSet(
@ -16,9 +17,24 @@ class TrackFavoriteViewSet(
viewsets.GenericViewSet,
):
filter_class = filters.TrackFavoriteFilter
serializer_class = serializers.UserTrackFavoriteSerializer
queryset = models.TrackFavorite.objects.all()
permission_classes = [ConditionalAuthentication]
queryset = (
models.TrackFavorite.objects.all()
.select_related("track__artist", "track__album__artist", "user")
.prefetch_related("track__files")
)
permission_classes = [
permissions.ConditionalAuthentication,
permissions.OwnerPermission,
IsAuthenticatedOrReadOnly,
]
owner_checks = ["write"]
def get_serializer_class(self):
if self.request.method.lower() in ["head", "get", "options"]:
return serializers.UserTrackFavoriteSerializer
return serializers.UserTrackFavoriteWriteSerializer
def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
@ -32,7 +48,10 @@ class TrackFavoriteViewSet(
)
def get_queryset(self):
return self.queryset.filter(user=self.request.user)
queryset = super().get_queryset()
return queryset.filter(
fields.privacy_level_query(self.request.user, "user__privacy_level")
)
def perform_create(self, serializer):
track = Track.objects.get(pk=serializer.data["track"])

View File

@ -5,6 +5,7 @@ import requests
import requests_http_signature
from django.conf import settings
from django.utils import timezone
from django.utils.http import http_date
from funkwhale_api.factories import registry
@ -39,7 +40,7 @@ class SignedRequestFactory(factory.Factory):
default_headers = {
"User-Agent": "Test",
"Host": "test.host",
"Date": "Right now",
"Date": http_date(timezone.now().timestamp()),
"Content-Type": "application/activity+json",
}
if extracted:
@ -170,6 +171,7 @@ class LibraryTrackFactory(factory.DjangoModelFactory):
audio_url = factory.Faker("url")
audio_mimetype = "audio/ogg"
metadata = factory.SubFactory(LibraryTrackMetadataFactory)
published_date = factory.LazyFunction(timezone.now)
class Meta:
model = models.LibraryTrack

View File

@ -1,6 +1,7 @@
import django_filters
from funkwhale_api.common import fields
from funkwhale_api.common import search
from . import models
@ -23,8 +24,21 @@ class LibraryFilter(django_filters.FilterSet):
class LibraryTrackFilter(django_filters.FilterSet):
library = django_filters.CharFilter("library__uuid")
status = django_filters.CharFilter(method="filter_status")
q = fields.SearchFilter(
search_fields=["artist_name", "title", "album_title", "library__actor__domain"]
q = fields.SmartSearchFilter(
config=search.SearchConfig(
search_fields={
"domain": {"to": "library__actor__domain"},
"artist": {"to": "artist_name"},
"album": {"to": "album_title"},
"title": {"to": "title"},
},
filter_fields={
"domain": {"to": "library__actor__domain"},
"artist": {"to": "artist_name__iexact"},
"album": {"to": "album_title__iexact"},
"title": {"to": "title__iexact"},
},
)
)
def filter_status(self, queryset, field_name, value):

View File

@ -1,4 +1,3 @@
import os
import tempfile
import uuid
@ -9,6 +8,7 @@ from django.db import models
from django.utils import timezone
from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils
from funkwhale_api.music import utils as music_utils
TYPE_CHOICES = [
@ -20,6 +20,11 @@ TYPE_CHOICES = [
]
class ActorQuerySet(models.QuerySet):
def local(self, include=True):
return self.exclude(user__isnull=include)
class Actor(models.Model):
ap_type = "Actor"
@ -47,6 +52,8 @@ class Actor(models.Model):
related_name="following",
)
objects = ActorQuerySet.as_manager()
class Meta:
unique_together = ["domain", "preferred_username"]
@ -141,12 +148,7 @@ class Library(models.Model):
)
def get_file_path(instance, filename):
uid = str(uuid.uuid4())
chunk_size = 2
chunks = [uid[i : i + chunk_size] for i in range(0, len(uid), chunk_size)]
parts = chunks[:3] + [filename]
return os.path.join("federation_cache", *parts)
get_file_path = common_utils.ChunkedPath("federation_cache")
class LibraryTrack(models.Model):

View File

@ -1,6 +1,8 @@
import logging
import mimetypes
import urllib.parse
from django.core.exceptions import ObjectDoesNotExist
from django.core.paginator import Paginator
from django.db import transaction
from rest_framework import serializers
@ -63,6 +65,15 @@ class ActorSerializer(serializers.Serializer):
ret["endpoints"] = {}
if instance.shared_inbox_url:
ret["endpoints"]["sharedInbox"] = instance.shared_inbox_url
try:
if instance.user.avatar:
ret["icon"] = {
"type": "Image",
"mediaType": mimetypes.guess_type(instance.user.avatar.path)[0],
"url": utils.full_url(instance.user.avatar.crop["400x400"].url),
}
except ObjectDoesNotExist:
pass
return ret
def prepare_missing_fields(self):

View File

@ -1,4 +1,10 @@
import datetime
import logging
import pytz
from django import forms
from django.utils import timezone
from django.utils.http import parse_http_date
import requests
import requests_http_signature
@ -7,8 +13,33 @@ from . import exceptions, utils
logger = logging.getLogger(__name__)
# the request Date should be between now - 30s and now + 30s
DATE_HEADER_VALID_FOR = 30
def verify_date(raw_date):
if not raw_date:
raise forms.ValidationError("Missing date header")
try:
ts = parse_http_date(raw_date)
except ValueError as e:
raise forms.ValidationError(str(e))
dt = datetime.datetime.utcfromtimestamp(ts)
dt = dt.replace(tzinfo=pytz.utc)
delta = datetime.timedelta(seconds=DATE_HEADER_VALID_FOR)
now = timezone.now()
if dt < now - delta or dt > now + delta:
raise forms.ValidationError(
"Request Date is too far in the future or in the past"
)
return dt
def verify(request, public_key):
verify_date(request.headers.get("Date"))
return requests_http_signature.HTTPSignatureAuth.verify(
request, key_resolver=lambda **kwargs: public_key, use_auth_header=False
)

View File

@ -8,6 +8,7 @@ music_router = routers.SimpleRouter(trailing_slash=False)
router.register(
r"federation/instance/actors", views.InstanceActorViewSet, "instance-actors"
)
router.register(r"federation/actors", views.ActorViewSet, "actors")
router.register(r".well-known", views.WellKnownViewSet, "well-known")
music_router.register(r"files", views.MusicFilesViewSet, "files")

View File

@ -5,6 +5,8 @@ def full_url(path):
"""
Given a relative path, return a full url usable for federation purpose
"""
if path.startswith("http://") or path.startswith("https://"):
return path
root = settings.FUNKWHALE_URL
if path.startswith("/") and root.endswith("/"):
return root + path[1:]

View File

@ -32,6 +32,24 @@ class FederationMixin(object):
return super().dispatch(request, *args, **kwargs)
class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
lookup_field = "user__username"
lookup_value_regex = ".*"
authentication_classes = [authentication.SignatureAuthentication]
permission_classes = []
renderer_classes = [renderers.ActivityPubRenderer]
queryset = models.Actor.objects.local().select_related("user")
serializer_class = serializers.ActorSerializer
@detail_route(methods=["get", "post"])
def inbox(self, request, *args, **kwargs):
return response.Response({}, status=200)
@detail_route(methods=["get", "post"])
def outbox(self, request, *args, **kwargs):
return response.Response({}, status=200)
class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet):
lookup_field = "actor"
lookup_value_regex = "[a-z]*"
@ -100,6 +118,8 @@ class WellKnownViewSet(viewsets.GenericViewSet):
resource_type, resource = webfinger.clean_resource(request.GET["resource"])
cleaner = getattr(webfinger, "clean_{}".format(resource_type))
result = cleaner(resource)
handler = getattr(self, "handler_{}".format(resource_type))
data = handler(result)
except forms.ValidationError as e:
return response.Response({"errors": {"resource": e.message}}, status=400)
except KeyError:
@ -107,14 +127,19 @@ class WellKnownViewSet(viewsets.GenericViewSet):
{"errors": {"resource": "This field is required"}}, status=400
)
handler = getattr(self, "handler_{}".format(resource_type))
data = handler(result)
return response.Response(data)
def handler_acct(self, clean_result):
username, hostname = clean_result
actor = actors.SYSTEM_ACTORS[username].get_actor_instance()
if username in actors.SYSTEM_ACTORS:
actor = actors.SYSTEM_ACTORS[username].get_actor_instance()
else:
try:
actor = models.Actor.objects.local().get(user__username=username)
except models.Actor.DoesNotExist:
raise forms.ValidationError("Invalid username")
return serializers.ActorWebfingerSerializer(actor).data

View File

@ -3,7 +3,7 @@ from django.conf import settings
from funkwhale_api.common import session
from . import actors, serializers
from . import serializers
VALID_RESOURCE_TYPES = ["acct"]
@ -32,9 +32,6 @@ def clean_acct(acct_string, ensure_local=True):
if ensure_local and hostname.lower() != settings.FEDERATION_HOSTNAME:
raise forms.ValidationError("Invalid hostname {}".format(hostname))
if ensure_local and username not in actors.SYSTEM_ACTORS:
raise forms.ValidationError("Invalid username")
return username, hostname

View File

@ -1,8 +1,8 @@
from rest_framework import serializers
from funkwhale_api.activity import serializers as activity_serializers
from funkwhale_api.music.serializers import TrackActivitySerializer
from funkwhale_api.users.serializers import UserActivitySerializer
from funkwhale_api.music.serializers import TrackActivitySerializer, TrackSerializer
from funkwhale_api.users.serializers import UserActivitySerializer, UserBasicSerializer
from . import models
@ -25,6 +25,20 @@ class ListeningActivitySerializer(activity_serializers.ModelSerializer):
class ListeningSerializer(serializers.ModelSerializer):
track = TrackSerializer(read_only=True)
user = UserBasicSerializer(read_only=True)
class Meta:
model = models.Listening
fields = ("id", "user", "track", "creation_date")
def create(self, validated_data):
validated_data["user"] = self.context["user"]
return super().create(validated_data)
class ListeningWriteSerializer(serializers.ModelSerializer):
class Meta:
model = models.Listening
fields = ("id", "user", "track", "creation_date")

View File

@ -1,17 +1,36 @@
from rest_framework import mixins, permissions, viewsets
from rest_framework import mixins, viewsets
from rest_framework.permissions import IsAuthenticatedOrReadOnly
from funkwhale_api.activity import record
from funkwhale_api.common import fields, permissions
from . import models, serializers
class ListeningViewSet(
mixins.CreateModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
mixins.CreateModelMixin,
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
viewsets.GenericViewSet,
):
serializer_class = serializers.ListeningSerializer
queryset = models.Listening.objects.all()
permission_classes = [permissions.IsAuthenticated]
queryset = (
models.Listening.objects.all()
.select_related("track__artist", "track__album__artist", "user")
.prefetch_related("track__files")
)
permission_classes = [
permissions.ConditionalAuthentication,
permissions.OwnerPermission,
IsAuthenticatedOrReadOnly,
]
owner_checks = ["write"]
def get_serializer_class(self):
if self.request.method.lower() in ["head", "get", "options"]:
return serializers.ListeningSerializer
return serializers.ListeningWriteSerializer
def perform_create(self, serializer):
r = super().perform_create(serializer)
@ -20,7 +39,9 @@ class ListeningViewSet(
def get_queryset(self):
queryset = super().get_queryset()
return queryset.filter(user=self.request.user)
return queryset.filter(
fields.privacy_level_query(self.request.user, "user__privacy_level")
)
def get_serializer_context(self):
context = super().get_serializer_context()

View File

@ -6,19 +6,9 @@ from funkwhale_api.common import fields
from . import models
class ListenableMixin(filters.FilterSet):
listenable = filters.BooleanFilter(name="_", method="filter_listenable")
def filter_listenable(self, queryset, name, value):
queryset = queryset.annotate(files_count=Count("tracks__files"))
if value:
return queryset.filter(files_count__gt=0)
else:
return queryset.filter(files_count=0)
class ArtistFilter(ListenableMixin):
class ArtistFilter(filters.FilterSet):
q = fields.SearchFilter(search_fields=["name"])
listenable = filters.BooleanFilter(name="_", method="filter_listenable")
class Meta:
model = models.Artist
@ -27,6 +17,13 @@ class ArtistFilter(ListenableMixin):
"listenable": "exact",
}
def filter_listenable(self, queryset, name, value):
queryset = queryset.annotate(files_count=Count("albums__tracks__files"))
if value:
return queryset.filter(files_count__gt=0)
else:
return queryset.filter(files_count=0)
class TrackFilter(filters.FilterSet):
q = fields.SearchFilter(search_fields=["title", "album__title", "artist__name"])
@ -72,10 +69,17 @@ class ImportJobFilter(filters.FilterSet):
}
class AlbumFilter(ListenableMixin):
class AlbumFilter(filters.FilterSet):
listenable = filters.BooleanFilter(name="_", method="filter_listenable")
q = fields.SearchFilter(search_fields=["title", "artist__name" "source"])
class Meta:
model = models.Album
fields = ["listenable", "q", "artist"]
def filter_listenable(self, queryset, name, value):
queryset = queryset.annotate(files_count=Count("tracks__files"))
if value:
return queryset.filter(files_count__gt=0)
else:
return queryset.filter(files_count=0)

View File

@ -3,12 +3,19 @@ def load(model, *args, **kwargs):
return importer.load(*args, **kwargs)
EXCLUDE_VALIDATION = {"Track": ["artist"]}
class Importer(object):
def __init__(self, model):
self.model = model
def load(self, cleaned_data, raw_data, import_hooks):
mbid = cleaned_data.pop("mbid")
# let's validate data, just in case
instance = self.model(**cleaned_data)
exclude = EXCLUDE_VALIDATION.get(self.model.__name__, [])
instance.full_clean(exclude=["mbid", "uuid"] + exclude)
m = self.model.objects.update_or_create(mbid=mbid, defaults=cleaned_data)[0]
for hook in import_hooks:
hook(m, cleaned_data, raw_data)

View File

@ -1,4 +1,3 @@
import cacheops
from django.core.management.base import BaseCommand
from django.db import transaction
from django.db.models import Q
@ -24,7 +23,6 @@ class Command(BaseCommand):
self.fix_mimetypes(**options)
self.fix_file_data(**options)
self.fix_file_size(**options)
cacheops.invalidate_model(models.TrackFile)
@transaction.atomic
def fix_mimetypes(self, dry_run, **kwargs):

View File

@ -15,7 +15,9 @@ from django.dispatch import receiver
from django.urls import reverse
from django.utils import timezone
from taggit.managers import TaggableManager
from versatileimagefield.fields import VersatileImageField
from versatileimagefield.image_warmer import VersatileImageFieldWarmer
from funkwhale_api import downloader, musicbrainz
from funkwhale_api.federation import utils as federation_utils
@ -319,9 +321,10 @@ class Track(APIModelMixin):
"mbid": {"musicbrainz_field_name": "id"},
"title": {"musicbrainz_field_name": "title"},
"artist": {
# we use the artist from the release to avoid #237
"musicbrainz_field_name": "release-list",
"converter": get_artist,
"musicbrainz_field_name": "artist-credit",
"converter": lambda v: Artist.get_or_create_from_api(
mbid=v[0]["artist"]["id"]
)[0],
},
"album": {"musicbrainz_field_name": "release-list", "converter": import_album},
}
@ -389,19 +392,37 @@ class Track(APIModelMixin):
tracks = [t for m in data["release"]["medium-list"] for t in m["track-list"]]
track_data = None
for track in tracks:
if track["recording"]["id"] == mbid:
if track["recording"]["id"] == str(mbid):
track_data = track
break
if not track_data:
raise ValueError("No track found matching this ID")
track_artist_mbid = None
for ac in track_data["recording"]["artist-credit"]:
try:
ac_mbid = ac["artist"]["id"]
except TypeError:
# it's probably a string, like "feat."
continue
if ac_mbid == str(album.artist.mbid):
continue
track_artist_mbid = ac_mbid
break
track_artist_mbid = track_artist_mbid or album.artist.mbid
if track_artist_mbid == str(album.artist.mbid):
track_artist = album.artist
else:
track_artist = Artist.get_or_create_from_api(track_artist_mbid)[0]
return cls.objects.update_or_create(
mbid=mbid,
defaults={
"position": int(track["position"]),
"title": track["recording"]["title"],
"album": album,
"artist": album.artist,
"artist": track_artist,
},
)
@ -622,3 +643,13 @@ def update_request_status(sender, instance, created, **kwargs):
# let's mark the request as imported since the import is over
instance.import_request.status = "imported"
return instance.import_request.save(update_fields=["status"])
@receiver(models.signals.post_save, sender=Album)
def warm_album_covers(sender, instance, **kwargs):
if not instance.cover:
return
album_covers_warmer = VersatileImageFieldWarmer(
instance_or_queryset=instance, rendition_key_set="square", image_attr="cover"
)
num_created, failed_to_create = album_covers_warmer.warm()

View File

@ -1,6 +1,7 @@
from django.db.models import Q
from rest_framework import serializers
from taggit.models import Tag
from versatileimagefield.serializers import VersatileImageFieldSerializer
from funkwhale_api.activity import serializers as activity_serializers
from funkwhale_api.users.serializers import UserBasicSerializer
@ -8,8 +9,12 @@ from funkwhale_api.users.serializers import UserBasicSerializer
from . import models, tasks
cover_field = VersatileImageFieldSerializer(allow_null=True, sizes="square")
class ArtistAlbumSerializer(serializers.ModelSerializer):
tracks_count = serializers.SerializerMethodField()
cover = cover_field
class Meta:
model = models.Album
@ -60,8 +65,15 @@ class TrackFileSerializer(serializers.ModelSerializer):
return url
class ArtistSimpleSerializer(serializers.ModelSerializer):
class Meta:
model = models.Artist
fields = ("id", "mbid", "name", "creation_date")
class AlbumTrackSerializer(serializers.ModelSerializer):
files = TrackFileSerializer(many=True, read_only=True)
artist = ArtistSimpleSerializer(read_only=True)
class Meta:
model = models.Track
@ -77,15 +89,10 @@ class AlbumTrackSerializer(serializers.ModelSerializer):
)
class ArtistSimpleSerializer(serializers.ModelSerializer):
class Meta:
model = models.Artist
fields = ("id", "mbid", "name", "creation_date")
class AlbumSerializer(serializers.ModelSerializer):
tracks = serializers.SerializerMethodField()
artist = ArtistSimpleSerializer(read_only=True)
cover = cover_field
class Meta:
model = models.Album
@ -110,6 +117,7 @@ class AlbumSerializer(serializers.ModelSerializer):
class TrackAlbumSerializer(serializers.ModelSerializer):
artist = ArtistSimpleSerializer(read_only=True)
cover = cover_field
class Meta:
model = models.Album
@ -155,6 +163,8 @@ class TagSerializer(serializers.ModelSerializer):
class SimpleAlbumSerializer(serializers.ModelSerializer):
cover = cover_field
class Meta:
model = models.Album
fields = ("id", "mbid", "title", "release_date", "cover")

View File

@ -1,47 +1,9 @@
import mimetypes
import re
import magic
import mutagen
from django.db.models import Q
def normalize_query(
query_string,
findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
normspace=re.compile(r"\s{2,}").sub,
):
""" Splits the query string in invidual keywords, getting rid of unecessary spaces
and grouping quoted words together.
Example:
>>> normalize_query(' some random words "with quotes " and spaces')
['some', 'random', 'words', 'with quotes', 'and', 'spaces']
"""
return [normspace(" ", (t[0] or t[1]).strip()) for t in findterms(query_string)]
def get_query(query_string, search_fields):
""" Returns a query, that is a combination of Q objects. That combination
aims to search keywords within a model by testing the given search fields.
"""
query = None # Query to search for every search term
terms = normalize_query(query_string)
for term in terms:
or_query = None # Query to search for a given term in each field
for field_name in search_fields:
q = Q(**{"%s__icontains" % field_name: term})
if or_query is None:
or_query = q
else:
or_query = or_query | q
if query is None:
query = or_query
else:
query = query & or_query
return query
from funkwhale_api.common.search import normalize_query, get_query # noqa
def guess_mimetype(f):

View File

@ -1,3 +1,4 @@
from django.db.models import Count
from django_filters import rest_framework as filters
from funkwhale_api.music import utils
@ -7,10 +8,23 @@ from . import models
class PlaylistFilter(filters.FilterSet):
q = filters.CharFilter(name="_", method="filter_q")
listenable = filters.BooleanFilter(name="_", method="filter_listenable")
class Meta:
model = models.Playlist
fields = {"user": ["exact"], "name": ["exact", "icontains"], "q": "exact"}
fields = {
"user": ["exact"],
"name": ["exact", "icontains"],
"q": "exact",
"listenable": "exact",
}
def filter_listenable(self, queryset, name, value):
queryset = queryset.annotate(plts_count=Count("playlist_tracks"))
if value:
return queryset.filter(plts_count__gt=0)
else:
return queryset.filter(plts_count=0)
def filter_q(self, queryset, name, value):
query = utils.get_query(value, ["name", "user__username"])

View File

@ -3,12 +3,41 @@ from django.utils import timezone
from rest_framework import exceptions
from funkwhale_api.common import fields, preferences
from funkwhale_api.music import models as music_models
class PlaylistQuerySet(models.QuerySet):
def with_tracks_count(self):
return self.annotate(_tracks_count=models.Count("playlist_tracks"))
def with_duration(self):
return self.annotate(
duration=models.Sum("playlist_tracks__track__files__duration")
)
def with_covers(self):
album_prefetch = models.Prefetch(
"album", queryset=music_models.Album.objects.only("cover")
)
track_prefetch = models.Prefetch(
"track",
queryset=music_models.Track.objects.prefetch_related(album_prefetch).only(
"id", "album_id"
),
)
plt_prefetch = models.Prefetch(
"playlist_tracks",
queryset=PlaylistTrack.objects.all()
.exclude(track__album__cover=None)
.exclude(track__album__cover="")
.order_by("index")
.only("id", "playlist_id", "track_id")
.prefetch_related(track_prefetch),
to_attr="plts_for_cover",
)
return self.prefetch_related(plt_prefetch)
class Playlist(models.Model):
name = models.CharField(max_length=50)

View File

@ -65,6 +65,8 @@ class PlaylistTrackWriteSerializer(serializers.ModelSerializer):
class PlaylistSerializer(serializers.ModelSerializer):
tracks_count = serializers.SerializerMethodField(read_only=True)
duration = serializers.SerializerMethodField(read_only=True)
album_covers = serializers.SerializerMethodField(read_only=True)
user = UserBasicSerializer(read_only=True)
class Meta:
@ -72,11 +74,13 @@ class PlaylistSerializer(serializers.ModelSerializer):
fields = (
"id",
"name",
"tracks_count",
"user",
"modification_date",
"creation_date",
"privacy_level",
"tracks_count",
"album_covers",
"duration",
)
read_only_fields = ["id", "modification_date", "creation_date"]
@ -87,6 +91,36 @@ class PlaylistSerializer(serializers.ModelSerializer):
# no annotation?
return obj.playlist_tracks.count()
def get_duration(self, obj):
try:
return obj.duration
except AttributeError:
# no annotation?
return 0
def get_album_covers(self, obj):
try:
plts = obj.plts_for_cover
except AttributeError:
return []
covers = []
max_covers = 5
for plt in plts:
url = plt.track.album.cover.crop["200x200"].url
if url in covers:
continue
covers.append(url)
if len(covers) >= max_covers:
break
full_urls = []
for url in covers:
if "request" in self.context:
url = self.context["request"].build_absolute_uri(url)
full_urls.append(url)
return full_urls
class PlaylistAddManySerializer(serializers.Serializer):
tracks = serializers.PrimaryKeyRelatedField(

View File

@ -24,6 +24,8 @@ class PlaylistViewSet(
models.Playlist.objects.all()
.select_related("user")
.annotate(tracks_count=Count("playlist_tracks"))
.with_covers()
.with_duration()
)
permission_classes = [
permissions.ConditionalAuthentication,

View File

@ -82,10 +82,31 @@ class Command(BaseCommand):
try:
for import_path in options["path"]:
matching += glob.glob(import_path, **glob_kwargs)
matching = sorted(list(set(matching)))
raw_matching = sorted(list(set(matching)))
except TypeError:
raise Exception("You need Python 3.5 to use the --recursive flag")
matching = []
for m in raw_matching:
# In some situations, the path is encoded incorrectly on the filesystem
# so we filter out faulty paths and display a warning to the user.
# see https://code.eliotberriot.com/funkwhale/funkwhale/issues/138
try:
m.encode("utf-8")
matching.append(m)
except UnicodeEncodeError:
try:
previous = matching[-1]
except IndexError:
previous = None
self.stderr.write(
self.style.WARNING(
"[warning] Ignoring undecodable path. Previous ok file was {}".format(
previous
)
)
)
if options["in_place"]:
self.stdout.write(
"Checking imported paths against settings.MUSIC_DIRECTORY_PATH"

View File

@ -19,7 +19,7 @@ def authenticate(username, password):
password = password.replace("enc:", "", 1)
password = binascii.unhexlify(password).decode("utf-8")
user = User.objects.get(
username=username, is_active=True, subsonic_api_token=password
username__iexact=username, is_active=True, subsonic_api_token=password
)
except (User.DoesNotExist, binascii.Error):
raise exceptions.AuthenticationFailed("Wrong username or password.")

View File

@ -56,7 +56,10 @@ class UserAdmin(AuthUserAdmin):
fieldsets = (
(None, {"fields": ("username", "password", "privacy_level")}),
(_("Personal info"), {"fields": ("first_name", "last_name", "email")}),
(
_("Personal info"),
{"fields": ("first_name", "last_name", "email", "avatar")},
),
(
_("Permissions"),
{

View File

@ -4,6 +4,8 @@ from django.utils import timezone
from funkwhale_api.factories import ManyToManyFromList, registry
from . import models
@registry.register
class GroupFactory(factory.django.DjangoModelFactory):
@ -47,6 +49,7 @@ class UserFactory(factory.django.DjangoModelFactory):
password = factory.PostGenerationMethodCall("set_password", "test")
subsonic_api_token = None
groups = ManyToManyFromList("groups")
avatar = factory.django.ImageField()
class Meta:
model = "users.User"
@ -71,6 +74,14 @@ class UserFactory(factory.django.DjangoModelFactory):
# A list of permissions were passed in, use them
self.user_permissions.add(*perms)
@factory.post_generation
def with_actor(self, create, extracted, **kwargs):
if not create or not extracted:
return
self.actor = models.create_actor(self)
self.save(update_fields=["actor"])
return self.actor
@registry.register(name="users.SuperUser")
class SuperUserFactory(UserFactory):

View File

@ -0,0 +1,20 @@
# Generated by Django 2.0.6 on 2018-07-10 20:09
from django.db import migrations, models
import funkwhale_api.common.utils
import funkwhale_api.common.validators
class Migration(migrations.Migration):
dependencies = [
('users', '0009_auto_20180619_2024'),
]
operations = [
migrations.AddField(
model_name='user',
name='avatar',
field=models.ImageField(blank=True, max_length=150, null=True, upload_to=funkwhale_api.common.utils.ChunkedPath('users/avatars'), validators=[funkwhale_api.common.validators.ImageDimensionsValidator(max_height=400, max_width=400, min_height=50, min_width=50)]),
),
]

View File

@ -0,0 +1,28 @@
# Generated by Django 2.0.7 on 2018-07-21 13:17
from django.db import migrations, models
import django.db.models.deletion
import funkwhale_api.common.utils
import funkwhale_api.common.validators
import versatileimagefield.fields
class Migration(migrations.Migration):
dependencies = [
('federation', '0006_auto_20180521_1702'),
('users', '0010_user_avatar'),
]
operations = [
migrations.AddField(
model_name='user',
name='actor',
field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='user', to='federation.Actor'),
),
migrations.AlterField(
model_name='user',
name='avatar',
field=versatileimagefield.fields.VersatileImageField(blank=True, max_length=150, null=True, upload_to=funkwhale_api.common.utils.ChunkedPath('users/avatars', preserve_file_name=False), validators=[funkwhale_api.common.validators.ImageDimensionsValidator(min_height=50, min_width=50), funkwhale_api.common.validators.FileValidator(allowed_extensions=['png', 'jpg', 'jpeg', 'gif'], max_size=2097152)]),
),
]

View File

@ -11,12 +11,21 @@ import uuid
from django.conf import settings
from django.contrib.auth.models import AbstractUser
from django.db import models
from django.dispatch import receiver
from django.urls import reverse
from django.utils import timezone
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from versatileimagefield.fields import VersatileImageField
from versatileimagefield.image_warmer import VersatileImageFieldWarmer
from funkwhale_api.common import fields, preferences
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common import validators as common_validators
from funkwhale_api.federation import keys
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import utils as federation_utils
def get_token():
@ -39,6 +48,9 @@ PERMISSIONS_CONFIGURATION = {
PERMISSIONS = sorted(PERMISSIONS_CONFIGURATION.keys())
get_file_path = common_utils.ChunkedPath("users/avatars", preserve_file_name=False)
@python_2_unicode_compatible
class User(AbstractUser):
@ -88,6 +100,26 @@ class User(AbstractUser):
blank=True,
on_delete=models.SET_NULL,
)
avatar = VersatileImageField(
upload_to=get_file_path,
null=True,
blank=True,
max_length=150,
validators=[
common_validators.ImageDimensionsValidator(min_width=50, min_height=50),
common_validators.FileValidator(
allowed_extensions=["png", "jpg", "jpeg", "gif"],
max_size=1024 * 1024 * 2,
),
],
)
actor = models.OneToOneField(
"federation.Actor",
related_name="user",
on_delete=models.SET_NULL,
null=True,
blank=True,
)
def __str__(self):
return self.username
@ -185,3 +217,41 @@ class Invitation(models.Model):
)
return super().save(**kwargs)
def create_actor(user):
username = user.username
private, public = keys.get_key_pair()
args = {
"preferred_username": username,
"domain": settings.FEDERATION_HOSTNAME,
"type": "Person",
"name": username,
"manually_approves_followers": False,
"url": federation_utils.full_url(
reverse("federation:actors-detail", kwargs={"user__username": username})
),
"shared_inbox_url": federation_utils.full_url(
reverse("federation:actors-inbox", kwargs={"user__username": username})
),
"inbox_url": federation_utils.full_url(
reverse("federation:actors-inbox", kwargs={"user__username": username})
),
"outbox_url": federation_utils.full_url(
reverse("federation:actors-outbox", kwargs={"user__username": username})
),
}
args["private_key"] = private.decode("utf-8")
args["public_key"] = public.decode("utf-8")
return federation_models.Actor.objects.create(**args)
@receiver(models.signals.post_save, sender=User)
def warm_user_avatar(sender, instance, **kwargs):
if not instance.avatar:
return
user_avatar_warmer = VersatileImageFieldWarmer(
instance_or_queryset=instance, rendition_key_set="square", image_attr="avatar"
)
num_created, failed_to_create = user_avatar_warmer.warm()

View File

@ -1,13 +1,33 @@
import re
from django.conf import settings
from django.core import validators
from django.utils.deconstruct import deconstructible
from django.utils.translation import gettext_lazy as _
from rest_auth.serializers import PasswordResetSerializer as PRS
from rest_auth.registration.serializers import RegisterSerializer as RS
from rest_framework import serializers
from versatileimagefield.serializers import VersatileImageFieldSerializer
from funkwhale_api.activity import serializers as activity_serializers
from . import models
@deconstructible
class ASCIIUsernameValidator(validators.RegexValidator):
regex = r"^[\w]+$"
message = _(
"Enter a valid username. This value may contain only English letters, "
"numbers, and _ characters."
)
flags = re.ASCII
username_validators = [ASCIIUsernameValidator()]
class RegisterSerializer(RS):
invitation = serializers.CharField(
required=False, allow_null=True, allow_blank=True
@ -27,6 +47,9 @@ class RegisterSerializer(RS):
if self.validated_data.get("invitation"):
user.invitation = self.validated_data.get("invitation")
user.save(update_fields=["invitation"])
user.actor = models.create_actor(user)
user.save(update_fields=["actor"])
return user
@ -43,21 +66,29 @@ class UserActivitySerializer(activity_serializers.ModelSerializer):
return "Person"
avatar_field = VersatileImageFieldSerializer(allow_null=True, sizes="square")
class UserBasicSerializer(serializers.ModelSerializer):
avatar = avatar_field
class Meta:
model = models.User
fields = ["id", "username", "name", "date_joined"]
fields = ["id", "username", "name", "date_joined", "avatar"]
class UserWriteSerializer(serializers.ModelSerializer):
avatar = avatar_field
class Meta:
model = models.User
fields = ["name", "privacy_level"]
fields = ["name", "privacy_level", "avatar"]
class UserReadSerializer(serializers.ModelSerializer):
permissions = serializers.SerializerMethodField()
avatar = avatar_field
class Meta:
model = models.User
@ -71,6 +102,7 @@ class UserReadSerializer(serializers.ModelSerializer):
"permissions",
"date_joined",
"privacy_level",
"avatar",
]
def get_permissions(self, o):

6
api/requirements.pac Normal file
View File

@ -0,0 +1,6 @@
curl
file
ffmpeg
libjpeg-turbo
libpqxx
python

View File

@ -37,7 +37,7 @@ oauth2client<4
google-api-python-client>=1.6,<1.7
arrow>=0.12,<0.13
persisting-theory>=0.2,<0.3
django-versatileimagefield>=1.8,<1.9
django-versatileimagefield>=1.9,<1.10
django-filter>=1.1,<1.2
django-rest-auth>=0.9,<0.10
beautifulsoup4>=4.6,<4.7
@ -58,7 +58,6 @@ python-magic==0.4.15
ffmpeg-python==0.1.10
channels>=2,<2.1
channels_redis>=2.1,<2.2
django-cacheops>=4,<4.1
daphne==2.0.4
cryptography>=2,<3

View File

@ -18,5 +18,4 @@ env =
EMAIL_CONFIG=consolemail://
CELERY_BROKER_URL=memory://
CELERY_TASK_ALWAYS_EAGER=True
CACHEOPS_ENABLED=False
FEDERATION_HOSTNAME=test.federation

View File

@ -0,0 +1,83 @@
import pytest
from django.db.models import Q
from funkwhale_api.common import search
from funkwhale_api.music import models as music_models
@pytest.mark.parametrize(
"query,expected",
[
("", [music_models.Album, music_models.Artist]),
("is:album", [music_models.Album]),
("is:artist is:album", [music_models.Artist, music_models.Album]),
],
)
def test_search_config_is(query, expected):
s = search.SearchConfig(
types=[("album", music_models.Album), ("artist", music_models.Artist)]
)
cleaned = s.clean(query)
assert cleaned["types"] == expected
@pytest.mark.parametrize(
"query,expected",
[
("", None),
("hello world", search.get_query("hello world", ["f1", "f2", "f3"])),
("hello in:field2", search.get_query("hello", ["f2"])),
("hello in:field1,field2", search.get_query("hello", ["f1", "f2"])),
],
)
def test_search_config_query(query, expected):
s = search.SearchConfig(
search_fields={
"field1": {"to": "f1"},
"field2": {"to": "f2"},
"field3": {"to": "f3"},
}
)
cleaned = s.clean(query)
assert cleaned["search_query"] == expected
@pytest.mark.parametrize(
"query,expected",
[
("", None),
("status:pending", Q(status="pending")),
('user:"silent bob"', Q(user__username__iexact="silent bob")),
(
"user:me status:pending",
Q(user__username__iexact="me") & Q(status="pending"),
),
],
)
def test_search_config_filter(query, expected):
s = search.SearchConfig(
filter_fields={
"user": {"to": "user__username__iexact"},
"status": {"to": "status"},
}
)
cleaned = s.clean(query)
assert cleaned["filter_query"] == expected
def test_apply():
cleaned = {
"filter_query": Q(batch__submitted_by__username__iexact="me"),
"search_query": Q(source="test"),
}
result = search.apply(music_models.ImportJob.objects.all(), cleaned)
assert str(result.query) == str(
music_models.ImportJob.objects.filter(
Q(batch__submitted_by__username__iexact="me"), Q(source="test")
).query
)

View File

@ -1,4 +1,7 @@
import datetime
import io
import PIL
import random
import shutil
import tempfile
@ -258,3 +261,14 @@ def now(mocker):
now = timezone.now()
mocker.patch("django.utils.timezone.now", return_value=now)
return now
@pytest.fixture()
def avatar():
i = PIL.Image.new("RGBA", (400, 400), random.choice(["red", "blue", "yellow"]))
f = io.BytesIO()
i.save(f, "png")
f.name = "avatar.png"
f.seek(0)
yield f
f.close()

View File

@ -4,6 +4,8 @@ import pytest
from django.urls import reverse
from funkwhale_api.favorites.models import TrackFavorite
from funkwhale_api.music import serializers as music_serializers
from funkwhale_api.users import serializers as users_serializers
def test_user_can_add_favorite(factories):
@ -15,21 +17,26 @@ def test_user_can_add_favorite(factories):
assert f.user == user
def test_user_can_get_his_favorites(factories, logged_in_client, client):
def test_user_can_get_his_favorites(api_request, factories, logged_in_client, client):
r = api_request.get("/")
favorite = factories["favorites.TrackFavorite"](user=logged_in_client.user)
factories["favorites.TrackFavorite"]()
url = reverse("api:v1:favorites:tracks-list")
response = logged_in_client.get(url)
response = logged_in_client.get(url, {"user": logged_in_client.user.pk})
expected = [
{
"track": favorite.track.pk,
"user": users_serializers.UserBasicSerializer(
favorite.user, context={"request": r}
).data,
"track": music_serializers.TrackSerializer(
favorite.track, context={"request": r}
).data,
"id": favorite.id,
"creation_date": favorite.creation_date.isoformat().replace("+00:00", "Z"),
}
]
parsed_json = json.loads(response.content.decode("utf-8"))
assert expected == parsed_json["results"]
assert response.status_code == 200
assert response.data["results"] == expected
def test_user_can_add_favorite_via_api(factories, logged_in_client, activity_muted):

View File

@ -0,0 +1,13 @@
import pytest
from django.urls import reverse
@pytest.mark.parametrize("level", ["instance", "me", "followers"])
def test_privacy_filter(preferences, level, factories, api_client):
preferences["common__api_authentication_required"] = False
factories["favorites.TrackFavorite"](user__privacy_level=level)
url = reverse("api:v1:favorites:tracks-list")
response = api_client.get(url)
assert response.status_code == 200
assert response.data["count"] == 0

View File

@ -681,3 +681,56 @@ def test_tapi_library_track_serializer_import_pending(factories):
serializer = serializers.APILibraryTrackSerializer(lt)
assert serializer.get_status(lt) == "import_pending"
def test_local_actor_serializer_to_ap(factories):
expected = {
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{},
],
"id": "https://test.federation/user",
"type": "Person",
"following": "https://test.federation/user/following",
"followers": "https://test.federation/user/followers",
"inbox": "https://test.federation/user/inbox",
"outbox": "https://test.federation/user/outbox",
"preferredUsername": "user",
"name": "Real User",
"summary": "Hello world",
"manuallyApprovesFollowers": False,
"publicKey": {
"id": "https://test.federation/user#main-key",
"owner": "https://test.federation/user",
"publicKeyPem": "yolo",
},
"endpoints": {"sharedInbox": "https://test.federation/inbox"},
}
ac = models.Actor.objects.create(
url=expected["id"],
inbox_url=expected["inbox"],
outbox_url=expected["outbox"],
shared_inbox_url=expected["endpoints"]["sharedInbox"],
followers_url=expected["followers"],
following_url=expected["following"],
public_key=expected["publicKey"]["publicKeyPem"],
preferred_username=expected["preferredUsername"],
name=expected["name"],
domain="test.federation",
summary=expected["summary"],
type="Person",
manually_approves_followers=False,
)
user = factories["users.User"]()
user.actor = ac
user.save()
ac.refresh_from_db()
expected["icon"] = {
"type": "Image",
"mediaType": "image/jpeg",
"url": utils.full_url(user.avatar.crop["400x400"].url),
}
serializer = serializers.ActorSerializer(ac)
assert serializer.data == expected

View File

@ -1,4 +1,7 @@
import cryptography.exceptions
import datetime
from django.utils.http import http_date
from django import forms
import pytest
from funkwhale_api.federation import keys, signing
@ -36,6 +39,20 @@ def test_verify_fails_with_wrong_key(nodb_factories):
signing.verify(prepared_request, wrong_public)
def test_verify_fails_with_wrong_date(nodb_factories, now):
too_old = now - datetime.timedelta(seconds=31)
too_old = http_date(too_old.timestamp())
private, public = nodb_factories["federation.KeyPair"]()
auth = nodb_factories["federation.SignatureAuth"](key=private)
request = nodb_factories["federation.SignedRequest"](
auth=auth, headers={"Date": too_old}
)
prepared_request = request.prepare()
with pytest.raises(forms.ValidationError):
signing.verify(prepared_request, public)
def test_can_verify_django_request(factories, fake_request):
private_key, public_key = keys.get_key_pair()
signed_request = factories["federation.SignedRequest"](
@ -95,14 +112,18 @@ def test_can_verify_django_request_digest_failure(factories, fake_request):
signing.verify_django(django_request, public_key)
def test_can_verify_django_request_failure(factories, fake_request):
def test_can_verify_django_request_failure(factories, fake_request, now):
private_key, public_key = keys.get_key_pair()
signed_request = factories["federation.SignedRequest"](
auth__key=private_key, auth__headers=["date"]
)
prepared = signed_request.prepare()
django_request = fake_request.get(
"/", **{"HTTP_DATE": "Wrong", "HTTP_SIGNATURE": prepared.headers["signature"]}
"/",
**{
"HTTP_DATE": http_date((now + datetime.timedelta(seconds=31)).timestamp()),
"HTTP_SIGNATURE": prepared.headers["signature"],
}
)
with pytest.raises(cryptography.exceptions.InvalidSignature):
with pytest.raises(forms.ValidationError):
signing.verify_django(django_request, public_key)

View File

@ -417,3 +417,28 @@ def test_library_track_action_import(factories, superuser_api_client, mocker):
for i, job in enumerate(batch.jobs.all()):
assert job.library_track == imported_lts[i]
mocked_run.assert_called_once_with(import_batch_id=batch.pk)
def test_local_actor_detail(factories, api_client):
user = factories["users.User"](with_actor=True)
url = reverse("federation:actors-detail", kwargs={"user__username": user.username})
serializer = serializers.ActorSerializer(user.actor)
response = api_client.get(url)
assert response.status_code == 200
assert response.data == serializer.data
def test_wellknown_webfinger_local(factories, api_client, settings, mocker):
user = factories["users.User"](with_actor=True)
url = reverse("federation:well-known-webfinger")
response = api_client.get(
url,
data={"resource": "acct:{}".format(user.actor.webfinger_subject)},
HTTP_ACCEPT="application/jrd+json",
)
serializer = serializers.ActorWebfingerSerializer(user.actor)
assert response.status_code == 200
assert response["Content-Type"] == "application/jrd+json"
assert response.data == serializer.data

View File

@ -0,0 +1,13 @@
import pytest
from django.urls import reverse
@pytest.mark.parametrize("level", ["instance", "me", "followers"])
def test_privacy_filter(preferences, level, factories, api_client):
preferences["common__api_authentication_required"] = False
factories["history.Listening"](user__privacy_level=level)
url = reverse("api:v1:history:listenings-list")
response = api_client.get(url)
assert response.status_code == 200
assert response.data["count"] == 0

View File

@ -1,10 +1,15 @@
import json
import os
import pytest
import uuid
from django import forms
from django.urls import reverse
from funkwhale_api.federation import actors
from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.music import importers
from funkwhale_api.music import models
from funkwhale_api.music import tasks
DATA_DIR = os.path.dirname(os.path.abspath(__file__))
@ -237,3 +242,9 @@ def test__do_import_in_place_mbid(factories, tmpfile):
assert bool(tf.audio_file) is False
assert tf.source == "file://{}".format(path)
assert tf.mimetype == "audio/ogg"
def test_importer_cleans():
importer = importers.Importer(models.Artist)
with pytest.raises(forms.ValidationError):
importer.load({"name": "", "mbid": uuid.uuid4()}, {}, [])

View File

@ -44,6 +44,7 @@ def test_import_album_stores_release_group(factories):
def test_import_track_from_release(factories, mocker):
album = factories["music.Album"](mbid="430347cb-0879-3113-9fde-c75b658c298e")
artist = factories["music.Artist"](mbid="a5211c65-2465-406b-93ec-213588869dc1")
album_data = {
"release": {
"id": album.mbid,
@ -64,6 +65,9 @@ def test_import_track_from_release(factories, mocker):
"id": "2109e376-132b-40ad-b993-2bb6812e19d4",
"title": "Teen Age Riot",
"length": "417973",
"artist-credit": [
{"artist": {"id": artist.mbid, "name": artist.name}}
],
},
"track_or_recording_length": "417973",
}
@ -84,10 +88,66 @@ def test_import_track_from_release(factories, mocker):
assert track.title == track_data["recording"]["title"]
assert track.mbid == track_data["recording"]["id"]
assert track.album == album
assert track.artist == album.artist
assert track.artist == artist
assert track.position == int(track_data["position"])
def test_import_track_with_different_artist_than_release(factories, mocker):
album = factories["music.Album"](mbid="430347cb-0879-3113-9fde-c75b658c298e")
recording_data = {
"recording": {
"id": "94ab07eb-bdf3-4155-b471-ba1dc85108bf",
"title": "Flaming Red Hair",
"length": "159000",
"artist-credit": [
{
"artist": {
"id": "a5211c65-2465-406b-93ec-213588869dc1",
"name": "Plan 9",
"sort-name": "Plan 9",
"disambiguation": "New Zealand group",
}
}
],
"release-list": [
{
"id": album.mbid,
"title": "The Lord of the Rings: The Fellowship of the Ring - The Complete Recordings",
"status": "Official",
"quality": "normal",
"text-representation": {"language": "eng", "script": "Latn"},
"artist-credit": [
{
"artist": {
"id": "9b58672a-e68e-4972-956e-a8985a165a1f",
"name": "Howard Shore",
"sort-name": "Shore, Howard",
}
}
],
"date": "2005-12-13",
"country": "US",
"release-event-count": 1,
"barcode": "093624945420",
"artist-credit-phrase": "Howard Shore",
}
],
"release-count": 3,
"artist-credit-phrase": "Plan 9",
}
}
artist = factories["music.Artist"](mbid="a5211c65-2465-406b-93ec-213588869dc1")
mocker.patch(
"funkwhale_api.musicbrainz.api.recordings.get", return_value=recording_data
)
track = models.Track.get_or_create_from_api(recording_data["recording"]["id"])[0]
assert track.title == recording_data["recording"]["title"]
assert track.mbid == recording_data["recording"]["id"]
assert track.album == album
assert track.artist == artist
def test_import_job_is_bound_to_track_file(factories, mocker):
track = factories["music.Track"]()
job = factories["music.ImportJob"](mbid=track.mbid)

View File

@ -12,7 +12,12 @@ def test_artist_album_serializer(factories, to_api_date):
"artist": album.artist.id,
"creation_date": to_api_date(album.creation_date),
"tracks_count": 1,
"cover": album.cover.url,
"cover": {
"original": album.cover.url,
"square_crop": album.cover.crop["400x400"].url,
"medium_square_crop": album.cover.crop["200x200"].url,
"small_square_crop": album.cover.crop["50x50"].url,
},
"release_date": to_api_date(album.release_date),
}
serializer = serializers.ArtistAlbumSerializer(album)
@ -43,7 +48,7 @@ def test_album_track_serializer(factories, to_api_date):
expected = {
"id": track.id,
"artist": track.artist.id,
"artist": serializers.ArtistSimpleSerializer(track.artist).data,
"album": track.album.id,
"mbid": str(track.mbid),
"title": track.title,
@ -83,7 +88,12 @@ def test_album_serializer(factories, to_api_date):
"title": album.title,
"artist": serializers.ArtistSimpleSerializer(album.artist).data,
"creation_date": to_api_date(album.creation_date),
"cover": album.cover.url,
"cover": {
"original": album.cover.url,
"square_crop": album.cover.crop["400x400"].url,
"medium_square_crop": album.cover.crop["200x200"].url,
"small_square_crop": album.cover.crop["50x50"].url,
},
"release_date": to_api_date(album.release_date),
"tracks": serializers.AlbumTrackSerializer([track2, track1], many=True).data,
}

View File

@ -63,3 +63,40 @@ def test_update_insert_is_called_when_index_is_provided(factories, mocker):
insert.assert_called_once_with(playlist, plt, 0)
assert plt.index == 0
assert first.index == 1
def test_playlist_serializer_include_covers(factories, api_request):
playlist = factories["playlists.Playlist"]()
t1 = factories["music.Track"]()
t2 = factories["music.Track"]()
t3 = factories["music.Track"](album__cover=None)
t4 = factories["music.Track"]()
t5 = factories["music.Track"]()
t6 = factories["music.Track"]()
t7 = factories["music.Track"]()
playlist.insert_many([t1, t2, t3, t4, t5, t6, t7])
request = api_request.get("/")
qs = playlist.__class__.objects.with_covers().with_tracks_count()
expected = [
request.build_absolute_uri(t1.album.cover.crop["200x200"].url),
request.build_absolute_uri(t2.album.cover.crop["200x200"].url),
request.build_absolute_uri(t4.album.cover.crop["200x200"].url),
request.build_absolute_uri(t5.album.cover.crop["200x200"].url),
request.build_absolute_uri(t6.album.cover.crop["200x200"].url),
]
serializer = serializers.PlaylistSerializer(qs.get(), context={"request": request})
assert serializer.data["album_covers"] == expected
def test_playlist_serializer_include_duration(factories, api_request):
playlist = factories["playlists.Playlist"]()
tf1 = factories["music.TrackFile"](duration=15)
tf2 = factories["music.TrackFile"](duration=30)
playlist.insert_many([tf1.track, tf2.track])
qs = playlist.__class__.objects.with_duration().with_tracks_count()
serializer = serializers.PlaylistSerializer(qs.get())
assert serializer.data["duration"] == 45

View File

@ -63,3 +63,15 @@ def test_auth_with_inactive_users(api_request, factories):
authenticator = authentication.SubsonicAuthentication()
with pytest.raises(exceptions.AuthenticationFailed):
authenticator.authenticate(request)
def test_auth_case_insensitive(api_request, factories):
user = factories["users.User"](username="Hello")
user.subsonic_api_token = "password"
user.save()
request = api_request.get("/", {"u": "hello", "p": "password"})
authenticator = authentication.SubsonicAuthentication()
u, _ = authenticator.authenticate(request)
assert user == u

View File

@ -37,6 +37,7 @@ def test_can_create_track_from_file_metadata_no_mbid(db, mocker):
def test_can_create_track_from_file_metadata_mbid(factories, mocker):
album = factories["music.Album"]()
artist = factories["music.Artist"]()
mocker.patch(
"funkwhale_api.music.models.Album.get_or_create_from_api",
return_value=(album, True),
@ -55,6 +56,9 @@ def test_can_create_track_from_file_metadata_mbid(factories, mocker):
"recording": {
"id": "2109e376-132b-40ad-b993-2bb6812e19d4",
"title": "Teen Age Riot",
"artist-credit": [
{"artist": {"id": artist.mbid, "name": artist.name}}
],
},
}
],
@ -79,7 +83,7 @@ def test_can_create_track_from_file_metadata_mbid(factories, mocker):
assert track.mbid == track_data["recording"]["id"]
assert track.position == 4
assert track.album == album
assert track.artist == album.artist
assert track.artist == artist
def test_management_command_requires_a_valid_username(factories, mocker):

View File

@ -1,7 +1,10 @@
import datetime
import pytest
from django.urls import reverse
from funkwhale_api.users import models
from funkwhale_api.federation import utils as federation_utils
def test__str__(factories):
@ -126,4 +129,27 @@ def test_can_filter_closed_invitations(factories):
used = factories["users.User"](invited=True).invitation
assert models.Invitation.objects.count() == 3
assert list(models.Invitation.objects.open(False)) == [expired, used]
assert list(models.Invitation.objects.order_by("id").open(False)) == [expired, used]
def test_creating_actor_from_user(factories, settings):
user = factories["users.User"]()
actor = models.create_actor(user)
assert actor.preferred_username == user.username
assert actor.domain == settings.FEDERATION_HOSTNAME
assert actor.type == "Person"
assert actor.name == user.username
assert actor.manually_approves_followers is False
assert actor.url == federation_utils.full_url(
reverse("federation:actors-detail", kwargs={"user__username": user.username})
)
assert actor.shared_inbox_url == federation_utils.full_url(
reverse("federation:actors-inbox", kwargs={"user__username": user.username})
)
assert actor.inbox_url == federation_utils.full_url(
reverse("federation:actors-inbox", kwargs={"user__username": user.username})
)
assert actor.outbox_url == federation_utils.full_url(
reverse("federation:actors-outbox", kwargs={"user__username": user.username})
)

View File

@ -20,6 +20,22 @@ def test_can_create_user_via_api(preferences, api_client, db):
assert u.username == "test1"
@pytest.mark.parametrize("username", ["wrong.name", "wrong-name", "éaeu", "wrong name"])
def test_username_only_accepts_letters_and_underscores(
username, preferences, api_client, db
):
url = reverse("rest_register")
data = {
"username": username,
"email": "test1@test.com",
"password1": "testtest",
"password2": "testtest",
}
preferences["users__registration_enabled"] = True
response = api_client.post(url, data)
assert response.status_code == 400
def test_can_restrict_usernames(settings, preferences, db, api_client):
url = reverse("rest_register")
preferences["users__registration_enabled"] = True
@ -235,3 +251,39 @@ def test_user_cannot_patch_another_user(method, logged_in_api_client, factories)
response = handler(url, payload)
assert response.status_code == 403
def test_user_can_patch_their_own_avatar(logged_in_api_client, avatar):
user = logged_in_api_client.user
url = reverse("api:v1:users:users-detail", kwargs={"username": user.username})
content = avatar.read()
avatar.seek(0)
payload = {"avatar": avatar}
response = logged_in_api_client.patch(url, payload)
assert response.status_code == 200
user.refresh_from_db()
assert user.avatar.read() == content
def test_creating_user_creates_actor_as_well(
api_client, factories, mocker, preferences
):
actor = factories["federation.Actor"]()
url = reverse("rest_register")
data = {
"username": "test1",
"email": "test1@test.com",
"password1": "testtest",
"password2": "testtest",
}
preferences["users__registration_enabled"] = True
mocker.patch("funkwhale_api.users.models.create_actor", return_value=actor)
response = api_client.post(url, data)
assert response.status_code == 201
user = User.objects.get(username="test1")
assert user.actor == actor

61
demo/README.md Normal file
View File

@ -0,0 +1,61 @@
# Setup the demo
We assume you want to store the demo data in `/srv/funkwhale-demo`.
This setup requires Docker and docker-compose.
## Create the necessary directories
`mkdir /srv/funkwhale-demo`
## Get some music
You can use your own music (put it in `/usr/share/music`, this is the directory the demo will look into by default).
If you don't have any music, you can use the repository https://code.eliotberriot.com/funkwhale/catalog, which
requires Git LFS.
## Create an env file
Copy the `env.sample` file to ``/srv/funkwhale-demo/.env`.
Edit the file according to your needs.
## Copy the setup script
Copy the `setup.sh` script to ``/srv/funkwhale-demo/setup.sh`.
Ensure it's executable with `chmod +x setup.sh`.
## Setup your nginx vhost
Setup your reverse proxy for the demo as described in https://docs.funkwhale.audio/installation/index.html#nginx.
This is outside of the scope of this guide, as you will probably want some SSL certificates, however,
ensure you point the vhost configuration to the proper static files:
- `root` should point to `/srv/funkwhale-demo/demo/front/dist`
- `/media` and `/_protected/media` should point to `/srv/funkwhale-demo/demo/data/media/`
- `/staticfiles` should point to `/srv/funkwhale-demo/demo/data/static`
## Launch
Setup the demo:
```
cd /srv/funkwhale-demo
sudo ENV_FILE=/srv/funkwhale-demo/.env ./setup.sh
```
## Automate
You'll probaby want to reset the demo every now and then. You can do that
using a cronjob:
```
sudo crontab -e
# in the crontab, put this:
SHELL=/bin/bash
0 */3 * * * cd /srv/funkwhale-demo && ENV_FILE=/srv/funkwhale-demo/env ./setup.sh > /srv/funkwhale-demo/crontab.log 2>&1
```
This will reset and restart the demo every 3 hours.

View File

@ -1,21 +0,0 @@
#! /bin/bash
set -e
[ -z $1 ] && echo "Path to list file missing" && exit 1
echo "This will download tracks from zip archives listed in $1"
LIST_CONTENT=$(cat $1)
mkdir -p data/music
cd data/music
echo "Downloading files..."
echo "$LIST_CONTENT" | grep "^[^#;]" | xargs -n 1 curl -LO
echo "Unzipping archives..."
find . -name "*.zip" | while read filename; do
dirname="${filename%.*}"
mkdir $dirname
unzip -o -d "$dirname" "$filename";
done;
echo "Done!"

6
demo/env.sample Normal file
View File

@ -0,0 +1,6 @@
FUNKWHALE_URL=https://demo.funkwhale.audio/
DJANGO_ALLOWED_HOSTS=demo.funkwhale.audio
FUNKWHALE_API_PORT=5001
DJANGO_SECRET_KEY=demo
DATABASE_URL=postgresql://postgres@postgres/postgres
CACHE_URL=redis://redis:6379/0

View File

@ -1,8 +0,0 @@
# You can get a pretty list of open music here: https://archive.org/details/jamendo-albums
https://archive.org/compress/jamendo-069098/formats=OGG%20VORBIS&file=/jamendo-069098.zip
https://archive.org/compress/jamendo-001144/formats=OGG%20VORBIS&file=/jamendo-001144.zip
https://archive.org/compress/jamendo-027690/formats=OGG%20VORBIS&file=/jamendo-027690.zip
https://archive.org/compress/jamendo-001469/formats=OGG%20VORBIS&file=/jamendo-001469.zip
https://archive.org/compress/jamendo-106323/formats=OGG%20VORBIS&file=/jamendo-106323.zip
https://archive.org/compress/jamendo-071149/formats=OGG%20VORBIS&file=/jamendo-071149.zip
https://archive.org/compress/jamendo-085030/formats=OGG%20VORBIS&file=/jamendo-085030.zip

110
demo/setup.sh Normal file → Executable file
View File

@ -1,30 +1,114 @@
#!/bin/bash -eux
version="develop"
music_path="/usr/share/music"
demo_path="/srv/funkwhale-demo/demo"
version=${VERSION:-develop}
music_path=${MUSIC_PATH:-/usr/share/music}
demo_path=${DEMO_PATH:-/srv/funkwhale-demo/demo}
env_file=${ENV_FILE}
echo 'Cleaning everything...'
mkdir -p $demo_path
cd $demo_path
/usr/local/bin/docker-compose down -v || echo 'Nothing to stop'
rm -rf /srv/funkwhale-demo/demo/*
sudo rm -rf $demo_path/*
mkdir -p $demo_path
echo 'Downloading demo files...'
curl -L -o docker-compose.yml "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/docker-compose.yml"
curl -L -o .env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/env.prod.sample"
mkdir data/
cp -r $music_path data/music
curl -L -o front.zip "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/$version/download?job=build_front"
unzip front.zip
echo "FUNKWHALE_URL=https://demo.funkwhale.audio/" >> .env
echo "DJANGO_SECRET_KEY=demo" >> .env
echo "DJANGO_ALLOWED_HOSTS=demo.funkwhale.audio" >> .env
cat $env_file >> .env
echo "FUNKWHALE_VERSION=$version" >> .env
echo "FUNKWHALE_API_PORT=5001" >> .env
/usr/local/bin/docker-compose pull
echo "MUSIC_DIRECTORY_SERVE_PATH=$music_path" >> .env
echo "MUSIC_DIRECTORY_PATH=$music_path" >> .env
echo "MEDIA_ROOT=$demo_path/data/media/" >> .env
echo "STATIC_ROOT=$demo_path/data/static/" >> .env
# /usr/local/bin/docker-compose pull
/usr/local/bin/docker-compose up -d postgres redis
sleep 5
/usr/local/bin/docker-compose run --rm api demo/load-demo-data.sh
cat .env
cat <<EOF | /usr/local/bin/docker-compose run --rm api python manage.py shell -i python
import subprocess
subprocess.call("pip install factory-boy", shell=True)
from django.core.management import call_command
call_command("migrate", interactive=False)
from funkwhale_api.users.models import User
print("Creating dummy user")
u = User.objects.create(email="demo@demo.com", username="demo", is_staff=True, is_superuser=True, privacy_level="everyone")
u.set_password("demo")
u.subsonic_api_token = "demo"
u.save()
from funkwhale_api.common import preferences
manager = preferences.global_preferences_registry.manager()
manager['common__api_authentication_required'] = False
manager['federation__music_needs_approval'] = False
manager['instance__name'] = "Login: demo / password: demo"
paths = [
"$music_path/**/*.ogg",
"$music_path/**/*.mp3",
"$music_path/**/*.flac",
]
print(paths)
call_command("import_files", *paths, username="demo", recursive=True, interactive=False)
print('Creating some dummy data...')
import random
import datetime
from funkwhale_api.music.models import Album, Track
from funkwhale_api.history.factories import ListeningFactory
from funkwhale_api.favorites.factories import TrackFavorite as TrackFavoriteFactory
from funkwhale_api.users.factories import UserFactory
from funkwhale_api.playlists.factories import PlaylistFactory
users = UserFactory.create_batch(size=15, privacy_level="everyone")
available_tracks = list(Track.objects.all())
available_albums = list(Album.objects.all())
def get_random_datetime():
from django.utils import timezone
import datetime
import random
now = timezone.now()
return now - datetime.timedelta(seconds=random.randint(1, 3600 * 24 * 7))
print('Updating album dates to have random sorting...')
for album in available_albums:
album.creation_date = get_random_datetime()
album.save(update_fields=['creation_date'])
for i in range(30):
print('Creating playlist {}'.format(i))
playlist = PlaylistFactory(user=random.choice(users), privacy_level="everyone", creation_date=get_random_datetime())
tracks = set()
for i in range(random.randint(5, 35)):
tracks.add(random.choice(available_tracks))
playlist.insert_many(tracks)
for user in users:
for i in range(random.randint(5, 35)):
print('Adding favorite {} for user {}'.format(i, user.username))
try:
TrackFavoriteFactory(user=user, track=random.choice(available_tracks), creation_date=get_random_datetime())
except:
pass
for i in range(random.randint(5, 35)):
print('Adding listening {} for user {}'.format(i, user.username))
try:
ListeningFactory(user=user, track=random.choice(available_tracks), creation_date=get_random_datetime())
except:
pass
EOF
chmod 777 -R front
/usr/local/bin/docker-compose up -d

View File

@ -34,7 +34,7 @@ Define MUSIC_DIRECTORY_PATH /srv/funkwhale/data/music
# TLS
# Feel free to use your own configuration for SSL here or simply remove the
# lines and move the configuration to the previous server block if you
# don't want to run funkwhale behind https (this is not recommanded)
# don't want to run funkwhale behind https (this is not recommended)
# have a look here for let's encrypt configuration:
# https://certbot.eff.org/lets-encrypt/debianstretch-apache.html
SSLEngine on

View File

@ -35,8 +35,8 @@ services:
environment:
- C_FORCE_ROOT=true
volumes:
- ./data/music:/music:ro
- ./data/media:/app/funkwhale_api/media
- "${MUSIC_DIRECTORY_SERVE_PATH-/srv/funkwhale/data/music}:${MUSIC_DIRECTORY_PATH-/music}:ro"
- "${MEDIA_ROOT}:${MEDIA_ROOT}"
celerybeat:
restart: unless-stopped
@ -52,9 +52,9 @@ services:
image: funkwhale/funkwhale:${FUNKWHALE_VERSION:-latest}
env_file: .env
volumes:
- ./data/music:/music:ro
- ./data/media:/app/funkwhale_api/media
- ./data/static:/app/staticfiles
- "${MUSIC_DIRECTORY_SERVE_PATH-/srv/funkwhale/data/music}:${MUSIC_DIRECTORY_PATH-/music}:ro"
- "${MEDIA_ROOT}:${MEDIA_ROOT}"
- "${STATIC_ROOT}:${STATIC_ROOT}"
- ./front/dist:/frontend
ports:
- "${FUNKWHALE_API_IP:-127.0.0.1}:${FUNKWHALE_API_PORT:-5000}:5000"

View File

@ -10,14 +10,13 @@
# On non-docker setup **only**, you'll also have to tweak/uncomment those variables:
# - DATABASE_URL
# - CACHE_URL
# - STATIC_ROOT
# - MEDIA_ROOT
#
# You **don't** need to update those variables on pure docker setups.
#
# Additional options you may want to check:
# - MUSIC_DIRECTORY_PATH and MUSIC_DIRECTORY_SERVE_PATH if you plan to use
# in-place import
#
# Docker only
# -----------
@ -25,7 +24,6 @@
# (it will be interpolated in docker-compose file)
# You can comment or ignore this if you're not using docker
FUNKWHALE_VERSION=latest
MUSIC_DIRECTORY_PATH=/music
# End of Docker-only configuration
@ -79,12 +77,12 @@ REVERSE_PROXY_TYPE=nginx
# Where media files (such as album covers or audio tracks) should be stored
# on your system?
# (Ensure this directory actually exists)
# MEDIA_ROOT=/srv/funkwhale/data/media
MEDIA_ROOT=/srv/funkwhale/data/media
# Where static files (such as API css or icons) should be compiled
# on your system?
# (Ensure this directory actually exists)
# STATIC_ROOT=/srv/funkwhale/data/static
STATIC_ROOT=/srv/funkwhale/data/static
# Update it to match the domain that will be used to reach your funkwhale
# instance
@ -112,5 +110,9 @@ RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f
# In-place import settings
# You can safely leave those settings uncommented if you don't plan to use
# in place imports.
# MUSIC_DIRECTORY_PATH=
# MUSIC_DIRECTORY_SERVE_PATH= # docker-only
# Typical docker setup:
# MUSIC_DIRECTORY_PATH=/srv/funkwhale/data/music
# MUSIC_DIRECTORY_SERVE_PATH=/music # docker-only
# Typical non-docker setup:
# MUSIC_DIRECTORY_PATH=/srv/funkwhale/data/music
# # MUSIC_DIRECTORY_SERVE_PATH= # stays commented, not needed

View File

@ -31,7 +31,7 @@ server {
# TLS
# Feel free to use your own configuration for SSL here or simply remove the
# lines and move the configuration to the previous server block if you
# don't want to run funkwhale behind https (this is not recommanded)
# don't want to run funkwhale behind https (this is not recommended)
# have a look here for let's encrypt configuration:
# https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx
ssl_protocols TLSv1.2;

View File

@ -1,4 +1,5 @@
FROM python:3.6-alpine
FROM python:3.6
RUN apt-get update && apt-get install -y graphviz
RUN pip install sphinx livereload
WORKDIR /app/docs

118
docs/architecture.rst Normal file
View File

@ -0,0 +1,118 @@
Architecture
============
Funkwhale is made of several components, each of them fulfilling a specific mission:
.. graphviz::
digraph {
node [shape=record];
rankdir=TB
concentrate=true
user [group="frontend" label="User" fontsize="9"]
webui [group="frontend" label="Web interface (VueJS SPA)" fontsize="9"]
subapps [group="frontend" label="Subsonic-compatible apps (DSub, Clementine)" fontsize="9"]
proxy [label="Reverse proxy (Nginx/Apache)" fontsize="9"]
api [label="API Server (Django)" fontsize="9"]
db [label="Database (PostgreSQL)" fontsize="9"]
cache [label="Cache and message queue (Redis)" fontsize="9"]
worker [label="Worker (Celery)" fontsize="9"]
scheduler [label="Task scheduler (Celery Beat)" fontsize="9"]
user -> subapps -> proxy
user -> webui -> proxy
cache -> worker
proxy -> api
api -> cache
api -> db
scheduler -> cache
worker -> cache
worker -> db
}
This graph may looks a bit scary, so we'll detail the role of each component below.
The user
--------
Funkwhale users can interact with your instance using:
- The official web interface
- Third-party apps
The web interface
-----------------
This refers to Funkwhale's built-in web interface, which is a Single Page application
written in Vue JS. This application will interact with Funkwhale's API to retrieve
or persist data.
Third-party apps
----------------
Since Funkwhale implements a subset of the Subsonic API, it's compatible with existing apps such
as DSub, Ultrasonic or Clementine that support this API. Those apps can be used as a replacement
or in conjunction of the web interface, but the underlying data is the same.
The reverse proxy
-----------------
Funkwhale's API server should never be exposed directly to the internet, as we require
a reverse proxy (Apache or Nginx) for performance and security reasons. The reverse proxy
will receive client HTTP requests, and:
- Proxy them to the API server
- Serve requested static files (Audio files, stylesheets, javascript, fonts...)
The API server
--------------
Funkwhale's API server is the central piece of the project. This component is responsible
for answering and processing user requests, manipulate data from the database, send long-running
tasks to workers, etc.
It's a Python/Django application.
The database
------------
Most of the data such as user accounts, favorites, music metadata or playlist is stored
in a PostgreSQL database.
The cache/message queue
-----------------------
Fetching data from the database is sometimes slow or resource hungry. To reduce
the load, Redis act as a cache for data that is considerably faster than a database.
It is also a message queue that will deliver tasks to the worker.
The worker
----------
Some operations are too long to live in the HTTP request/response cycle. Typically,
importing a bunch of uploaded tracks could take a minute or two.
To keep the API response time as fast as possible, we offload long-running tasks
to a background process, also known as the Celery worker.
Typical tasks include:
- Handling music imports
- Handling federation/ActivityPub messages
- Scanning other instances libraries
This worker is also able to retry failed tasks, or spawn automatically
more process when the number of received tasks increase.
The scheduler
-------------
Some long-running tasks are not triggered by user or external input, but on a recurring
basis instead. The scheduler is responsible for triggering those tasks and put the corresponding
messages in the message queue so the worker can process them.
Recurring tasks include:
- Cache cleaning
- Music metadata refreshing

View File

@ -20,7 +20,7 @@
import os
import sys
sys.path.insert(0, os.path.abspath('../api'))
sys.path.insert(0, os.path.abspath("../api"))
import funkwhale_api # NOQA
@ -33,24 +33,24 @@ import funkwhale_api # NOQA
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = []
extensions = ["sphinx.ext.graphviz"]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
source_suffix = ".rst"
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = 'funkwhale'
copyright = '2017, Eliot Berriot'
author = 'Eliot Berriot'
project = "funkwhale"
copyright = "2017, Eliot Berriot"
author = "Eliot Berriot"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -71,10 +71,10 @@ language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
@ -85,7 +85,7 @@ todo_include_todos = False
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
html_theme = "alabaster"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
@ -96,13 +96,13 @@ html_theme = 'alabaster'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = ["_static"]
# -- Options for HTMLHelp output ------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'funkwhaledoc'
htmlhelp_basename = "funkwhaledoc"
# -- Options for LaTeX output ---------------------------------------------
@ -111,15 +111,12 @@ latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
@ -129,8 +126,7 @@ latex_elements = {
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'funkwhale.tex', 'funkwhale Documentation',
'Eliot Berriot', 'manual'),
(master_doc, "funkwhale.tex", "funkwhale Documentation", "Eliot Berriot", "manual")
]
@ -138,10 +134,7 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'funkwhale', 'funkwhale Documentation',
[author], 1)
]
man_pages = [(master_doc, "funkwhale", "funkwhale Documentation", [author], 1)]
# -- Options for Texinfo output -------------------------------------------
@ -150,7 +143,13 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'funkwhale', 'funkwhale Documentation',
author, 'funkwhale', 'One line description of project.',
'Miscellaneous'),
(
master_doc,
"funkwhale",
"funkwhale Documentation",
author,
"funkwhale",
"One line description of project.",
"Miscellaneous",
)
]

View File

@ -88,7 +88,7 @@ Default: ``Funkwhale <noreply@yourdomain>``
Default: ``None``
The path on your server where Funwkhale can import files using :ref:`in-place import
<in-place-import>`. It must be readable by the webserver and funkwhale
<in-place-import>`. It must be readable by the webserver and Funkwhale
api and worker processes.
On docker installations, we recommend you use the default of ``/music``
@ -124,6 +124,16 @@ On non-docker setup, you don't need to configure this setting.
.. note:: This path should not include any trailing slash
.. _setting-REVERSE_PROXY_TYPE:
``REVERSE_PROXY_TYPE``
^^^^^^^^^^^^^^^^^^^^^^
Default: ``nginx``
The type of reverse-proxy behind which Funkwhale is served. Either ``apache2``
or ``nginx``. This is only used if you are using in-place import.
User permissions
----------------

View File

@ -9,12 +9,12 @@ Funkwhale is a web based music server. It is similar in term of goals and featur
A social platform
------------------
However, funkwhale is better-suited for small to medium communities and was designed to be not only a music server and player, but also a place to socialize around music and discover new content. While some of these features are not currently implemented, our roadmap includes:
However, Funkwhale is better-suited for small to medium communities and was designed to be not only a music server and player, but also a place to socialize around music and discover new content. While some of these features are not currently implemented, our roadmap includes:
- Radios, to discover the music of a given user, artist, genre...
- Playlists
- Favorites
- Broadcasts, as they existed in Grooveshark, for exemple
- Broadcasts, as they existed in Grooveshark, for example
- Recommendations
Music acquisition

View File

@ -22,7 +22,7 @@ Acquire music via federation
----------------------------
Instance libraries are protected by default. To access another instance
library, you have to follow it. Each funkwhale instance gets a dedicated
library, you have to follow it. Each Funkwhale instance gets a dedicated
ActivityPub Actor you can follow via the username "library@yourinstance.domain".
When submitted, a follow request will be sent to

View File

@ -4,8 +4,8 @@ Importing music
From music directory on the server
----------------------------------
You can import music files in funkwhale assuming they are located on the server
and readable by the funkwhale application. Your music files should contain at
You can import music files in Funkwhale assuming they are located on the server
and readable by the Funkwhale application. Your music files should contain at
least an ``artist``, ``album`` and ``title`` tags, but we recommend you tag
it extensively using a proper tool, such as Beets or Musicbrainz Picard.
@ -59,7 +59,7 @@ to import and don't want to double your disk usage.
The CLI importer supports an additional ``--in-place`` option that triggers the
following behaviour during import:
1. Imported files are not store in funkwhale anymore
1. Imported files are not store in Funkwhale anymore
2. Instead, Funkwhale will store the file path and use it to serve the music
Because those files are not managed by Funkwhale, we offer additional
@ -98,14 +98,14 @@ directory is mounted as a volume as well::
volumes:
- ./data/music:/music:ro
- ./data/media:/app/funkwhale_api/media
# add your symlinked dirs here
# add your symlinked dirs here
- /media/nfsshare:/media/nfsshare:ro
api:
volumes:
- ./data/music:/music:ro
- ./data/media:/app/funkwhale_api/media
# add your symlinked dirs here
# add your symlinked dirs here
- /media/nfsshare:/media/nfsshare:ro

View File

@ -13,14 +13,17 @@ Funkwhale is a self-hosted, modern free and open-source music server, heavily in
users/index
features
architecture
installation/index
upgrading
configuration
troubleshooting
importing-music
federation
api
third-party
contributing
translators
changelog
Indices and tables

View File

@ -1,15 +1,15 @@
Debian installation
===================
Debian and Arch Linux installation
==================================
.. note::
This guide targets Debian 9 (Stretch), which is the latest Debian.
This guide targets Debian 9 (Stretch), which is the latest Debian, as well as Arch Linux.
External dependencies
---------------------
The guides will focus on installing funkwhale-specific components and
dependencies. However, funkwhale requires a
The guides will focus on installing Funkwhale-specific components and
dependencies. However, Funkwhale requires a
:doc:`few external dependencies <./external_dependencies>` for which
documentation is outside of this document scope.
@ -17,18 +17,23 @@ Install utilities
-----------------
You'll need a few utilities during this guide that are not always present by
default on system. You can install them using:
default on system. On Debian-like systems, you can install them using:
.. code-block:: shell
sudo apt-get update
sudo apt-get install curl python3-pip python3-venv git unzip
On Arch Linux and its derivatives:
.. code-block:: shell
sudo pacman -S curl python-pip python-virtualenv git unzip
Layout
-------
All funkwhale-related files will be located under ``/srv/funkwhale`` apart
All Funkwhale-related files will be located under ``/srv/funkwhale`` apart
from database files and a few configuration files. We will also have a
dedicated ``funkwhale`` user to launch the processes we need and own those files.
@ -41,7 +46,7 @@ Create the user and the directory:
.. code-block:: shell
sudo adduser --system --home /srv/funkwhale funkwhale
sudo useradd -r -s /usr/bin/nologin -d /srv/funkwhale -m funkwhale
cd /srv/funkwhale
Log in as the newly created user from now on:
@ -57,7 +62,7 @@ Now let's setup our directory layout. Here is how it will look like::
├── api # api code of your instance
├── data # persistent data, such as music files
├── front # frontend files for the web user interface
└── virtualenv # python dependencies for funkwhale
└── virtualenv # python dependencies for Funkwhale
Create the aforementionned directories:
@ -67,7 +72,7 @@ Create the aforementionned directories:
The ``virtualenv`` directory is a bit special and will be created separately.
Download latest funkwhale release
Download latest Funkwhale release
----------------------------------
Funkwhale is splitted in two components:
@ -79,7 +84,7 @@ Those components are packaged in subsequent releases, such as 0.1, 0.2, etc.
You can browse the :doc:`changelog </changelog>` for a list of available releases
and pick the one you want to install, usually the latest one should be okay.
In this guide, we'll assume you want to install the latest version of funkwhale,
In this guide, we'll assume you want to install the latest version of Funkwhale,
which is |version|:
First, we'll download the latest api release.
@ -101,6 +106,24 @@ Then we'll download the frontend files:
mv extracted/front .
rm -rf extracted
.. note::
You can also choose to get the code directly from the git repo. In this
case, run
cd /srv
rm -r funkwhale
git clone https://code.eliotberriot.com/funkwhale/funkwhale funkwhale
cd funkwhale
You'll also need to re-create the folders we make earlier:
mkdir -p config data/static data/media data/music front
You will still need to get the frontend files as specified before, because
we're not going to build them.
You can leave the ZIP archives in the directory, this will help you know
which version you've installed next time you want to upgrade your installation.
@ -113,22 +136,20 @@ First, switch to the api directory:
cd api
A few OS packages are required in order to run funkwhale. The list is available
in ``api/requirements.apt`` or by running
``./install_os_dependencies.sh list``.
.. note::
Ensure you are running the next commands as root or using sudo
(and not as the funkwhale) user.
You can install those packages all at once:
A few OS packages are required in order to run Funkwhale. On Debian-like
systems, they can be installed with
.. code-block:: shell
./install_os_dependencies.sh install
sudo apt install build-essential ffmpeg libjpeg-dev libmagic-dev libpq-dev postgresql-client python3-dev
From now on you can switch back to the funkwhale user.
On Arch, run
.. code-block:: shell
pacman -S $(cat api/requirements.pac)
From now on, you should use the funkwhale user for all commands.
Python dependencies
--------------------
@ -143,11 +164,12 @@ To avoid collisions with other software on your system, Python dependencies
will be installed in a dedicated
`virtualenv <https://docs.python.org/3/library/venv.html>`_.
First, create the virtualenv:
First, create the virtualenv and install wheel:
.. code-block:: shell
python3 -m venv /srv/funkwhale/virtualenv
pip3 install wheel
This will result in a ``virtualenv`` directory being created in
``/srv/funkwhale/virtualenv``.
@ -165,12 +187,11 @@ Finally, install the python dependencies:
.. code-block:: shell
pip install wheel
pip install -r api/requirements.txt
.. important::
further commands involving python should always be run after you activated
Further commands involving python should always be run after you activated
the virtualenv, as described earlier, otherwise those commands will raise
errors
@ -178,7 +199,7 @@ Finally, install the python dependencies:
Environment file
----------------
You can now start to configure funkwhale. The main way to achieve that is by
You can now start to configure Funkwhale. The main way to achieve that is by
adding an environment file that will host settings that are relevant to your
installation.
@ -188,8 +209,15 @@ Download the sample environment file:
curl -L -o config/.env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/env.prod.sample"
.. note::
if you used git to get the latest version of the code earlier, you can instead do
cp /srv/funkwhale/deploy/env.prod.sample /srv/funkwhale/config/.env
You can then edit it: the file is heavily commented, and the most relevant
configuration options are mentionned at the top of the file.
configuration options are mentioned at the top of the file.
Especially, populate the ``DATABASE_URL`` and ``CACHE_URL`` values based on
how you configured your PostgreSQL and Redis servers in
@ -198,30 +226,15 @@ how you configured your PostgreSQL and Redis servers in
When you want to run command on the API server, such as to create the
database or compile static files, you have to ensure you source
the environment variables.
the environment variables in that file.
This can be done like this::
export $(cat config/.env | grep -v ^# | xargs)
The easier thing to do is to store this in a script::
cat > /srv/funkwhale/load_env <<'EOL'
#!/bin/bash
export $(cat /srv/funkwhale/config/.env | grep -v ^# | xargs)
EOL
chmod +x /srv/funkwhale/load_env
You should now be able to run the following to populate your environment
variables easily:
.. code-block:: shell
source /srv/funkwhale/load_env
.. note::
Remember to source ``load_env`` whenever you edit your .env file.
Remember to reload these variables whenever you edit your .env file.
Database setup
---------------
@ -267,7 +280,7 @@ Collect static files
--------------------
Static files are the static assets used by the API server (icon PNGs, CSS, etc.).
We need to collect them explicitely, so they can be served by the webserver:
We need to collect them explicitly, so they can be served by the webserver:
.. code-block:: shell

View File

@ -1,7 +1,7 @@
Docker installation
====================
Docker is the easiest way to get a funkwhale instance up and running.
Docker is the easiest way to get a Funkwhale instance up and running.
First, ensure you have `Docker <https://docs.docker.com/engine/installation/>`_ and `docker-compose <https://github.com/docker/compose/releases>`_ installed.

View File

@ -14,12 +14,20 @@ Funkwhale requires a PostgreSQL database to work properly. Please refer
to the `PostgreSQL documentation <https://www.postgresql.org/download/>`_
for installation instructions specific to your os.
On debian-like systems, you would install the database server like this:
On Debian-like systems, you would install the database server like this:
.. code-block:: shell
sudo apt-get install postgresql postgresql-contrib
On Arch Linux and its derivatives:
.. code-block:: shell
sudo pacman -S postgresql
On Arch, you'll also need to initialize the database. See `the Arch Linux wiki <https://wiki.archlinux.org/index.php/Postgresql#Initial_configuration>`_.
The remaining steps are heavily inspired from `this Digital Ocean guide <https://www.digitalocean.com/community/tutorials/how-to-set-up-django-with-postgres-nginx-and-gunicorn-on-ubuntu-16-04>`_.
Open a database shell:
@ -37,22 +45,23 @@ Create the project database and user:
CREATE USER funkwhale;
GRANT ALL PRIVILEGES ON DATABASE funkwhale TO funkwhale;
Assuming you already have :ref:`created your funkwhale user <create-funkwhale-user>`,
you should now be able to open a postgresql shell:
.. warning::
It's importing that you use utf-8 encoding for your database,
It's important that you use utf-8 encoding for your database,
otherwise you'll end up with errors and crashes later on when dealing
with music metedata that contains non-ascii chars.
Assuming you already have :ref:`created your funkwhale user <create-funkwhale-user>`,
you should now be able to open a postgresql shell:
.. code-block:: shell
sudo -u funkwhale -H psql
Unless you give a superuser access to the database user, you should also
enable some extensions on your database server, as those are required
for funkwhale to work properly:
for Funkwhale to work properly:
.. code-block:: shell
@ -68,11 +77,17 @@ Funkwhale also requires a cache server:
queries
- To handle asynchronous tasks such as music import
On debian-like distributions, a redis package is available, and you can
On Debian-like distributions, a redis package is available, and you can
install it:
.. code-block:: shell
sudo apt-get install redis-server
On Arch Linux and its derivatives:
.. code-block:: shell
sudo pacman -S redis
This should be enough to have your redis server set up.

View File

@ -56,7 +56,7 @@ Available installation methods
-------------------------------
Docker is the recommended and easiest way to setup your Funkwhale instance.
We also maintain an installation guide for Debian 9.
We also maintain an installation guide for Debian 9 and Arch Linux.
.. toctree::
:maxdepth: 1
@ -96,17 +96,23 @@ Files for the web frontend are purely static and can simply be downloaded, unzip
Reverse proxy
--------------
In order to make funkwhale accessible from outside your server and to play nicely with other applications on your machine, you should configure a reverse proxy.
In order to make Funkwhale accessible from outside your server and to play nicely with other applications on your machine, you should configure a reverse proxy.
Nginx
^^^^^
Ensure you have a recent version of nginx on your server. On debian-like system, you would have to run the following:
Ensure you have a recent version of nginx on your server. On Debian-like system, you would have to run the following:
.. code-block:: bash
apt-get update
apt-get install nginx
sudo apt-get update
sudo apt-get install nginx
On Arch Linux and its derivatives:
.. code-block:: bash
sudo pacman -S nginx
Then, download our sample virtualhost file and proxy conf:
@ -129,10 +135,15 @@ If everything is fine, you can restart your nginx server with ``service nginx re
Apache2
^^^^^^^
Ensure you have a recent version of apache2 installed on your server.
.. note::
These instructions are for Debian only.
For Arch Linux please refer to the `Arch Linux wiki <https://wiki.archlinux.org/index.php/Apache>`_.
Ensure you have a recent version of Apache2 installed on your server.
You'll also need the following dependencies::
apt install libapache2-mod-xsendfile
sudo apt-get install libapache2-mod-xsendfile
Then, download our sample virtualhost file:

View File

@ -2,7 +2,7 @@ Optimizing your Funkwhale instance
==================================
Depending on your requirements, you may want to reduce as much as possible
Funkwhale's footprint.
Funkwhale's memory footprint.
Reduce workers concurrency
--------------------------
@ -14,9 +14,9 @@ memory usage.
You can control this behaviour using the ``--concurrency`` flag.
For instance, setting ``--concurrency=1`` will spawn only one worker.
This flag should be appended after the ``celery -A funkwhale_api.taskapp worker``
command in your :file:`docker-compose.yml` file if your using Docker, or in your
:file:`/etc/systemd/system/funkwhale-worker.service` otherwise.
This flag should be appended after the ``celery -A funkwhale_api.taskapp
worker`` command in your :file:`docker-compose.yml` file if your using Docker,
or in your :file:`/etc/systemd/system/funkwhale-worker.service` otherwise.
.. note::
@ -33,5 +33,5 @@ Using the ``solo`` pool type should reduce your memory consumption.
You can control this behaviour using the ``--pool=solo`` flag.
This flag should be appended after the ``celery -A funkwhale_api.taskapp worker``
command in your :file:`docker-compose.yml` file if your using Docker, or in your
:file:`/etc/systemd/system/funkwhale-worker.service` otherwise.
command in your :file:`docker-compose.yml` file if you're using Docker, or in
your :file:`/etc/systemd/system/funkwhale-worker.service` otherwise.

View File

@ -1,10 +1,10 @@
Systemd configuration
----------------------
Systemd offers a convenient way to manage your funkwhale instance if you're
Systemd offers a convenient way to manage your Funkwhale instance if you're
not using docker.
We'll see how to setup systemd to proprely start a funkwhale instance.
We'll see how to setup systemd to proprely start a Funkwhale instance.
First, download the sample unitfiles:
@ -17,9 +17,9 @@ First, download the sample unitfiles:
This will download three unitfiles:
- ``funkwhale-server.service`` to launch the funkwhale web server
- ``funkwhale-worker.service`` to launch the funkwhale task worker
- ``funkwhale-beat.service`` to launch the funkwhale task beat (this is for recurring tasks)
- ``funkwhale-server.service`` to launch the Funkwhale web server
- ``funkwhale-worker.service`` to launch the Funkwhale task worker
- ``funkwhale-beat.service`` to launch the Funkwhale task beat (this is for recurring tasks)
- ``funkwhale.target`` to easily stop and start all of the services at once
You can of course review and edit them to suit your deployment scenario

1
docs/translators.rst Normal file
View File

@ -0,0 +1 @@
.. include:: ../TRANSLATORS.rst

212
docs/troubleshooting.rst Normal file
View File

@ -0,0 +1,212 @@
Troubleshooting
===============
Various errors and issues can arise on your Funkwhale instance, caused by configuration errors,
deployment/environment specific issues, or bugs in the software itself.
On this document, you'll find:
- Tools and commands you can use to better understand the issues
- A list of common pitfalls and errors and how to solve them
- A collection of links and advice to get help from the community and report new issues
Diagnose problems
^^^^^^^^^^^^^^^^^
Funkwhale is made of several components, each one being a potential cause for failure. Having an even basic overview
of Funkwhale's technical architecture can help you understand what is going on. You can refer to :doc:`the technical architecture </architecture>` for that.
Problems usually fall into one of those categories:
- **Frontend**: Funkwhale's interface is not loading, not behaving as expected, music is not playing
- **API**: the interface do not display any data or show errors
- **Import**: uploaded/imported tracks are not imported correctly or at all
- **Federation**: you cannot contact other Funkwhale servers, access their library, play federated tracks
- **Everything else**
Each category comes with its own set of diagnose tools and/or commands we will detail below. We'll also give you simple
steps for each type of problem. Please try those to see if it fix your issues. If none of those works, please report your issue on our
issue tracker.
Frontend issues
^^^^^^^^^^^^^^^
Diagnostic tools:
- Javascript and network logs from your browser console (see instructions on how to open it in `Chrome <https://developers.google.com/web/tools/chrome-devtools/console/>`_ and `Firefox <https://developer.mozilla.org/en-US/docs/Tools/Web_Console/Opening_the_Web_Console>`_
- Proxy and API access and error logs (see :ref:`access-logs`)
- The same operation works from a different browser
Common problems
***************
The front-end is completely blank
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You are visiting Funkwhale, but you don't see anything.
- Try from a different browser
- Check network errors in your browser console. If you see responses with 40X or 50X statuses, there is probably an issue with the webserver configuration
- If you don't see anything wrong in the network console, check the Javascript console
- Disable your browser extensions (like adblockers)
Music is not playing
~~~~~~~~~~~~~~~~~~~~
You have some tracks in your queue that don't play, or the queue is jumping from one track to the next until
there is no more track available:
- Try with other tracks. If it works with some tracks but not other tracks, this may means that the failing tracks are not probably imported
or that your browser does not support a specific audio format
- Check the network and javascript console for potential errors
Tracks are not appending to the queue
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
When clicking on "Play", "Play all albums" or "Play all" buttons, some tracks are not appended to the queue. This is
actually a feature of Funkwhale: those tracks have no file associated with them, so we cannot play them.
Specific pages are loading forever or blank
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
When viewing a given page, the page load never ends (you continue to see the spinner), or nothing seems to appear at all:
- Ensure your internet connection is up and running
- Ensure your instance is up and running
- Check the network and javascript console for potential errors
Backend issues
^^^^^^^^^^^^^^
Diagnostic tools:
- Reverse proxy logs:
- Apache logs should be available at :file:`/var/log/apache/access.log` and :file:`/var/log/apache/error.log`
- Nginx logs should be available at :file:`/var/log/nginx/access.log` and :file:`/var/log/nginx/error.log`
- API logs:
- Docker setup: ``docker-compose logs -f --tail=50 api`` (remove the ``--tail`` flag to get the full logs)
- Non-docker setup: ``journalctl -xn -u funkwhale-server``
.. note::
If you edit your .env file to test a new configuration, you have to restart your services to pick up the changes:
- Docker setup: ``docker-compose up -d``
- Non-docker setup: ``systemctl restart funkwhale.target``
Common problems
***************
Instance work properly, but audio files are not served (404 error)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- If you're using docker, ensure the ``MEDIA_ROOT`` variable is commented in your env file
- Ensure the ``_protected/media`` block points toward the path where media files are stored (``/srv/funkwhale/data/media``, by default)
- If you're using in-place import, ensure :ref:`setting-MUSIC_DIRECTORY_PATH`, :ref:`setting-MUSIC_DIRECTORY_SERVE_PATH` and :ref:`setting-REVERSE_PROXY_TYPE` are configured properly, and that the files are readable by the webserver
Weakref error when running ``python manage.py <command>``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
On Python <3.6, you may see this kind of errors when running commands like ``python manage.py migrate``::
Exception ignored in: <function WeakValueDictionary.__init__.<locals>.remove at 0x107e7a6a8>
Traceback (most recent call last):
File "/srv/funkwhale/venv/lib/python3.5/weakref.py", line 117, in remove
TypeError: 'NoneType' object is not callable
This is caused by a bug in Python (cf https://github.com/celery/celery/issues/3818), and is not affecting in any way
the command you execute. You can safely ignore this error.
``Your models have changes that are not yet reflected in a migration`` warning
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
When running ``python manage.py migrate`` (both in docker or non-docker), you may end-up with this::
Operations to perform:
Apply all migrations: account, admin, auth, authtoken, common, contenttypes, dynamic_preferences, favorites, federation, history, music, playlists, radios, requests, sessions, sites, socialaccount, taggit, users
Running migrations:
No migrations to apply.
Your models have changes that are not yet reflected in a migration, and so won't be applied.
Run 'manage.py makemigrations' to make new migrations, and then re-run 'manage.py migrate' to apply them.
This warning can be safely ignored. You should not run the suggested ``manage.py makemigrations`` command.
File import issues
^^^^^^^^^^^^^^^^^^
Unless you are using the CLI to import files, imports are send as tasks in a queue to a celery worker that will process them.
Diagnostic tools:
- Celery worker logs:
- Docker setup: ``docker-compose logs -f --tail=50 celeryworker`` (remove the ``--tail`` flag to get the full logs)
- Non-docker setup: ``journalctl -xn -u funkwhale-worker``
Federation issues
^^^^^^^^^^^^^^^^^
Received federations messages are sent to a dedicated task queue and processed asynchronously by a celery worker.
Diagnostic tools:
- API logs:
- Docker setup: ``docker-compose logs -f --tail=50 api`` (remove the ``--tail`` flag to get the full logs)
- Non-docker setup: ``journalctl -xn -u funkwhale-server``
- Celery worker logs:
- Docker setup: ``docker-compose logs -f --tail=50 celeryworker`` (remove the ``--tail`` flag to get the full logs)
- Non-docker setup: ``journalctl -xn -u funkwhale-worker``
Common problems
***************
I have no access to another instance library
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Check if it works with the demo library (library@demo.funkwhale.audio)
- Check if the remote library received your follow request and approved it
- Trigger a scan via the interface
- Have a look in the celery logs for potential errors during the scan
Other problems
^^^^^^^^^^^^^^
It's a bit hard to give targeted advice about problems that do not fit in the previous categories. However, we can recommend to:
- Try to identify the scope of the issue and reproduce it reliably
- Ensure your instance is configured as detailed in the installation documentation, and if you did not use the default
values, to check what you changed
- To read the .env file carefuly, as most of the options are described in the comments
Report an issue or get help
^^^^^^^^^^^^^^^^^^^^^^^^^^^
Well be more than happy to help you to debug installation and configuration issues. The main channel
for receiving support about your Funkwhale installation is the `#funkwhale-troubleshooting:matrix.org <https://riot.im/app/#/room/#funkwhale-troubleshooting:matrix.org>`_ Matrix channel.
Before asking for help, we'd really appreciate if you took the time to go through this document and try to diagnose the problem yourself. But if you don't find
anything relevant or don't have the time, we'll be there for you!
Here are a few recommendations on how to structure and what to include in your help requests:
- Give us as much context as possible about your installation (OS, version, Docker/non-docker, reverse-proxy type, relevant logs and errors, etc.)
- Including screenshots or small gifs or videos can help us considerably when debugging front-end issues
You can also open issues on our `issue tracker <https://code.eliotberriot.com/funkwhale/funkwhale/issues>`_. Please have a quick look for
similar issues before doing that, and use the issue tracker only to report bugs, suggest enhancements (both in the software and the documentation) or new features.
.. warning::
If you ever need to share screenshots or urls with someone else, ensure those do not include your personnal token.
This token is binded to your account and can be used to connect and use your account.
Urls that includes your token looks like: ``https://your.instance/api/v1/trackfiles/42/serve/?jwt=yoursecrettoken``
Improving this documentation
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
If you feel like something should be improved in this document (and in the documentation in general), feel free to `edit
it <https://code.eliotberriot.com/funkwhale/funkwhale/tree/develop/docs>`_ and open a Merge Request. If you lack time or skills
to do that, you can open an issue to discuss that, and someone else will do it.

View File

@ -1,4 +1,4 @@
Upgrading your funkwhale instance to a newer version
Upgrading your Funkwhale instance to a newer version
====================================================
.. note::
@ -7,7 +7,7 @@ Upgrading your funkwhale instance to a newer version
the database and the media files.
We're commited to make upgrade as easy and straightforward as possible,
however, funkwhale is still in development and you'll be safer with a backup.
however, Funkwhale is still in development and you'll be safer with a backup.
Reading the release notes
@ -72,7 +72,7 @@ Upgrading the API
^^^^^^^^^^^^^^^^^
On non-docker, upgrade involves a few more commands. We assume your setup
match what is described in :doc:`debian`:
match what is described in :doc:`/installation/debian`:
.. parsed-literal::

View File

@ -61,6 +61,22 @@ Then, when using a client, you'll have to input some information about your serv
In your client configuration, please double check the "ID3" or "Browse with tags"
setting is enabled.
Ultrasonic (Android)
^^^^^^^^^^^^^^^^^^^^
- Price: free
- F-Droid: https://f-droid.org/en/packages/org.moire.ultrasonic/
- Google Play: https://play.google.com/store/apps/details?id=org.moire.ultrasonic
- Sources: https://github.com/ultrasonic/ultrasonic
Ultrasonic is a full-featured Subsonic client with Playlists, Stars, Search,
Offline mode, etc.
It's one of the recommended Android client to use with Funkwhale, as we are doing
our Android tests on this one.
DSub (Android)
^^^^^^^^^^^^^^
@ -76,7 +92,7 @@ DSub is a full-featured Subsonic client that works great, and has a lot of featu
- Search
- Offline cache (with configurable size, playlist download, queue prefetching, etc.)
It's the recommended Android client to use with Funkwhale, as we are doing
It's one of the recommended Android client to use with Funkwhale, as we are doing
our Android tests on this one.
Clementine (Desktop)

Some files were not shown because too many files have changed in this diff Show More