diff --git a/.env.dev b/.env.dev index de58e2758..bc2d667b1 100644 --- a/.env.dev +++ b/.env.dev @@ -1,3 +1,3 @@ -BACKEND_URL=http://localhost:12081 -YOUTUBE_API_KEY= -API_AUTHENTICATION_REQUIRED=False +BACKEND_URL=http://localhost:6001 +API_AUTHENTICATION_REQUIRED=True +CACHALOT_ENABLED=False diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b85d6ac3b..58602d296 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,6 +70,7 @@ docker_develop: stage: deploy before_script: - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD + - cp -r front/dist api/frontend - cd api script: - docker build -t $IMAGE . @@ -83,6 +84,7 @@ docker_release: stage: deploy before_script: - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD + - cp -r front/dist api/frontend - cd api script: - docker build -t $IMAGE -t $IMAGE_LATEST . diff --git a/api/compose/django/entrypoint.sh b/api/compose/django/entrypoint.sh index a4060f658..98b3681e1 100755 --- a/api/compose/django/entrypoint.sh +++ b/api/compose/django/entrypoint.sh @@ -9,10 +9,15 @@ export REDIS_URL=redis://redis:6379/0 # the official postgres image uses 'postgres' as default user if not set explictly. if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then export POSTGRES_ENV_POSTGRES_USER=postgres -fi +fi export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER export CELERY_BROKER_URL=$REDIS_URL -exec "$@" \ No newline at end of file +# we copy the frontend files, if any so we can serve them from the outside +if [ -d "frontend" ]; then + mkdir -p /frontend + cp -r frontend/* /frontend/ +fi +exec "$@" diff --git a/api/compose/nginx/Dockerfile b/api/compose/nginx/Dockerfile deleted file mode 100644 index 196395763..000000000 --- a/api/compose/nginx/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM nginx:latest -ADD nginx.conf /etc/nginx/nginx.conf \ No newline at end of file diff --git a/api/config/api_urls.py b/api/config/api_urls.py index b56944d4e..5ed4cffdd 100644 --- a/api/config/api_urls.py +++ b/api/config/api_urls.py @@ -4,26 +4,40 @@ from funkwhale_api.music import views from funkwhale_api.playlists import views as playlists_views from rest_framework_jwt import views as jwt_views +from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet +from dynamic_preferences.users.viewsets import UserPreferencesViewSet router = routers.SimpleRouter() +router.register(r'settings', GlobalPreferencesViewSet, base_name='settings') router.register(r'tags', views.TagViewSet, 'tags') router.register(r'tracks', views.TrackViewSet, 'tracks') +router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles') router.register(r'artists', views.ArtistViewSet, 'artists') router.register(r'albums', views.AlbumViewSet, 'albums') router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches') router.register(r'submit', views.SubmitViewSet, 'submit') router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists') -router.register(r'playlist-tracks', playlists_views.PlaylistTrackViewSet, 'playlist-tracks') +router.register( + r'playlist-tracks', + playlists_views.PlaylistTrackViewSet, + 'playlist-tracks') v1_patterns = router.urls v1_patterns += [ - url(r'^providers/', include('funkwhale_api.providers.urls', namespace='providers')), - url(r'^favorites/', include('funkwhale_api.favorites.urls', namespace='favorites')), - url(r'^search$', views.Search.as_view(), name='search'), - url(r'^radios/', include('funkwhale_api.radios.urls', namespace='radios')), - url(r'^history/', include('funkwhale_api.history.urls', namespace='history')), - url(r'^users/', include('funkwhale_api.users.api_urls', namespace='users')), - url(r'^token/', jwt_views.obtain_jwt_token), + url(r'^providers/', + include('funkwhale_api.providers.urls', namespace='providers')), + url(r'^favorites/', + include('funkwhale_api.favorites.urls', namespace='favorites')), + url(r'^search$', + views.Search.as_view(), name='search'), + url(r'^radios/', + include('funkwhale_api.radios.urls', namespace='radios')), + url(r'^history/', + include('funkwhale_api.history.urls', namespace='history')), + url(r'^users/', + include('funkwhale_api.users.api_urls', namespace='users')), + url(r'^token/', + jwt_views.obtain_jwt_token), url(r'^token/refresh/', jwt_views.refresh_jwt_token), ] diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 93381c4f5..70804c3c9 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -53,6 +53,7 @@ THIRD_PARTY_APPS = ( 'rest_auth', 'rest_auth.registration', 'mptt', + 'dynamic_preferences', ) # Apps specific for this project go here. @@ -65,6 +66,7 @@ LOCAL_APPS = ( 'funkwhale_api.history', 'funkwhale_api.playlists', 'funkwhale_api.providers.audiofile', + 'funkwhale_api.providers.youtube', ) # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps @@ -217,7 +219,6 @@ STATICFILES_FINDERS = ( # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root MEDIA_ROOT = str(APPS_DIR('media')) -USE_SAMPLE_TRACK = env.bool("USE_SAMPLE_TRACK", False) # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url @@ -261,7 +262,6 @@ BROKER_URL = env("CELERY_BROKER_URL", default='django://') # Location of root django.contrib.admin URL, use {% url 'admin:index' %} ADMIN_URL = r'^admin/' -SESSION_SAVE_EVERY_REQUEST = True # Your common stuff: Below this line define 3rd party library settings CELERY_DEFAULT_RATE_LIMIT = 1 CELERYD_TASK_TIME_LIMIT = 300 @@ -290,6 +290,7 @@ REST_FRAMEWORK = { 'PAGE_SIZE': 25, 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS', 'rest_framework_jwt.authentication.JSONWebTokenAuthentication', 'rest_framework.authentication.SessionAuthentication', 'rest_framework.authentication.BasicAuthentication', @@ -299,9 +300,24 @@ REST_FRAMEWORK = { ) } -FUNKWHALE_PROVIDERS = { - 'youtube': { - 'api_key': env('YOUTUBE_API_KEY', default='REPLACE_ME') - } -} ATOMIC_REQUESTS = False + +# Wether we should check user permission before serving audio files (meaning +# return an obfuscated url) +# This require a special configuration on the reverse proxy side +# See https://wellfire.co/learn/nginx-django-x-accel-redirects/ for example +PROTECT_AUDIO_FILES = env.bool('PROTECT_AUDIO_FILES', default=True) + +# Which path will be used to process the internal redirection +# **DO NOT** put a slash at the end +PROTECT_FILES_PATH = env('PROTECT_FILES_PATH', default='/_protected') + + +# use this setting to tweak for how long you want to cache +# musicbrainz results. (value is in seconds) +MUSICBRAINZ_CACHE_DURATION = env.int( + 'MUSICBRAINZ_CACHE_DURATION', + default=300 +) + +CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True) diff --git a/api/funkwhale_api/__init__.py b/api/funkwhale_api/__init__.py index 6b304975e..70d6b5ac1 100644 --- a/api/funkwhale_api/__init__.py +++ b/api/funkwhale_api/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__version__ = '0.1.0' +__version__ = '0.2.0' __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')]) diff --git a/api/funkwhale_api/common/authentication.py b/api/funkwhale_api/common/authentication.py new file mode 100644 index 000000000..b75f3b516 --- /dev/null +++ b/api/funkwhale_api/common/authentication.py @@ -0,0 +1,20 @@ +from rest_framework import exceptions +from rest_framework_jwt import authentication +from rest_framework_jwt.settings import api_settings + + +class JSONWebTokenAuthenticationQS( + authentication.BaseJSONWebTokenAuthentication): + + www_authenticate_realm = 'api' + + def get_jwt_value(self, request): + token = request.query_params.get('jwt') + if 'jwt' in request.query_params and not token: + msg = _('Invalid Authorization header. No credentials provided.') + raise exceptions.AuthenticationFailed(msg) + return token + + def authenticate_header(self, request): + return '{0} realm="{1}"'.format( + api_settings.JWT_AUTH_HEADER_PREFIX, self.www_authenticate_realm) diff --git a/api/funkwhale_api/common/tests/test_jwt_querystring.py b/api/funkwhale_api/common/tests/test_jwt_querystring.py new file mode 100644 index 000000000..90e63775d --- /dev/null +++ b/api/funkwhale_api/common/tests/test_jwt_querystring.py @@ -0,0 +1,32 @@ +from test_plus.test import TestCase +from rest_framework_jwt.settings import api_settings + +from funkwhale_api.users.models import User + + +jwt_payload_handler = api_settings.JWT_PAYLOAD_HANDLER +jwt_encode_handler = api_settings.JWT_ENCODE_HANDLER + + +class TestJWTQueryString(TestCase): + www_authenticate_realm = 'api' + + def test_can_authenticate_using_token_param_in_url(self): + user = User.objects.create_superuser( + username='test', email='test@test.com', password='test') + + url = self.reverse('api:v1:tracks-list') + with self.settings(API_AUTHENTICATION_REQUIRED=True): + response = self.client.get(url) + + self.assertEqual(response.status_code, 401) + + payload = jwt_payload_handler(user) + token = jwt_encode_handler(payload) + print(payload, token) + with self.settings(API_AUTHENTICATION_REQUIRED=True): + response = self.client.get(url, data={ + 'jwt': token + }) + + self.assertEqual(response.status_code, 200) diff --git a/api/funkwhale_api/music/metadata.py b/api/funkwhale_api/music/metadata.py index d1be9a4e1..31d13d495 100644 --- a/api/funkwhale_api/music/metadata.py +++ b/api/funkwhale_api/music/metadata.py @@ -37,13 +37,26 @@ def get_mp3_recording_id(f, k): except IndexError: raise TagNotFound(k) + +def convert_track_number(v): + try: + return int(v) + except ValueError: + # maybe the position is of the form "1/4" + pass + + try: + return int(v.split('/')[0]) + except (ValueError, AttributeError, IndexError): + pass + CONF = { 'OggVorbis': { 'getter': lambda f, k: f[k][0], 'fields': { 'track_number': { 'field': 'TRACKNUMBER', - 'to_application': int + 'to_application': convert_track_number }, 'title': { 'field': 'title' @@ -74,7 +87,7 @@ CONF = { 'fields': { 'track_number': { 'field': 'TPOS', - 'to_application': lambda v: int(v.split('/')[0]) + 'to_application': convert_track_number }, 'title': { 'field': 'TIT2' diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index 702308477..6a55dfc00 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -8,7 +8,6 @@ import markdown from django.conf import settings from django.db import models -from django.contrib.staticfiles.templatetags.staticfiles import static from django.core.files.base import ContentFile from django.core.files import File from django.core.urlresolvers import reverse @@ -354,10 +353,12 @@ class TrackFile(models.Model): @property def path(self): - if settings.USE_SAMPLE_TRACK: - return static('music/sample1.ogg') + if settings.PROTECT_AUDIO_FILES: + return reverse( + 'api:v1:trackfiles-serve', kwargs={'pk': self.pk}) return self.audio_file.url + class ImportBatch(models.Model): creation_date = models.DateTimeField(default=timezone.now) submitted_by = models.ForeignKey('users.User', related_name='imports') diff --git a/api/funkwhale_api/music/tests/factories.py b/api/funkwhale_api/music/tests/factories.py new file mode 100644 index 000000000..dfa7a75e2 --- /dev/null +++ b/api/funkwhale_api/music/tests/factories.py @@ -0,0 +1,39 @@ +import factory + + +class ArtistFactory(factory.django.DjangoModelFactory): + name = factory.Sequence(lambda n: 'artist-{0}'.format(n)) + mbid = factory.Faker('uuid4') + + class Meta: + model = 'music.Artist' + + +class AlbumFactory(factory.django.DjangoModelFactory): + title = factory.Sequence(lambda n: 'album-{0}'.format(n)) + mbid = factory.Faker('uuid4') + release_date = factory.Faker('date') + cover = factory.django.ImageField() + artist = factory.SubFactory(ArtistFactory) + + class Meta: + model = 'music.Album' + + +class TrackFactory(factory.django.DjangoModelFactory): + title = factory.Sequence(lambda n: 'track-{0}'.format(n)) + mbid = factory.Faker('uuid4') + album = factory.SubFactory(AlbumFactory) + artist = factory.SelfAttribute('album.artist') + position = 1 + + class Meta: + model = 'music.Track' + + +class TrackFileFactory(factory.django.DjangoModelFactory): + track = factory.SubFactory(TrackFactory) + audio_file = factory.django.FileField() + + class Meta: + model = 'music.TrackFile' diff --git a/api/funkwhale_api/music/tests/test_api.py b/api/funkwhale_api/music/tests/test_api.py index d8f56eeb9..21a567084 100644 --- a/api/funkwhale_api/music/tests/test_api.py +++ b/api/funkwhale_api/music/tests/test_api.py @@ -10,6 +10,8 @@ from funkwhale_api.music import serializers from funkwhale_api.users.models import User from . import data as api_data +from . import factories + class TestAPI(TMPDirTestCaseMixin, TestCase): @@ -214,3 +216,26 @@ class TestAPI(TMPDirTestCaseMixin, TestCase): with self.settings(API_AUTHENTICATION_REQUIRED=False): response = getattr(self.client, method)(url) self.assertEqual(response.status_code, 200) + + def test_track_file_url_is_restricted_to_authenticated_users(self): + f = factories.TrackFileFactory() + self.assertNotEqual(f.audio_file, None) + url = f.path + + with self.settings(API_AUTHENTICATION_REQUIRED=True): + response = self.client.get(url) + + self.assertEqual(response.status_code, 401) + + user = User.objects.create_superuser( + username='test', email='test@test.com', password='test') + self.client.login(username=user.username, password='test') + with self.settings(API_AUTHENTICATION_REQUIRED=True): + response = self.client.get(url) + + self.assertEqual(response.status_code, 200) + + self.assertEqual( + response['X-Accel-Redirect'], + '/_protected{}'.format(f.audio_file.url) + ) diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index 772f4173e..4a4032c57 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -1,8 +1,11 @@ import os import json +import unicodedata +import urllib from django.core.urlresolvers import reverse from django.db import models, transaction from django.db.models.functions import Length +from django.conf import settings from rest_framework import viewsets, views from rest_framework.decorators import detail_route, list_route from rest_framework.response import Response @@ -51,6 +54,7 @@ class ArtistViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet): search_fields = ['name'] ordering_fields = ('creation_date',) + class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet): queryset = ( models.Album.objects.all() @@ -63,6 +67,7 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet): search_fields = ['title'] ordering_fields = ('creation_date',) + class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet): queryset = models.ImportBatch.objects.all().order_by('-creation_date') serializer_class = serializers.ImportBatchSerializer @@ -70,6 +75,7 @@ class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet): def get_queryset(self): return super().get_queryset().filter(submitted_by=self.request.user) + class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet): """ A simple ViewSet for viewing and editing accounts. @@ -120,6 +126,29 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet): return Response(serializer.data) +class TrackFileViewSet(viewsets.ReadOnlyModelViewSet): + queryset = (models.TrackFile.objects.all().order_by('-id')) + serializer_class = serializers.TrackFileSerializer + permission_classes = [ConditionalAuthentication] + + @detail_route(methods=['get']) + def serve(self, request, *args, **kwargs): + try: + f = models.TrackFile.objects.get(pk=kwargs['pk']) + except models.TrackFile.DoesNotExist: + return Response(status=404) + + response = Response() + filename = "filename*=UTF-8''{}{}".format( + urllib.parse.quote(f.track.full_name), + os.path.splitext(f.audio_file.name)[-1]) + response["Content-Disposition"] = "attachment; {}".format(filename) + response['X-Accel-Redirect'] = "{}{}".format( + settings.PROTECT_FILES_PATH, + f.audio_file.url) + return response + + class TagViewSet(viewsets.ReadOnlyModelViewSet): queryset = Tag.objects.all().order_by('name') serializer_class = serializers.TagSerializer diff --git a/api/funkwhale_api/musicbrainz/client.py b/api/funkwhale_api/musicbrainz/client.py index e281555b2..049ed298c 100644 --- a/api/funkwhale_api/musicbrainz/client.py +++ b/api/funkwhale_api/musicbrainz/client.py @@ -1,11 +1,17 @@ import musicbrainzngs +import memoize.djangocache from django.conf import settings from funkwhale_api import __version__ + _api = musicbrainzngs _api.set_useragent('funkwhale', str(__version__), 'contact@eliotberriot.com') +store = memoize.djangocache.Cache('default') +memo = memoize.Memoizer(store, namespace='memoize:musicbrainz') + + def clean_artist_search(query, **kwargs): cleaned_kwargs = {} if kwargs.get('name'): @@ -17,30 +23,55 @@ class API(object): _api = _api class artists(object): - search = clean_artist_search - get = _api.get_artist_by_id + search = memo( + clean_artist_search, max_age=settings.MUSICBRAINZ_CACHE_DURATION) + get = memo( + _api.get_artist_by_id, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) class images(object): - get_front = _api.get_image_front + get_front = memo( + _api.get_image_front, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) class recordings(object): - search = _api.search_recordings - get = _api.get_recording_by_id + search = memo( + _api.search_recordings, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + get = memo( + _api.get_recording_by_id, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) class works(object): - search = _api.search_works - get = _api.get_work_by_id + search = memo( + _api.search_works, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + get = memo( + _api.get_work_by_id, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) class releases(object): - search = _api.search_releases - get = _api.get_release_by_id - browse = _api.browse_releases + search = memo( + _api.search_releases, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + get = memo( + _api.get_release_by_id, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + browse = memo( + _api.browse_releases, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) # get_image_front = _api.get_image_front class release_groups(object): - search = _api.search_release_groups - get = _api.get_release_group_by_id - browse = _api.browse_release_groups + search = memo( + _api.search_release_groups, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + get = memo( + _api.get_release_group_by_id, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + browse = memo( + _api.browse_release_groups, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) # get_image_front = _api.get_image_front api = API() diff --git a/api/funkwhale_api/musicbrainz/tests/test_cache.py b/api/funkwhale_api/musicbrainz/tests/test_cache.py new file mode 100644 index 000000000..d2d1260ec --- /dev/null +++ b/api/funkwhale_api/musicbrainz/tests/test_cache.py @@ -0,0 +1,17 @@ +import unittest +from test_plus.test import TestCase + +from funkwhale_api.musicbrainz import client + + +class TestAPI(TestCase): + def test_can_search_recording_in_musicbrainz_api(self, *mocks): + r = {'hello': 'world'} + mocked = 'funkwhale_api.musicbrainz.client._api.search_artists' + with unittest.mock.patch(mocked, return_value=r) as m: + self.assertEqual(client.api.artists.search('test'), r) + # now call from cache + self.assertEqual(client.api.artists.search('test'), r) + self.assertEqual(client.api.artists.search('test'), r) + + self.assertEqual(m.call_count, 1) diff --git a/api/funkwhale_api/musicbrainz/views.py b/api/funkwhale_api/musicbrainz/views.py index fac8089b9..5c101b161 100644 --- a/api/funkwhale_api/musicbrainz/views.py +++ b/api/funkwhale_api/musicbrainz/views.py @@ -2,6 +2,7 @@ from rest_framework import viewsets from rest_framework.views import APIView from rest_framework.response import Response from rest_framework.decorators import list_route +import musicbrainzngs from funkwhale_api.common.permissions import ConditionalAuthentication @@ -44,7 +45,7 @@ class ReleaseBrowse(APIView): def get(self, request, *args, **kwargs): result = api.releases.browse( release_group=kwargs['release_group_uuid'], - includes=['recordings']) + includes=['recordings', 'artist-credits']) return Response(result) @@ -54,17 +55,18 @@ class SearchViewSet(viewsets.ViewSet): @list_route(methods=['get']) def recordings(self, request, *args, **kwargs): query = request.GET['query'] - results = api.recordings.search(query, artist=query) + results = api.recordings.search(query) return Response(results) @list_route(methods=['get']) def releases(self, request, *args, **kwargs): query = request.GET['query'] - results = api.releases.search(query, artist=query) + results = api.releases.search(query) return Response(results) @list_route(methods=['get']) def artists(self, request, *args, **kwargs): query = request.GET['query'] results = api.artists.search(query) + # results = musicbrainzngs.search_artists(query) return Response(results) diff --git a/api/funkwhale_api/providers/youtube/client.py b/api/funkwhale_api/providers/youtube/client.py index 7c0ea326c..792e501d7 100644 --- a/api/funkwhale_api/providers/youtube/client.py +++ b/api/funkwhale_api/providers/youtube/client.py @@ -4,21 +4,20 @@ from apiclient.discovery import build from apiclient.errors import HttpError from oauth2client.tools import argparser -from django.conf import settings +from dynamic_preferences.registries import ( + global_preferences_registry as registry) -# Set DEVELOPER_KEY to the API key value from the APIs & auth > Registered apps -# tab of -# https://cloud.google.com/console -# Please ensure that you have enabled the YouTube Data API for your project. -DEVELOPER_KEY = settings.FUNKWHALE_PROVIDERS['youtube']['api_key'] YOUTUBE_API_SERVICE_NAME = "youtube" YOUTUBE_API_VERSION = "v3" VIDEO_BASE_URL = 'https://www.youtube.com/watch?v={0}' def _do_search(query): - youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, - developerKey=DEVELOPER_KEY) + manager = registry.manager() + youtube = build( + YOUTUBE_API_SERVICE_NAME, + YOUTUBE_API_VERSION, + developerKey=manager['providers_youtube__api_key']) return youtube.search().list( q=query, @@ -55,4 +54,33 @@ class Client(object): return results + def to_funkwhale(self, result): + """ + We convert youtube results to something more generic. + + { + "id": "video id", + "type": "youtube#video", + "url": "https://www.youtube.com/watch?v=id", + "description": "description", + "channelId": "Channel id", + "title": "Title", + "channelTitle": "channel Title", + "publishedAt": "2012-08-22T18:41:03.000Z", + "cover": "http://coverurl" + } + """ + return { + 'id': result['id']['videoId'], + 'url': 'https://www.youtube.com/watch?v={}'.format( + result['id']['videoId']), + 'type': result['id']['kind'], + 'title': result['snippet']['title'], + 'description': result['snippet']['description'], + 'channelId': result['snippet']['channelId'], + 'channelTitle': result['snippet']['channelTitle'], + 'publishedAt': result['snippet']['publishedAt'], + 'cover': result['snippet']['thumbnails']['high']['url'], + } + client = Client() diff --git a/api/funkwhale_api/providers/youtube/dynamic_preferences_registry.py b/api/funkwhale_api/providers/youtube/dynamic_preferences_registry.py new file mode 100644 index 000000000..fc7f7d793 --- /dev/null +++ b/api/funkwhale_api/providers/youtube/dynamic_preferences_registry.py @@ -0,0 +1,13 @@ +from dynamic_preferences.types import StringPreference, Section +from dynamic_preferences.registries import global_preferences_registry + +youtube = Section('providers_youtube') + + +@global_preferences_registry.register +class APIKey(StringPreference): + section = youtube + name = 'api_key' + default = 'CHANGEME' + verbose_name = 'YouTube API key' + help_text = 'The API key used to query YouTube. Get one at https://console.developers.google.com/.' diff --git a/api/funkwhale_api/providers/youtube/tests/test_youtube.py b/api/funkwhale_api/providers/youtube/tests/test_youtube.py index ca0a95628..db6bd8413 100644 --- a/api/funkwhale_api/providers/youtube/tests/test_youtube.py +++ b/api/funkwhale_api/providers/youtube/tests/test_youtube.py @@ -8,7 +8,7 @@ from funkwhale_api.providers.youtube.client import client from . import data as api_data class TestAPI(TestCase): - + maxDiff = None @unittest.mock.patch( 'funkwhale_api.providers.youtube.client._do_search', return_value=api_data.search['8 bit adventure']) @@ -25,11 +25,23 @@ class TestAPI(TestCase): return_value=api_data.search['8 bit adventure']) def test_can_get_search_results_from_funkwhale(self, *mocks): query = '8 bit adventure' - expected = json.dumps(client.search(query)) url = self.reverse('api:v1:providers:youtube:search') response = self.client.get(url + '?query={0}'.format(query)) + # we should cast the youtube result to something more generic + expected = { + "id": "0HxZn6CzOIo", + "url": "https://www.youtube.com/watch?v=0HxZn6CzOIo", + "type": "youtube#video", + "description": "Make sure to apply adhesive evenly before use. GET IT HERE: http://adhesivewombat.bandcamp.com/album/marsupial-madness Facebook: ...", + "channelId": "UCps63j3krzAG4OyXeEyuhFw", + "title": "AdhesiveWombat - 8 Bit Adventure", + "channelTitle": "AdhesiveWombat", + "publishedAt": "2012-08-22T18:41:03.000Z", + "cover": "https://i.ytimg.com/vi/0HxZn6CzOIo/hqdefault.jpg" + } - self.assertJSONEqual(expected, json.loads(response.content.decode('utf-8'))) + self.assertEqual( + json.loads(response.content.decode('utf-8'))[0], expected) @unittest.mock.patch( 'funkwhale_api.providers.youtube.client._do_search', @@ -66,9 +78,22 @@ class TestAPI(TestCase): 'q': '8 bit adventure', } - expected = json.dumps(client.search_multiple(queries)) + expected = { + "id": "0HxZn6CzOIo", + "url": "https://www.youtube.com/watch?v=0HxZn6CzOIo", + "type": "youtube#video", + "description": "Make sure to apply adhesive evenly before use. GET IT HERE: http://adhesivewombat.bandcamp.com/album/marsupial-madness Facebook: ...", + "channelId": "UCps63j3krzAG4OyXeEyuhFw", + "title": "AdhesiveWombat - 8 Bit Adventure", + "channelTitle": "AdhesiveWombat", + "publishedAt": "2012-08-22T18:41:03.000Z", + "cover": "https://i.ytimg.com/vi/0HxZn6CzOIo/hqdefault.jpg" + } + url = self.reverse('api:v1:providers:youtube:searchs') response = self.client.post( url, json.dumps(queries), content_type='application/json') - self.assertJSONEqual(expected, json.loads(response.content.decode('utf-8'))) + self.assertEqual( + expected, + json.loads(response.content.decode('utf-8'))['1'][0]) diff --git a/api/funkwhale_api/providers/youtube/views.py b/api/funkwhale_api/providers/youtube/views.py index 7ad2c2c3d..989b33090 100644 --- a/api/funkwhale_api/providers/youtube/views.py +++ b/api/funkwhale_api/providers/youtube/views.py @@ -10,7 +10,10 @@ class APISearch(APIView): def get(self, request, *args, **kwargs): results = client.search(request.GET['query']) - return Response(results) + return Response([ + client.to_funkwhale(result) + for result in results + ]) class APISearchs(APIView): @@ -18,4 +21,10 @@ class APISearchs(APIView): def post(self, request, *args, **kwargs): results = client.search_multiple(request.data) - return Response(results) + return Response({ + key: [ + client.to_funkwhale(result) + for result in group + ] + for key, group in results.items() + }) diff --git a/api/funkwhale_api/users/models.py b/api/funkwhale_api/users/models.py index 1abbbb51f..c5ca896ab 100644 --- a/api/funkwhale_api/users/models.py +++ b/api/funkwhale_api/users/models.py @@ -19,8 +19,11 @@ class User(AbstractUser): relevant_permissions = { # internal_codename : {external_codename} 'music.add_importbatch': { - 'external_codename': 'import.launch' - } + 'external_codename': 'import.launch', + }, + 'dynamic_preferences.change_globalpreferencemodel': { + 'external_codename': 'settings.change', + }, } def __str__(self): diff --git a/api/funkwhale_api/users/tests/test_views.py b/api/funkwhale_api/users/tests/test_views.py index 6250a7ca7..52826cfa4 100644 --- a/api/funkwhale_api/users/tests/test_views.py +++ b/api/funkwhale_api/users/tests/test_views.py @@ -47,7 +47,13 @@ class UserTestCase(TestCase): # login required self.assertEqual(response.status_code, 401) - user = UserFactory(is_staff=True, perms=['music.add_importbatch']) + user = UserFactory( + is_staff=True, + perms=[ + 'music.add_importbatch', + 'dynamic_preferences.change_globalpreferencemodel', + ] + ) self.assertTrue(user.has_perm('music.add_importbatch')) self.login(user) @@ -63,3 +69,5 @@ class UserTestCase(TestCase): self.assertEqual(payload['name'], user.name) self.assertEqual( payload['permissions']['import.launch']['status'], True) + self.assertEqual( + payload['permissions']['settings.change']['status'], True) diff --git a/api/requirements/base.txt b/api/requirements/base.txt index ae851962a..bdf17cf9a 100644 --- a/api/requirements/base.txt +++ b/api/requirements/base.txt @@ -50,3 +50,9 @@ beautifulsoup4==4.6.0 Markdown==2.6.8 ipython==6.1.0 mutagen==1.38 + + +# Until this is merged +git+https://github.com/EliotBerriot/PyMemoize.git@django + +django-dynamic-preferences>=1.2,<1.3 diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 1a1b81b39..4ffede783 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -6,13 +6,15 @@ services: restart: unless-stopped env_file: .env image: postgres:9.4 + volumes: + - ./data/postgres:/var/lib/postgresql/data redis: restart: unless-stopped env_file: .env image: redis:3 volumes: - - ./data:/data + - ./data/redis:/data celeryworker: restart: unless-stopped @@ -46,6 +48,7 @@ services: - ./data/music:/music:ro - ./data/media:/app/funkwhale_api/media - ./data/static:/app/staticfiles + - ./front/dist:/frontend ports: - "${FUNKWHALE_API_IP:-127.0.0.1}:${FUNKWHALE_API_PORT:-5000}:5000" links: diff --git a/deploy/nginx.conf b/deploy/nginx.conf index 7395e37d9..6a0a9f509 100644 --- a/deploy/nginx.conf +++ b/deploy/nginx.conf @@ -47,7 +47,17 @@ server { location /media/ { alias /srv/funkwhale/data/media/; } + + location /_protected/media { + # this is an internal location that is used to serve + # audio files once correct permission / authentication + # has been checked on API side + internal; + alias /srv/funkwhale/data/media; + } + location /staticfiles/ { + # django static files alias /srv/funkwhale/data/static/; } } diff --git a/dev.yml b/dev.yml index 21b0912e3..f0fc8845a 100644 --- a/dev.yml +++ b/dev.yml @@ -27,7 +27,7 @@ services: env_file: .env.dev build: context: ./api - dockerfile: docker/Dockerfile.local + dockerfile: docker/Dockerfile.test links: - postgres - redis @@ -53,12 +53,14 @@ services: - redis - celeryworker - # nginx: - # env_file: .env.dev - # build: ./api/compose/nginx - # links: - # - api - # volumes: - # - ./api/funkwhale_api/media:/staticfiles/media - # ports: - # - "0.0.0.0:6001:80" + nginx: + env_file: .env.dev + image: nginx + links: + - api + - front + volumes: + - ./docker/nginx/conf.dev:/etc/nginx/nginx.conf + - ./api/funkwhale_api/media:/protected/media + ports: + - "0.0.0.0:6001:80" diff --git a/api/compose/nginx/nginx.conf b/docker/nginx/conf.dev similarity index 72% rename from api/compose/nginx/nginx.conf rename to docker/nginx/conf.dev index 331d8d45f..6ca395fb1 100644 --- a/api/compose/nginx/nginx.conf +++ b/docker/nginx/conf.dev @@ -27,27 +27,21 @@ http { #gzip on; - upstream app { - server django:12081; - } - server { listen 80; charset utf-8; - root /staticfiles; + location /_protected/media { + internal; + alias /protected/media; + } location / { - # checks for static file, if not found proxy to app - try_files $uri @proxy_to_app; - } - - location @proxy_to_app { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; proxy_redirect off; - - proxy_pass http://app; + proxy_pass http://api:12081/; } - } } diff --git a/docs/changelog.rst b/docs/changelog.rst index c4092dc8b..6e609aac9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,24 @@ Changelog ========= +0.2 +------- + +2017-07-09 + +* [feature] can now import artist and releases from youtube and musicbrainz. + This requires a YouTube API key for the search +* [breaking] we now check for user permission before serving audio files, which requires +a specific configuration block in your reverse proxy configuration: + +.. code-block:: + + location /_protected/media { + internal; + alias /srv/funkwhale/data/media; + } + + 0.1 ------- diff --git a/docs/installation/docker.rst b/docs/installation/docker.rst index 9f7a288f3..76958fb0b 100644 --- a/docs/installation/docker.rst +++ b/docs/installation/docker.rst @@ -46,7 +46,7 @@ Then launch the whole thing: docker-compose up -d -Now, you just need to setup the :ref:`frontend files `, and configure your :ref:`reverse-proxy `. Don't worry, it's quite easy. +Now, you just need to configure your :ref:`reverse-proxy `. Don't worry, it's quite easy. About music acquisition ----------------------- diff --git a/docs/installation/index.rst b/docs/installation/index.rst index 33ac3bb75..1544dfbf0 100644 --- a/docs/installation/index.rst +++ b/docs/installation/index.rst @@ -26,6 +26,11 @@ Available installation methods Frontend setup --------------- +.. note:: + + You do not need to do this if you are deploying using Docker, as frontend files + are already included in the funkwhale docker image. + Files for the web frontend are purely static and can simply be downloaded, unzipped and served from any webserver: .. code-block:: bash diff --git a/front/package.json b/front/package.json index 732fdb406..7cec50319 100644 --- a/front/package.json +++ b/front/package.json @@ -19,7 +19,8 @@ "semantic-ui-css": "^2.2.10", "vue": "^2.3.3", "vue-resource": "^1.3.4", - "vue-router": "^2.3.1" + "vue-router": "^2.3.1", + "vuedraggable": "^2.14.1" }, "devDependencies": { "autoprefixer": "^6.7.2", diff --git a/front/src/App.vue b/front/src/App.vue index 2704ad151..f81d7d3da 100644 --- a/front/src/App.vue +++ b/front/src/App.vue @@ -2,6 +2,26 @@
+
+
@@ -27,12 +47,16 @@ export default { -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; } -.main.pusher { +.main.pusher, .footer { margin-left: 350px !important; transform: none !important; +} +.main-pusher { padding: 1.5rem 0; } - +#footer { + padding: 1.5rem; +} .ui.stripe.segment { padding: 4em; } diff --git a/front/src/audio/index.js b/front/src/audio/index.js index 22cc85ad3..48f610443 100644 --- a/front/src/audio/index.js +++ b/front/src/audio/index.js @@ -1,12 +1,5 @@ import logger from '@/logging' - -const pad = (val) => { - val = Math.floor(val) - if (val < 10) { - return '0' + val - } - return val + '' -} +import time from '@/utils/time' const Cov = { on (el, type, func) { @@ -108,7 +101,7 @@ class Audio { }) this.state.duration = Math.round(this.$Audio.duration * 100) / 100 this.state.loaded = Math.round(10000 * this.$Audio.buffered.end(0) / this.$Audio.duration) / 100 - this.state.durationTimerFormat = this.timeParse(this.state.duration) + this.state.durationTimerFormat = time.parse(this.state.duration) } updatePlayState (e) { @@ -116,9 +109,9 @@ class Audio { this.state.duration = Math.round(this.$Audio.duration * 100) / 100 this.state.progress = Math.round(10000 * this.state.currentTime / this.state.duration) / 100 - this.state.durationTimerFormat = this.timeParse(this.state.duration) - this.state.currentTimeFormat = this.timeParse(this.state.currentTime) - this.state.lastTimeFormat = this.timeParse(this.state.duration - this.state.currentTime) + this.state.durationTimerFormat = time.parse(this.state.duration) + this.state.currentTimeFormat = time.parse(this.state.currentTime) + this.state.lastTimeFormat = time.parse(this.state.duration - this.state.currentTime) this.hook.playState.forEach(func => { func(this.state) @@ -181,14 +174,6 @@ class Audio { } this.$Audio.currentTime = time } - - timeParse (sec) { - let min = 0 - min = Math.floor(sec / 60) - sec = sec - min * 60 - return pad(min) + ':' + pad(sec) - } - } export default Audio diff --git a/front/src/audio/queue.js b/front/src/audio/queue.js index ba0af486f..efa3dcdf7 100644 --- a/front/src/audio/queue.js +++ b/front/src/audio/queue.js @@ -5,6 +5,8 @@ import Audio from '@/audio' import backend from '@/audio/backend' import radios from '@/radios' import Vue from 'vue' +import url from '@/utils/url' +import auth from '@/auth' class Queue { constructor (options = {}) { @@ -92,6 +94,24 @@ class Queue { } cache.set('volume', newValue) } + + reorder (oldIndex, newIndex) { + // called when the user uses drag / drop to reorder + // tracks in queue + if (oldIndex === this.currentIndex) { + this.currentIndex = newIndex + return + } + if (oldIndex < this.currentIndex && newIndex >= this.currentIndex) { + // item before was moved after + this.currentIndex -= 1 + } + if (oldIndex > this.currentIndex && newIndex <= this.currentIndex) { + // item after was moved before + this.currentIndex += 1 + } + } + append (track, index) { this.previousQueue = null index = index || this.tracks.length @@ -163,7 +183,17 @@ class Queue { if (!file) { return this.next() } - this.audio = new Audio(backend.absoluteUrl(file.path), { + let path = backend.absoluteUrl(file.path) + + if (auth.user.authenticated) { + // we need to send the token directly in url + // so authentication can be checked by the backend + // because for audio files we cannot use the regular Authentication + // header + path = url.updateQueryString(path, 'jwt', auth.getAuthToken()) + } + + this.audio = new Audio(path, { preload: true, autoplay: true, rate: 1, diff --git a/front/src/auth/index.js b/front/src/auth/index.js index b5a3fb5ad..802369428 100644 --- a/front/src/auth/index.js +++ b/front/src/auth/index.js @@ -10,14 +10,13 @@ const LOGIN_URL = config.API_URL + 'token/' const USER_PROFILE_URL = config.API_URL + 'users/users/me/' // const SIGNUP_URL = API_URL + 'users/' -export default { - - // User object will let us check authentication status - user: { - authenticated: false, - username: '', - profile: null - }, +let userData = { + authenticated: false, + username: '', + availablePermissions: {}, + profile: {} +} +let auth = { // Send a request to the login URL and save the returned JWT login (context, creds, redirect, onError) { @@ -50,7 +49,7 @@ export default { checkAuth () { logger.default.info('Checking authentication...') - var jwt = cache.get('token') + var jwt = this.getAuthToken() var username = cache.get('username') if (jwt) { this.user.authenticated = true @@ -63,9 +62,13 @@ export default { } }, + getAuthToken () { + return cache.get('token') + }, + // The object to be passed as a header for authenticated requests getAuthHeader () { - return 'JWT ' + cache.get('token') + return 'JWT ' + this.getAuthToken() }, fetchProfile () { @@ -83,7 +86,14 @@ export default { let self = this this.fetchProfile().then(data => { Vue.set(self.user, 'profile', data) + Object.keys(data.permissions).forEach(function (key) { + // this makes it easier to check for permissions in templates + Vue.set(self.user.availablePermissions, key, data.permissions[String(key)].status) + }) }) favoriteTracks.fetch() } } + +Vue.set(auth, 'user', userData) +export default auth diff --git a/front/src/components/Home.vue b/front/src/components/Home.vue index 891e99ae0..dd324943f 100644 --- a/front/src/components/Home.vue +++ b/front/src/components/Home.vue @@ -6,7 +6,7 @@ Welcome on funkwhale

We think listening music should be simple.

- + Get me to the library @@ -60,7 +60,7 @@

Clean library

-

Funkwhale takes care of fealing your music.

+

Funkwhale takes care of handling your music.

@@ -90,9 +90,9 @@

Funkwhale is dead simple to use.

- +
- No add-ons, no plugins : you only need a web browser + No add-ons, no plugins : you only need a web libraryr
diff --git a/front/src/components/Sidebar.vue b/front/src/components/Sidebar.vue index c98dc2f01..e39dd16b9 100644 --- a/front/src/components/Sidebar.vue +++ b/front/src/components/Sidebar.vue @@ -13,7 +13,7 @@
- Browse + Browse Queue