diff --git a/.env.dev b/.env.dev index a7413b0ff..bc2d667b1 100644 --- a/.env.dev +++ b/.env.dev @@ -1,3 +1,3 @@ BACKEND_URL=http://localhost:6001 -YOUTUBE_API_KEY= API_AUTHENTICATION_REQUIRED=True +CACHALOT_ENABLED=False diff --git a/api/config/api_urls.py b/api/config/api_urls.py index 9c612c94d..5ed4cffdd 100644 --- a/api/config/api_urls.py +++ b/api/config/api_urls.py @@ -4,8 +4,11 @@ from funkwhale_api.music import views from funkwhale_api.playlists import views as playlists_views from rest_framework_jwt import views as jwt_views +from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet +from dynamic_preferences.users.viewsets import UserPreferencesViewSet router = routers.SimpleRouter() +router.register(r'settings', GlobalPreferencesViewSet, base_name='settings') router.register(r'tags', views.TagViewSet, 'tags') router.register(r'tracks', views.TrackViewSet, 'tracks') router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles') @@ -14,17 +17,27 @@ router.register(r'albums', views.AlbumViewSet, 'albums') router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches') router.register(r'submit', views.SubmitViewSet, 'submit') router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists') -router.register(r'playlist-tracks', playlists_views.PlaylistTrackViewSet, 'playlist-tracks') +router.register( + r'playlist-tracks', + playlists_views.PlaylistTrackViewSet, + 'playlist-tracks') v1_patterns = router.urls v1_patterns += [ - url(r'^providers/', include('funkwhale_api.providers.urls', namespace='providers')), - url(r'^favorites/', include('funkwhale_api.favorites.urls', namespace='favorites')), - url(r'^search$', views.Search.as_view(), name='search'), - url(r'^radios/', include('funkwhale_api.radios.urls', namespace='radios')), - url(r'^history/', include('funkwhale_api.history.urls', namespace='history')), - url(r'^users/', include('funkwhale_api.users.api_urls', namespace='users')), - url(r'^token/', jwt_views.obtain_jwt_token), + url(r'^providers/', + include('funkwhale_api.providers.urls', namespace='providers')), + url(r'^favorites/', + include('funkwhale_api.favorites.urls', namespace='favorites')), + url(r'^search$', + views.Search.as_view(), name='search'), + url(r'^radios/', + include('funkwhale_api.radios.urls', namespace='radios')), + url(r'^history/', + include('funkwhale_api.history.urls', namespace='history')), + url(r'^users/', + include('funkwhale_api.users.api_urls', namespace='users')), + url(r'^token/', + jwt_views.obtain_jwt_token), url(r'^token/refresh/', jwt_views.refresh_jwt_token), ] diff --git a/api/config/settings/common.py b/api/config/settings/common.py index b6e195ca2..70804c3c9 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -53,6 +53,7 @@ THIRD_PARTY_APPS = ( 'rest_auth', 'rest_auth.registration', 'mptt', + 'dynamic_preferences', ) # Apps specific for this project go here. @@ -65,6 +66,7 @@ LOCAL_APPS = ( 'funkwhale_api.history', 'funkwhale_api.playlists', 'funkwhale_api.providers.audiofile', + 'funkwhale_api.providers.youtube', ) # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps @@ -298,11 +300,6 @@ REST_FRAMEWORK = { ) } -FUNKWHALE_PROVIDERS = { - 'youtube': { - 'api_key': env('YOUTUBE_API_KEY', default='REPLACE_ME') - } -} ATOMIC_REQUESTS = False # Wether we should check user permission before serving audio files (meaning @@ -314,3 +311,13 @@ PROTECT_AUDIO_FILES = env.bool('PROTECT_AUDIO_FILES', default=True) # Which path will be used to process the internal redirection # **DO NOT** put a slash at the end PROTECT_FILES_PATH = env('PROTECT_FILES_PATH', default='/_protected') + + +# use this setting to tweak for how long you want to cache +# musicbrainz results. (value is in seconds) +MUSICBRAINZ_CACHE_DURATION = env.int( + 'MUSICBRAINZ_CACHE_DURATION', + default=300 +) + +CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True) diff --git a/api/funkwhale_api/music/metadata.py b/api/funkwhale_api/music/metadata.py index d1be9a4e1..31d13d495 100644 --- a/api/funkwhale_api/music/metadata.py +++ b/api/funkwhale_api/music/metadata.py @@ -37,13 +37,26 @@ def get_mp3_recording_id(f, k): except IndexError: raise TagNotFound(k) + +def convert_track_number(v): + try: + return int(v) + except ValueError: + # maybe the position is of the form "1/4" + pass + + try: + return int(v.split('/')[0]) + except (ValueError, AttributeError, IndexError): + pass + CONF = { 'OggVorbis': { 'getter': lambda f, k: f[k][0], 'fields': { 'track_number': { 'field': 'TRACKNUMBER', - 'to_application': int + 'to_application': convert_track_number }, 'title': { 'field': 'title' @@ -74,7 +87,7 @@ CONF = { 'fields': { 'track_number': { 'field': 'TPOS', - 'to_application': lambda v: int(v.split('/')[0]) + 'to_application': convert_track_number }, 'title': { 'field': 'TIT2' diff --git a/api/funkwhale_api/musicbrainz/client.py b/api/funkwhale_api/musicbrainz/client.py index e281555b2..049ed298c 100644 --- a/api/funkwhale_api/musicbrainz/client.py +++ b/api/funkwhale_api/musicbrainz/client.py @@ -1,11 +1,17 @@ import musicbrainzngs +import memoize.djangocache from django.conf import settings from funkwhale_api import __version__ + _api = musicbrainzngs _api.set_useragent('funkwhale', str(__version__), 'contact@eliotberriot.com') +store = memoize.djangocache.Cache('default') +memo = memoize.Memoizer(store, namespace='memoize:musicbrainz') + + def clean_artist_search(query, **kwargs): cleaned_kwargs = {} if kwargs.get('name'): @@ -17,30 +23,55 @@ class API(object): _api = _api class artists(object): - search = clean_artist_search - get = _api.get_artist_by_id + search = memo( + clean_artist_search, max_age=settings.MUSICBRAINZ_CACHE_DURATION) + get = memo( + _api.get_artist_by_id, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) class images(object): - get_front = _api.get_image_front + get_front = memo( + _api.get_image_front, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) class recordings(object): - search = _api.search_recordings - get = _api.get_recording_by_id + search = memo( + _api.search_recordings, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + get = memo( + _api.get_recording_by_id, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) class works(object): - search = _api.search_works - get = _api.get_work_by_id + search = memo( + _api.search_works, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + get = memo( + _api.get_work_by_id, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) class releases(object): - search = _api.search_releases - get = _api.get_release_by_id - browse = _api.browse_releases + search = memo( + _api.search_releases, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + get = memo( + _api.get_release_by_id, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + browse = memo( + _api.browse_releases, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) # get_image_front = _api.get_image_front class release_groups(object): - search = _api.search_release_groups - get = _api.get_release_group_by_id - browse = _api.browse_release_groups + search = memo( + _api.search_release_groups, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + get = memo( + _api.get_release_group_by_id, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) + browse = memo( + _api.browse_release_groups, + max_age=settings.MUSICBRAINZ_CACHE_DURATION) # get_image_front = _api.get_image_front api = API() diff --git a/api/funkwhale_api/musicbrainz/tests/test_cache.py b/api/funkwhale_api/musicbrainz/tests/test_cache.py new file mode 100644 index 000000000..d2d1260ec --- /dev/null +++ b/api/funkwhale_api/musicbrainz/tests/test_cache.py @@ -0,0 +1,17 @@ +import unittest +from test_plus.test import TestCase + +from funkwhale_api.musicbrainz import client + + +class TestAPI(TestCase): + def test_can_search_recording_in_musicbrainz_api(self, *mocks): + r = {'hello': 'world'} + mocked = 'funkwhale_api.musicbrainz.client._api.search_artists' + with unittest.mock.patch(mocked, return_value=r) as m: + self.assertEqual(client.api.artists.search('test'), r) + # now call from cache + self.assertEqual(client.api.artists.search('test'), r) + self.assertEqual(client.api.artists.search('test'), r) + + self.assertEqual(m.call_count, 1) diff --git a/api/funkwhale_api/musicbrainz/views.py b/api/funkwhale_api/musicbrainz/views.py index fac8089b9..5c101b161 100644 --- a/api/funkwhale_api/musicbrainz/views.py +++ b/api/funkwhale_api/musicbrainz/views.py @@ -2,6 +2,7 @@ from rest_framework import viewsets from rest_framework.views import APIView from rest_framework.response import Response from rest_framework.decorators import list_route +import musicbrainzngs from funkwhale_api.common.permissions import ConditionalAuthentication @@ -44,7 +45,7 @@ class ReleaseBrowse(APIView): def get(self, request, *args, **kwargs): result = api.releases.browse( release_group=kwargs['release_group_uuid'], - includes=['recordings']) + includes=['recordings', 'artist-credits']) return Response(result) @@ -54,17 +55,18 @@ class SearchViewSet(viewsets.ViewSet): @list_route(methods=['get']) def recordings(self, request, *args, **kwargs): query = request.GET['query'] - results = api.recordings.search(query, artist=query) + results = api.recordings.search(query) return Response(results) @list_route(methods=['get']) def releases(self, request, *args, **kwargs): query = request.GET['query'] - results = api.releases.search(query, artist=query) + results = api.releases.search(query) return Response(results) @list_route(methods=['get']) def artists(self, request, *args, **kwargs): query = request.GET['query'] results = api.artists.search(query) + # results = musicbrainzngs.search_artists(query) return Response(results) diff --git a/api/funkwhale_api/providers/youtube/client.py b/api/funkwhale_api/providers/youtube/client.py index 7c0ea326c..792e501d7 100644 --- a/api/funkwhale_api/providers/youtube/client.py +++ b/api/funkwhale_api/providers/youtube/client.py @@ -4,21 +4,20 @@ from apiclient.discovery import build from apiclient.errors import HttpError from oauth2client.tools import argparser -from django.conf import settings +from dynamic_preferences.registries import ( + global_preferences_registry as registry) -# Set DEVELOPER_KEY to the API key value from the APIs & auth > Registered apps -# tab of -# https://cloud.google.com/console -# Please ensure that you have enabled the YouTube Data API for your project. -DEVELOPER_KEY = settings.FUNKWHALE_PROVIDERS['youtube']['api_key'] YOUTUBE_API_SERVICE_NAME = "youtube" YOUTUBE_API_VERSION = "v3" VIDEO_BASE_URL = 'https://www.youtube.com/watch?v={0}' def _do_search(query): - youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, - developerKey=DEVELOPER_KEY) + manager = registry.manager() + youtube = build( + YOUTUBE_API_SERVICE_NAME, + YOUTUBE_API_VERSION, + developerKey=manager['providers_youtube__api_key']) return youtube.search().list( q=query, @@ -55,4 +54,33 @@ class Client(object): return results + def to_funkwhale(self, result): + """ + We convert youtube results to something more generic. + + { + "id": "video id", + "type": "youtube#video", + "url": "https://www.youtube.com/watch?v=id", + "description": "description", + "channelId": "Channel id", + "title": "Title", + "channelTitle": "channel Title", + "publishedAt": "2012-08-22T18:41:03.000Z", + "cover": "http://coverurl" + } + """ + return { + 'id': result['id']['videoId'], + 'url': 'https://www.youtube.com/watch?v={}'.format( + result['id']['videoId']), + 'type': result['id']['kind'], + 'title': result['snippet']['title'], + 'description': result['snippet']['description'], + 'channelId': result['snippet']['channelId'], + 'channelTitle': result['snippet']['channelTitle'], + 'publishedAt': result['snippet']['publishedAt'], + 'cover': result['snippet']['thumbnails']['high']['url'], + } + client = Client() diff --git a/api/funkwhale_api/providers/youtube/dynamic_preferences_registry.py b/api/funkwhale_api/providers/youtube/dynamic_preferences_registry.py new file mode 100644 index 000000000..fc7f7d793 --- /dev/null +++ b/api/funkwhale_api/providers/youtube/dynamic_preferences_registry.py @@ -0,0 +1,13 @@ +from dynamic_preferences.types import StringPreference, Section +from dynamic_preferences.registries import global_preferences_registry + +youtube = Section('providers_youtube') + + +@global_preferences_registry.register +class APIKey(StringPreference): + section = youtube + name = 'api_key' + default = 'CHANGEME' + verbose_name = 'YouTube API key' + help_text = 'The API key used to query YouTube. Get one at https://console.developers.google.com/.' diff --git a/api/funkwhale_api/providers/youtube/tests/test_youtube.py b/api/funkwhale_api/providers/youtube/tests/test_youtube.py index ca0a95628..db6bd8413 100644 --- a/api/funkwhale_api/providers/youtube/tests/test_youtube.py +++ b/api/funkwhale_api/providers/youtube/tests/test_youtube.py @@ -8,7 +8,7 @@ from funkwhale_api.providers.youtube.client import client from . import data as api_data class TestAPI(TestCase): - + maxDiff = None @unittest.mock.patch( 'funkwhale_api.providers.youtube.client._do_search', return_value=api_data.search['8 bit adventure']) @@ -25,11 +25,23 @@ class TestAPI(TestCase): return_value=api_data.search['8 bit adventure']) def test_can_get_search_results_from_funkwhale(self, *mocks): query = '8 bit adventure' - expected = json.dumps(client.search(query)) url = self.reverse('api:v1:providers:youtube:search') response = self.client.get(url + '?query={0}'.format(query)) + # we should cast the youtube result to something more generic + expected = { + "id": "0HxZn6CzOIo", + "url": "https://www.youtube.com/watch?v=0HxZn6CzOIo", + "type": "youtube#video", + "description": "Make sure to apply adhesive evenly before use. GET IT HERE: http://adhesivewombat.bandcamp.com/album/marsupial-madness Facebook: ...", + "channelId": "UCps63j3krzAG4OyXeEyuhFw", + "title": "AdhesiveWombat - 8 Bit Adventure", + "channelTitle": "AdhesiveWombat", + "publishedAt": "2012-08-22T18:41:03.000Z", + "cover": "https://i.ytimg.com/vi/0HxZn6CzOIo/hqdefault.jpg" + } - self.assertJSONEqual(expected, json.loads(response.content.decode('utf-8'))) + self.assertEqual( + json.loads(response.content.decode('utf-8'))[0], expected) @unittest.mock.patch( 'funkwhale_api.providers.youtube.client._do_search', @@ -66,9 +78,22 @@ class TestAPI(TestCase): 'q': '8 bit adventure', } - expected = json.dumps(client.search_multiple(queries)) + expected = { + "id": "0HxZn6CzOIo", + "url": "https://www.youtube.com/watch?v=0HxZn6CzOIo", + "type": "youtube#video", + "description": "Make sure to apply adhesive evenly before use. GET IT HERE: http://adhesivewombat.bandcamp.com/album/marsupial-madness Facebook: ...", + "channelId": "UCps63j3krzAG4OyXeEyuhFw", + "title": "AdhesiveWombat - 8 Bit Adventure", + "channelTitle": "AdhesiveWombat", + "publishedAt": "2012-08-22T18:41:03.000Z", + "cover": "https://i.ytimg.com/vi/0HxZn6CzOIo/hqdefault.jpg" + } + url = self.reverse('api:v1:providers:youtube:searchs') response = self.client.post( url, json.dumps(queries), content_type='application/json') - self.assertJSONEqual(expected, json.loads(response.content.decode('utf-8'))) + self.assertEqual( + expected, + json.loads(response.content.decode('utf-8'))['1'][0]) diff --git a/api/funkwhale_api/providers/youtube/views.py b/api/funkwhale_api/providers/youtube/views.py index 7ad2c2c3d..989b33090 100644 --- a/api/funkwhale_api/providers/youtube/views.py +++ b/api/funkwhale_api/providers/youtube/views.py @@ -10,7 +10,10 @@ class APISearch(APIView): def get(self, request, *args, **kwargs): results = client.search(request.GET['query']) - return Response(results) + return Response([ + client.to_funkwhale(result) + for result in results + ]) class APISearchs(APIView): @@ -18,4 +21,10 @@ class APISearchs(APIView): def post(self, request, *args, **kwargs): results = client.search_multiple(request.data) - return Response(results) + return Response({ + key: [ + client.to_funkwhale(result) + for result in group + ] + for key, group in results.items() + }) diff --git a/api/funkwhale_api/users/models.py b/api/funkwhale_api/users/models.py index 1abbbb51f..c5ca896ab 100644 --- a/api/funkwhale_api/users/models.py +++ b/api/funkwhale_api/users/models.py @@ -19,8 +19,11 @@ class User(AbstractUser): relevant_permissions = { # internal_codename : {external_codename} 'music.add_importbatch': { - 'external_codename': 'import.launch' - } + 'external_codename': 'import.launch', + }, + 'dynamic_preferences.change_globalpreferencemodel': { + 'external_codename': 'settings.change', + }, } def __str__(self): diff --git a/api/funkwhale_api/users/tests/test_views.py b/api/funkwhale_api/users/tests/test_views.py index 6250a7ca7..52826cfa4 100644 --- a/api/funkwhale_api/users/tests/test_views.py +++ b/api/funkwhale_api/users/tests/test_views.py @@ -47,7 +47,13 @@ class UserTestCase(TestCase): # login required self.assertEqual(response.status_code, 401) - user = UserFactory(is_staff=True, perms=['music.add_importbatch']) + user = UserFactory( + is_staff=True, + perms=[ + 'music.add_importbatch', + 'dynamic_preferences.change_globalpreferencemodel', + ] + ) self.assertTrue(user.has_perm('music.add_importbatch')) self.login(user) @@ -63,3 +69,5 @@ class UserTestCase(TestCase): self.assertEqual(payload['name'], user.name) self.assertEqual( payload['permissions']['import.launch']['status'], True) + self.assertEqual( + payload['permissions']['settings.change']['status'], True) diff --git a/api/requirements/base.txt b/api/requirements/base.txt index ae851962a..bdf17cf9a 100644 --- a/api/requirements/base.txt +++ b/api/requirements/base.txt @@ -50,3 +50,9 @@ beautifulsoup4==4.6.0 Markdown==2.6.8 ipython==6.1.0 mutagen==1.38 + + +# Until this is merged +git+https://github.com/EliotBerriot/PyMemoize.git@django + +django-dynamic-preferences>=1.2,<1.3 diff --git a/dev.yml b/dev.yml index 712288492..f0fc8845a 100644 --- a/dev.yml +++ b/dev.yml @@ -27,7 +27,7 @@ services: env_file: .env.dev build: context: ./api - dockerfile: docker/Dockerfile.local + dockerfile: docker/Dockerfile.test links: - postgres - redis diff --git a/front/src/audio/index.js b/front/src/audio/index.js index 22cc85ad3..48f610443 100644 --- a/front/src/audio/index.js +++ b/front/src/audio/index.js @@ -1,12 +1,5 @@ import logger from '@/logging' - -const pad = (val) => { - val = Math.floor(val) - if (val < 10) { - return '0' + val - } - return val + '' -} +import time from '@/utils/time' const Cov = { on (el, type, func) { @@ -108,7 +101,7 @@ class Audio { }) this.state.duration = Math.round(this.$Audio.duration * 100) / 100 this.state.loaded = Math.round(10000 * this.$Audio.buffered.end(0) / this.$Audio.duration) / 100 - this.state.durationTimerFormat = this.timeParse(this.state.duration) + this.state.durationTimerFormat = time.parse(this.state.duration) } updatePlayState (e) { @@ -116,9 +109,9 @@ class Audio { this.state.duration = Math.round(this.$Audio.duration * 100) / 100 this.state.progress = Math.round(10000 * this.state.currentTime / this.state.duration) / 100 - this.state.durationTimerFormat = this.timeParse(this.state.duration) - this.state.currentTimeFormat = this.timeParse(this.state.currentTime) - this.state.lastTimeFormat = this.timeParse(this.state.duration - this.state.currentTime) + this.state.durationTimerFormat = time.parse(this.state.duration) + this.state.currentTimeFormat = time.parse(this.state.currentTime) + this.state.lastTimeFormat = time.parse(this.state.duration - this.state.currentTime) this.hook.playState.forEach(func => { func(this.state) @@ -181,14 +174,6 @@ class Audio { } this.$Audio.currentTime = time } - - timeParse (sec) { - let min = 0 - min = Math.floor(sec / 60) - sec = sec - min * 60 - return pad(min) + ':' + pad(sec) - } - } export default Audio diff --git a/front/src/auth/index.js b/front/src/auth/index.js index 219a1531f..802369428 100644 --- a/front/src/auth/index.js +++ b/front/src/auth/index.js @@ -10,14 +10,13 @@ const LOGIN_URL = config.API_URL + 'token/' const USER_PROFILE_URL = config.API_URL + 'users/users/me/' // const SIGNUP_URL = API_URL + 'users/' -export default { - - // User object will let us check authentication status - user: { - authenticated: false, - username: '', - profile: null - }, +let userData = { + authenticated: false, + username: '', + availablePermissions: {}, + profile: {} +} +let auth = { // Send a request to the login URL and save the returned JWT login (context, creds, redirect, onError) { @@ -87,7 +86,14 @@ export default { let self = this this.fetchProfile().then(data => { Vue.set(self.user, 'profile', data) + Object.keys(data.permissions).forEach(function (key) { + // this makes it easier to check for permissions in templates + Vue.set(self.user.availablePermissions, key, data.permissions[String(key)].status) + }) }) favoriteTracks.fetch() } } + +Vue.set(auth, 'user', userData) +export default auth diff --git a/front/src/components/Home.vue b/front/src/components/Home.vue index ea0f5edc0..dd324943f 100644 --- a/front/src/components/Home.vue +++ b/front/src/components/Home.vue @@ -6,7 +6,7 @@ Welcome on funkwhale

We think listening music should be simple.

- + Get me to the library @@ -90,9 +90,9 @@

Funkwhale is dead simple to use.

- +
- No add-ons, no plugins : you only need a web browser + No add-ons, no plugins : you only need a web libraryr
diff --git a/front/src/components/Sidebar.vue b/front/src/components/Sidebar.vue index 90e6d2d06..e39dd16b9 100644 --- a/front/src/components/Sidebar.vue +++ b/front/src/components/Sidebar.vue @@ -13,7 +13,7 @@
- Browse + Browse Queue