From 3af3327311af2d9bf5cc9756fc63d4b9b01df49e Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 22 Mar 2018 14:18:55 +0100 Subject: [PATCH 01/83] Added missing choice of version in the docker deployment guide --- docs/installation/docker.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installation/docker.rst b/docs/installation/docker.rst index 34e8187c5..dc031caed 100644 --- a/docs/installation/docker.rst +++ b/docs/installation/docker.rst @@ -17,7 +17,9 @@ Create your env file: .. parsed-literal:: + export FUNKWHALE_VERSION="|version|" curl -L -o .env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/env.prod.sample" + sed -i "s/FUNKWHALE_VERSION=latest/FUNKWHALE_VERSION=$FUNKWHALE_VERSION/" .env Ensure to edit it to match your needs (this file is heavily commented) From 2cfe4fa3a12d12c1b285e80856350d92eb82a4fb Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 22 Mar 2018 14:19:26 +0100 Subject: [PATCH 02/83] Fixed wrong mv command in debian guide --- docs/installation/debian.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/debian.rst b/docs/installation/debian.rst index 86ccb4dd3..c4e54218d 100644 --- a/docs/installation/debian.rst +++ b/docs/installation/debian.rst @@ -89,7 +89,7 @@ First, we'll download the latest api release. curl -L -o "api-|version|.zip" "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/|version|/download?job=build_api" unzip "api-|version|.zip" -d extracted - mv extracted/api api + mv extracted/api/* api/ rmdir extracted @@ -100,7 +100,7 @@ Then we'll download the frontend files: curl -L -o "front-|version|.zip" "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/|version|/download?job=build_front" unzip "front-|version|.zip" -d extracted mv extracted/front . - rmdir extracted + rm -rf extracted You can leave the ZIP archives in the directory, this will help you know which version you've installed next time you want to upgrade your installation. From 1c251ac7c7ffbbef4b202c6903a8875bd8ff91ad Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 22 Mar 2018 14:22:39 +0100 Subject: [PATCH 03/83] Fix #127: Added upgrade documentation --- changes/changelog.d/127.doc | 0 docs/index.rst | 1 + docs/upgrading.rst | 88 +++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 changes/changelog.d/127.doc create mode 100644 docs/upgrading.rst diff --git a/changes/changelog.d/127.doc b/changes/changelog.d/127.doc new file mode 100644 index 000000000..e69de29bb diff --git a/docs/index.rst b/docs/index.rst index 17e9fe7f0..f5acecce5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,6 +15,7 @@ Funkwhale is a self-hosted, modern free and open-source music server, heavily in installation/index configuration importing-music + upgrading changelog Indices and tables diff --git a/docs/upgrading.rst b/docs/upgrading.rst new file mode 100644 index 000000000..24f5502a6 --- /dev/null +++ b/docs/upgrading.rst @@ -0,0 +1,88 @@ +Upgrading your funkwhale instance to a newer version +==================================================== + +.. note:: + + Before upgrading your instance, we strongly advise you to make a database + backup. We're commited to make upgrade as easy and straightforward as possible, + however, funkwhale is still in development and you'll be safer with a backup. + +Reading the release notes +------------------------- + +Please take a few minutes to read the :doc:`changelog`: updates should work +similarly from version to version, but some of them may require additional steps. +Those steps would be described in the version release notes. + +Upgrade the static files +------------------------ + +Regardless of your deployment choice (docker/non-docker) the front-end app +is updated separately from the API. This is as simple as downloading +the zip with the static files and extracting it in the correct place. + +The following example assume your setup match :ref:`frontend-setup`. + +.. parsed-literal:: + + # this assumes you want to upgrade to version "|version|" + export FUNKWHALE_VERSION="|version|" + cd /srv/funkwhale + curl -L -o front.zip "https://code.eliotberriot.com/funkwhale/funkwhale/builds/artifacts/$FUNKWHALE_VERSION/download?job=build_front" + unzip -o front.zip + rm front.zip + +Upgrading the API +----------------- + +Docker setup +^^^^^^^^^^^^ + +If you've followed the setup instructions in :doc:`Docker`, upgrade path is +easy: + +.. parsed-literal:: + + cd /srv/funkwhale + # hardcode the targeted version your env file + # (look for the FUNKWHALE_VERSION variable) + nano .env + # Pull the new version containers + docker-compose pull + # Apply the database migrations + docker-compose run --rm api python manage.py migrate + # Relaunch the containers + docker-compose up -d + +Non-docker setup +^^^^^^^^^^^^^^^^ + +On non docker-setup, upgrade involves a few more commands. We assume your setup +match what is described in :doc:`debian`: + +.. parsed-literal:: + + # this assumes you want to upgrade to version "|version|" + export FUNKWALE_VERSION="|version|" + cd /srv/funkwhale + + # download more recent API files + curl -L -o "api-|version|.zip" "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/|version|/download?job=build_api" + unzip "api-|version|.zip" -d extracted + mv extracted/api/* api/ + rm -rf extracted + + # update os dependencies + sudo api/install_os_dependencies.sh install + # update python dependencies + source /srv/funkwhale/load_env + source /srv/funkwhale/virtualenv/bin/activate + pip install -r api/requirements.txt + + # apply database migrations + python api/manage.py migrate + # collect static files + python api/manage.py collectstatic --no-input + + # restart the services + sudo systemctl restart funkwhale.target From c89b957f05c42fefc5d14d4a74b6fc4dd30b2c66 Mon Sep 17 00:00:00 2001 From: Maxence Bothorel Date: Thu, 22 Mar 2018 18:06:43 +0100 Subject: [PATCH 04/83] convert favicon from ico to png file --- front/static/favicon.png | Bin 8190 -> 991 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/front/static/favicon.png b/front/static/favicon.png index 089442fab7cb4e68cec9de87d421b64495e3263a..0220a4202cf807816c57b0433bed5bc97fa3f875 100644 GIT binary patch literal 991 zcmeAS@N?(olHy`uVBq!ia0vp^fk5oV!3-q7IC#VWDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C+1{@Ck7R(zZ@6xrIfo z9bNMlEUc`pOH56hIAyAbuYYJ{bis=AH5)E3IC|f)XaAIaw^J6Lj+k}4WBYZV$%p^{ z|9@3oxB;l2u_VYZn8D%MjWiG^$=lsUm}hs)G$4n)#M9T6{SmX6khPJY?`jza2ByiL zE{-7)hp$e%*>qTe!_{}X$!xd3|NG}=3M(rt$h77!z8_-46n9zb^YK?^vFpBGxV`uO z;;k#(r<-P6vE~TroX)pW{E(YWS@xO0ls*%Wx4z*WBK5k;A&XvG^!B$%?|AB{+9$i- z{pJ&A(fAUjP1C29ESTX@E-bF)VmQ@t#{Me}McS2LOBc*}D!qMDN%m9TT$`_T_dGYu ztn3SKy?6fcmg;Ca!zaq1uY z5#jr@+#aQ$f4^UF{d2vvs;su=l@dKaIj21=*>Gd}wv)9RM4Iic+8iu6{w7r6SPpZ$ zR+M7P@nQj!4;>kz=MN!bO|K)wI?MN*+1j%RWK$r>?4{;enJ3C+tcDY}XuE&fnwAeYPTg0<$URze;bd6EPO{w#F@iMLt7S@r? zaf`JlJo%YenXA0jXC9Z3e$(~EJCuytTczG}d}zD4k#*nw!`YnecS;yqw4ep^j#OZ`^o6W8w<3Np5? zF-`dRN9mVg!I#2aRu^8J=Ra-!vFFpDC&!Dom4rQHS+98h#x;jUp--;<)z8w)KRJKx zwkxllKXzBuX`Q=s^!gX(ttUcnI^LQP3rw-9C9V-ADTyViR>?)FK#IZ0z|cb1z*yJF zD8$gh%D~vl$Uxh`(8|EzY0%nC}Q!>*kQ8buanOa(zLN(k!|NACT1B0ilpUXO@ GgeCw0;Hl#P literal 8190 zcmeI1zi;J45XZ;&lIJA%a!y=O;+8)GWPvCU0--CAKvYOThXhn0L7pj)PKX8(nk&#< zM@dHqmJp(#prD|jz$N7?T|7NAvpch%-Ssb_6d=W?*yG*V&wl5J?I;J3;J^DX0-jIc z?*iZn0C)<2?&Hrj{?Pc?@%(eoz$-7m0B^khD!lXdoAAN=@50q5AHk1bJPSX4^%8vk z;4}E;;cM{q^_TGLe`2GR>`NK8*_4A`K{O$N{25vL(AI?C2 z(+9>nkG#w_o6UN?#`DGHy!$NkUWzdPIL)TH)6Ba*Ox@j^z)mnTg%~i?i!tmdlkzur zjO1X1IipcFO9v}9%#7kwYDAO62Wc8(!2;a ziplBZ9C@PEBPxQud9J*jOBFMiFpF4+gi2!!_3Fa&B8O^Z-GTrNlL5x&6BlhLD$We) zhVq4Pqwn`WL#jg}U4flk};1U_Eijc&`lMCmf`d9Or(cr}mb^pYhy>GjbcC(}Tj zMc+9e9|@c z-66AcM`i3fp#fv>0Sd7j{AP-ct6*z=(8V~B|Bmd`jyGiE0vfNNKsRdx(ra9d;|IvP zu7#3py5K-T_Rh$tGO`=mvujdgr)2C_WY<}T65539s%!zWJsQx#7B^nurGf)`;s2mo z8!vcltFnT9av-~`iwfG+9H@1E!kHI<+d)-uC7+7pX~$LSyKXz?^nB3 zlcF@TQA(Tc*lr4(6T4&^_Gwcd>^KJF?4(WWr=PUrpYF5f!)(+GyOo3^qelq=y-PQ Date: Fri, 23 Mar 2018 12:10:18 +0100 Subject: [PATCH 05/83] Ensure we update all requirements when installing from pip/requirements.txt --- api/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/api/requirements.txt b/api/requirements.txt index d1197135e..00be27c53 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,3 +1,4 @@ # This file is here because many Platforms as a Service look for # requirements.txt in the root directory of a project. +-r requirements/base.txt -r requirements/production.txt From e3fc33a2c425a7862576316bce3afd4802039d96 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Fri, 23 Mar 2018 12:15:21 +0100 Subject: [PATCH 06/83] Minor tweaks to upgrade documentation --- docs/upgrading.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 24f5502a6..674878ba7 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -3,10 +3,13 @@ Upgrading your funkwhale instance to a newer version .. note:: - Before upgrading your instance, we strongly advise you to make a database - backup. We're commited to make upgrade as easy and straightforward as possible, + Before upgrading your instance, we strongly advise you to make at least a database backup. Ideally, you should make a full backup, including + the database and the media files. + + We're commited to make upgrade as easy and straightforward as possible, however, funkwhale is still in development and you'll be safer with a backup. + Reading the release notes ------------------------- @@ -62,14 +65,17 @@ match what is described in :doc:`debian`: .. parsed-literal:: + # stop the services + sudo systemctl stop funkwhale.target + # this assumes you want to upgrade to version "|version|" export FUNKWALE_VERSION="|version|" cd /srv/funkwhale # download more recent API files - curl -L -o "api-|version|.zip" "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/|version|/download?job=build_api" - unzip "api-|version|.zip" -d extracted - mv extracted/api/* api/ + curl -L -o "api-|version|.zip" "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/$FUNKWALE_VERSION/download?job=build_api" + unzip "api-$FUNKWALE_VERSION.zip" -d extracted + rm -rf api/ && mv extracted/api . rm -rf extracted # update os dependencies From 4409a9cac9b6e5ff962480fc7bfab4df207fbbe4 Mon Sep 17 00:00:00 2001 From: Maxence Bothorel Date: Fri, 23 Mar 2018 13:23:23 +0100 Subject: [PATCH 07/83] Add changelog --- changes/changelog.d/130.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/changelog.d/130.bugfix diff --git a/changes/changelog.d/130.bugfix b/changes/changelog.d/130.bugfix new file mode 100644 index 000000000..f8331ad67 --- /dev/null +++ b/changes/changelog.d/130.bugfix @@ -0,0 +1 @@ +Converted favicon from .ico to .png (#130) From cd89785344f14925c8316abfc788afd492a7a3f2 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Fri, 23 Mar 2018 14:59:12 +0100 Subject: [PATCH 08/83] Fix #101: Added credits to Francis Gading for the logotype --- changes/changelog.d/101.enhancement | 1 + front/src/App.vue | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/changelog.d/101.enhancement diff --git a/changes/changelog.d/101.enhancement b/changes/changelog.d/101.enhancement new file mode 100644 index 000000000..e6d1fde90 --- /dev/null +++ b/changes/changelog.d/101.enhancement @@ -0,0 +1 @@ +Added credits to Francis Gading for the logotype (#101) diff --git a/front/src/App.vue b/front/src/App.vue index d15eebdba..bff52e97e 100644 --- a/front/src/App.vue +++ b/front/src/App.vue @@ -21,6 +21,7 @@

About funkwhale

Funkwhale is a free and open-source project run by volunteers. You can help us improve the platform by reporting bugs, suggesting features and share the project with your friends!

+

The funkwhale logo was kindly designed and provided by Francis Gading.

From d726a959abcc082f13fc7f1d5e43ffeba1a7062d Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Fri, 23 Mar 2018 15:54:04 +0100 Subject: [PATCH 09/83] Fix #131: Removed player from queue tab and consistently show current track in queue --- changes/changelog.d/131.enhancement | 1 + front/src/components/Sidebar.vue | 57 +++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 changes/changelog.d/131.enhancement diff --git a/changes/changelog.d/131.enhancement b/changes/changelog.d/131.enhancement new file mode 100644 index 000000000..260a8bb6f --- /dev/null +++ b/changes/changelog.d/131.enhancement @@ -0,0 +1 @@ +Removed player from queue tab and consistently show current track in queue (#131) diff --git a/front/src/components/Sidebar.vue b/front/src/components/Sidebar.vue index f225313b6..42a923b6b 100644 --- a/front/src/components/Sidebar.vue +++ b/front/src/components/Sidebar.vue @@ -16,8 +16,8 @@
- Browse - + Browse + Queue   @@ -121,6 +122,7 @@ export default { }, data () { return { + selectedTab: 'library', backend: backend, isCollapsed: true } @@ -140,11 +142,36 @@ export default { }), reorder: function (oldValue, newValue) { this.$store.commit('queue/reorder', {oldValue, newValue}) + }, + scrollToCurrent () { + let current = $(this.$el).find('[data-tab="queue"] .active')[0] + if (!current) { + return + } + let container = $(this.$el).find('.tabs')[0] + // Position container at the top line then scroll current into view + container.scrollTop = 0 + current.scrollIntoView(true) + // Scroll back nothing if element is at bottom of container else do it + // for half the height of the containers display area + var scrollBack = (container.scrollHeight - container.scrollTop <= container.clientHeight) ? 0 : container.clientHeight / 2 + container.scrollTop = container.scrollTop - scrollBack + console.log(container.scrollHeight - container.scrollTop, container.clientHeight) } }, watch: { url: function () { this.isCollapsed = true + }, + selectedTab: function (newValue) { + if (newValue === 'queue') { + this.scrollToCurrent() + } + }, + '$store.state.queue.currentIndex': function () { + if (this.selectedTab !== 'queue') { + this.scrollToCurrent() + } } } } @@ -204,19 +231,38 @@ $sidebar-color: #3D3E3F; } } .tabs { + flex: 1; + display: flex; + flex-direction: column; overflow-y: auto; + justify-content: space-between; @include media(">tablet") { height: 0px; } @include media(" .menu { + flex: 1; + flex-grow: 1; + } + > .player-wrapper { + width: 100%; + } +} .sidebar .segment { margin: 0; border-radius: 0; @@ -225,9 +271,6 @@ $sidebar-color: #3D3E3F; .ui.inverted.segment.header-wrapper { padding: 0; } -.tabs { - flex: 1; -} .logo { cursor: pointer; From 6a30e59aa28555dc508e8f84445ab790639ef01d Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Fri, 23 Mar 2018 16:28:30 +0100 Subject: [PATCH 10/83] Fix #97: Queue shuffle now apply only to tracks after the current one --- changes/changelog.d/97.enhancement | 1 + front/src/store/queue.js | 5 +++-- front/test/unit/specs/store/queue.spec.js | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 changes/changelog.d/97.enhancement diff --git a/changes/changelog.d/97.enhancement b/changes/changelog.d/97.enhancement new file mode 100644 index 000000000..21a552ae2 --- /dev/null +++ b/changes/changelog.d/97.enhancement @@ -0,0 +1 @@ +Queue shuffle now apply only to tracks after the current one (#97) diff --git a/front/src/store/queue.js b/front/src/store/queue.js index 07263da63..a864405cf 100644 --- a/front/src/store/queue.js +++ b/front/src/store/queue.js @@ -41,7 +41,6 @@ export default { state.currentIndex += 1 } } - }, getters: { currentTrack: state => { @@ -141,7 +140,9 @@ export default { commit('ended', true) }, shuffle ({dispatch, commit, state}) { - let shuffled = _.shuffle(state.tracks) + let toKeep = state.tracks.slice(0, state.currentIndex + 1) + let toShuffle = state.tracks.slice(state.currentIndex + 1) + let shuffled = toKeep.concat(_.shuffle(toShuffle)) commit('player/currentTime', 0, {root: true}) commit('tracks', []) dispatch('appendMany', {tracks: shuffled}) diff --git a/front/test/unit/specs/store/queue.spec.js b/front/test/unit/specs/store/queue.spec.js index 5439362dc..b445229ec 100644 --- a/front/test/unit/specs/store/queue.spec.js +++ b/front/test/unit/specs/store/queue.spec.js @@ -316,18 +316,18 @@ describe('store/queue', () => { }) it('shuffle', (done) => { let _shuffle = sandbox.stub(_, 'shuffle') - let tracks = [1, 2, 3] - let shuffledTracks = [2, 3, 1] + let tracks = ['a', 'b', 'c', 'd', 'e'] + let shuffledTracks = ['e', 'd', 'c'] _shuffle.returns(shuffledTracks) testAction({ action: store.actions.shuffle, - params: {state: {tracks: tracks}}, + params: {state: {currentIndex: 1, tracks: tracks}}, expectedMutations: [ { type: 'player/currentTime', payload: 0 , options: {root: true}}, { type: 'tracks', payload: [] } ], expectedActions: [ - { type: 'appendMany', payload: {tracks: shuffledTracks} } + { type: 'appendMany', payload: {tracks: ['a', 'b'].concat(shuffledTracks)} } ] }, done) }) From d134b23a6edb3d3e02fedb00a50f8501b2696fc8 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Fri, 23 Mar 2018 21:45:17 +0100 Subject: [PATCH 11/83] Make more explicit the fact that most variables don't need editing on docker setups --- deploy/env.prod.sample | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy/env.prod.sample b/deploy/env.prod.sample index e1a381b94..a016b34c7 100644 --- a/deploy/env.prod.sample +++ b/deploy/env.prod.sample @@ -4,12 +4,13 @@ # - DJANGO_ALLOWED_HOSTS # - FUNKWHALE_URL -# Additionaly, on non-docker setup, you'll also have to tweak/uncomment those -# variables: +# Additionaly, on non-docker setup **only**, you'll also have to tweak/uncomment those variables: # - DATABASE_URL # - CACHE_URL # - STATIC_ROOT # - MEDIA_ROOT +# +# You **don't** need to update those variables on pure docker setups. # Docker only # ----------- From aa7365b71fa96da8808a462d9afdc664e8e2d41f Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 24 Mar 2018 15:20:15 +0100 Subject: [PATCH 12/83] Basic logic for signing/verifying requests --- api/config/settings/common.py | 1 + api/funkwhale_api/federation/__init__.py | 0 api/funkwhale_api/federation/factories.py | 30 ++++++++++++++++++++ api/funkwhale_api/federation/signing.py | 21 ++++++++++++++ api/requirements/base.txt | 2 ++ api/tests/conftest.py | 12 ++++++-- api/tests/federation/test_signing.py | 34 +++++++++++++++++++++++ 7 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 api/funkwhale_api/federation/__init__.py create mode 100644 api/funkwhale_api/federation/factories.py create mode 100644 api/funkwhale_api/federation/signing.py create mode 100644 api/tests/federation/test_signing.py diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 077566d1c..1144e8891 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -89,6 +89,7 @@ LOCAL_APPS = ( 'funkwhale_api.music', 'funkwhale_api.requests', 'funkwhale_api.favorites', + 'funkwhale_api.federation', 'funkwhale_api.radios', 'funkwhale_api.history', 'funkwhale_api.playlists', diff --git a/api/funkwhale_api/federation/__init__.py b/api/funkwhale_api/federation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py new file mode 100644 index 000000000..29aed4baf --- /dev/null +++ b/api/funkwhale_api/federation/factories.py @@ -0,0 +1,30 @@ +import factory +import requests +import requests_http_signature + +from funkwhale_api.factories import registry + +from . import signing + + +registry.register(signing.get_key_pair, name='federation.KeyPair') + + +@registry.register(name='federation.SignatureAuth') +class SignatureAuthFactory(factory.Factory): + algorithm = 'rsa-sha256' + key = factory.LazyFunction(lambda: signing.get_key_pair()[0]) + key_id = factory.Faker('url') + + class Meta: + model = requests_http_signature.HTTPSignatureAuth + + +@registry.register(name='federation.SignedRequest') +class SignedRequestFactory(factory.Factory): + url = factory.Faker('url') + method = 'get' + auth = factory.SubFactory(SignatureAuthFactory) + + class Meta: + model = requests.Request diff --git a/api/funkwhale_api/federation/signing.py b/api/funkwhale_api/federation/signing.py new file mode 100644 index 000000000..626dfbe86 --- /dev/null +++ b/api/funkwhale_api/federation/signing.py @@ -0,0 +1,21 @@ +from cryptography.hazmat.primitives import serialization as crypto_serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.backends import default_backend as crypto_default_backend + + +def get_key_pair(size=2048): + key = rsa.generate_private_key( + backend=crypto_default_backend(), + public_exponent=65537, + key_size=size + ) + private_key = key.private_bytes( + crypto_serialization.Encoding.PEM, + crypto_serialization.PrivateFormat.PKCS8, + crypto_serialization.NoEncryption()) + public_key = key.public_key().public_bytes( + crypto_serialization.Encoding.PEM, + crypto_serialization.PublicFormat.PKCS1 + ) + + return private_key, public_key diff --git a/api/requirements/base.txt b/api/requirements/base.txt index efcc4eea4..d4408dbdc 100644 --- a/api/requirements/base.txt +++ b/api/requirements/base.txt @@ -60,3 +60,5 @@ channels_redis>=2.1,<2.2 django-cacheops>=4,<4.1 daphne==2.0.4 +cryptography>=2,<3 +requests-http-signature==0.0.3 diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 62bc5ada6..d5b3b24f4 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -31,7 +31,11 @@ def cache(): def factories(db): from funkwhale_api import factories for v in factories.registry.values(): - v._meta.strategy = factory.CREATE_STRATEGY + try: + v._meta.strategy = factory.CREATE_STRATEGY + except AttributeError: + # probably not a class based factory + pass yield factories.registry @@ -39,7 +43,11 @@ def factories(db): def nodb_factories(): from funkwhale_api import factories for v in factories.registry.values(): - v._meta.strategy = factory.BUILD_STRATEGY + try: + v._meta.strategy = factory.BUILD_STRATEGY + except AttributeError: + # probably not a class based factory + pass yield factories.registry diff --git a/api/tests/federation/test_signing.py b/api/tests/federation/test_signing.py new file mode 100644 index 000000000..712f2bd35 --- /dev/null +++ b/api/tests/federation/test_signing.py @@ -0,0 +1,34 @@ +import cryptography.exceptions +import io +import pytest +import requests_http_signature + +from funkwhale_api.federation import signing + + +def test_can_sign_and_verify_request(factories): + private, public = factories['federation.KeyPair']() + auth = factories['federation.SignatureAuth'](key=private) + request = factories['federation.SignedRequest']( + auth=auth + ) + prepared_request = request.prepare() + assert 'date' in prepared_request.headers + assert 'authorization' in prepared_request.headers + assert prepared_request.headers['authorization'].startswith('Signature') + assert requests_http_signature.HTTPSignatureAuth.verify( + prepared_request, + key_resolver=lambda **kwargs: public + ) is None + + +def test_verify_fails_with_wrong_key(factories): + wrong_private, wrong_public = factories['federation.KeyPair']() + request = factories['federation.SignedRequest']() + prepared_request = request.prepare() + + with pytest.raises(cryptography.exceptions.InvalidSignature): + requests_http_signature.HTTPSignatureAuth.verify( + prepared_request, + key_resolver=lambda **kwargs: wrong_public + ) From 4522f5997e472c0df254a3c7ca57ad88ef73c010 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 24 Mar 2018 16:24:10 +0100 Subject: [PATCH 13/83] More test cases for request signing and added helpers to verify signature --- api/funkwhale_api/federation/signing.py | 35 +++++++++ api/tests/federation/test_signing.py | 100 ++++++++++++++++++++++-- 2 files changed, 127 insertions(+), 8 deletions(-) diff --git a/api/funkwhale_api/federation/signing.py b/api/funkwhale_api/federation/signing.py index 626dfbe86..e8d79097c 100644 --- a/api/funkwhale_api/federation/signing.py +++ b/api/funkwhale_api/federation/signing.py @@ -1,3 +1,6 @@ +import requests +import requests_http_signature + from cryptography.hazmat.primitives import serialization as crypto_serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.backends import default_backend as crypto_default_backend @@ -19,3 +22,35 @@ def get_key_pair(size=2048): ) return private_key, public_key + + +def verify(request, public_key): + return requests_http_signature.HTTPSignatureAuth.verify( + request, + key_resolver=lambda **kwargs: public_key + ) + + +def verify_django(django_request, public_key): + """ + Given a django WSGI request, create an underlying requests.PreparedRequest + instance we can verify + """ + headers = django_request.META.get('headers', {}).copy() + for h, v in list(headers.items()): + # we include lower-cased version of the headers for compatibility + # with requests_http_signature + headers[h.lower()] = v + try: + signature = headers['authorization'] + except KeyError: + raise exceptions.MissingSignature + + request = requests.Request( + method=django_request.method, + url='http://noop', + data=django_request.body, + headers=headers) + + prepared_request = request.prepare() + return verify(request, public_key) diff --git a/api/tests/federation/test_signing.py b/api/tests/federation/test_signing.py index 712f2bd35..5187faa52 100644 --- a/api/tests/federation/test_signing.py +++ b/api/tests/federation/test_signing.py @@ -16,10 +16,23 @@ def test_can_sign_and_verify_request(factories): assert 'date' in prepared_request.headers assert 'authorization' in prepared_request.headers assert prepared_request.headers['authorization'].startswith('Signature') - assert requests_http_signature.HTTPSignatureAuth.verify( - prepared_request, - key_resolver=lambda **kwargs: public - ) is None + assert signing.verify(prepared_request, public) is None + + +def test_can_sign_and_verify_request_digest(factories): + private, public = factories['federation.KeyPair']() + auth = factories['federation.SignatureAuth'](key=private) + request = factories['federation.SignedRequest']( + auth=auth, + method='post', + data=b'hello=world' + ) + prepared_request = request.prepare() + assert 'date' in prepared_request.headers + assert 'digest' in prepared_request.headers + assert 'authorization' in prepared_request.headers + assert prepared_request.headers['authorization'].startswith('Signature') + assert signing.verify(prepared_request, public) is None def test_verify_fails_with_wrong_key(factories): @@ -28,7 +41,78 @@ def test_verify_fails_with_wrong_key(factories): prepared_request = request.prepare() with pytest.raises(cryptography.exceptions.InvalidSignature): - requests_http_signature.HTTPSignatureAuth.verify( - prepared_request, - key_resolver=lambda **kwargs: wrong_public - ) + signing.verify(prepared_request, wrong_public) + + +def test_can_verify_django_request(factories, api_request): + private_key, public_key = signing.get_key_pair() + signed_request = factories['federation.SignedRequest']( + auth__key=private_key + ) + prepared = signed_request.prepare() + django_request = api_request.get( + '/', + headers={ + 'Date': prepared.headers['date'], + 'Authorization': prepared.headers['authorization'], + } + ) + assert signing.verify_django(django_request, public_key) is None + + +def test_can_verify_django_request_digest(factories, api_request): + private_key, public_key = signing.get_key_pair() + signed_request = factories['federation.SignedRequest']( + auth__key=private_key, + method='post', + data=b'hello=world' + ) + prepared = signed_request.prepare() + django_request = api_request.post( + '/', + headers={ + 'Date': prepared.headers['date'], + 'Digest': prepared.headers['digest'], + 'Authorization': prepared.headers['authorization'], + } + ) + + assert signing.verify_django(django_request, public_key) is None + + +def test_can_verify_django_request_digest_failure(factories, api_request): + private_key, public_key = signing.get_key_pair() + signed_request = factories['federation.SignedRequest']( + auth__key=private_key, + method='post', + data=b'hello=world' + ) + prepared = signed_request.prepare() + django_request = api_request.post( + '/', + headers={ + 'Date': prepared.headers['date'], + 'Digest': prepared.headers['digest'] + 'noop', + 'Authorization': prepared.headers['authorization'], + } + ) + + with pytest.raises(cryptography.exceptions.InvalidSignature): + signing.verify_django(django_request, public_key) + + +def test_can_verify_django_request_failure(factories, api_request): + private_key, public_key = signing.get_key_pair() + signed_request = factories['federation.SignedRequest']( + auth__key=private_key + ) + prepared = signed_request.prepare() + django_request = api_request.get( + '/', + headers={ + 'Date': 'Wrong', + 'Authorization': prepared.headers['authorization'], + } + ) + with pytest.raises(cryptography.exceptions.InvalidSignature): + signing.verify_django(django_request, public_key) From 4b69d64db2915ab76303798c3f610e4b6e011cff Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 24 Mar 2018 20:31:36 +0100 Subject: [PATCH 14/83] Fix #139: We now restrict some usernames from being used during signup --- api/config/settings/common.py | 9 +++++++++ api/tests/users/test_views.py | 17 +++++++++++++++++ changes/changelog.d/139.enhancement | 1 + 3 files changed, 27 insertions(+) create mode 100644 changes/changelog.d/139.enhancement diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 077566d1c..2c72865f6 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -385,3 +385,12 @@ CSRF_USE_SESSIONS = True # Playlist settings PLAYLISTS_MAX_TRACKS = env.int('PLAYLISTS_MAX_TRACKS', default=250) + +ACCOUNT_USERNAME_BLACKLIST = [ + 'funkwhale', + 'root', + 'admin', + 'owner', + 'superuser', + 'staff', +] + env.list('ACCOUNT_USERNAME_BLACKLIST', default=[]) diff --git a/api/tests/users/test_views.py b/api/tests/users/test_views.py index 02b903bf4..4be586965 100644 --- a/api/tests/users/test_views.py +++ b/api/tests/users/test_views.py @@ -23,6 +23,23 @@ def test_can_create_user_via_api(preferences, client, db): assert u.username == 'test1' +def test_can_restrict_usernames(settings, preferences, db, client): + url = reverse('rest_register') + preferences['users__registration_enabled'] = True + settings.USERNAME_BLACKLIST = ['funkwhale'] + data = { + 'username': 'funkwhale', + 'email': 'contact@funkwhale.io', + 'password1': 'testtest', + 'password2': 'testtest', + } + + response = client.post(url, data) + + assert response.status_code == 400 + assert 'username' in response.data + + def test_can_disable_registration_view(preferences, client, db): url = reverse('rest_register') data = { diff --git a/changes/changelog.d/139.enhancement b/changes/changelog.d/139.enhancement new file mode 100644 index 000000000..c6648d139 --- /dev/null +++ b/changes/changelog.d/139.enhancement @@ -0,0 +1 @@ +We now restrict some usernames from being used during signup (#139) From f1cf250e2959e2fe7a333994fd10fe1817795e3b Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Mar 2018 15:40:37 +0200 Subject: [PATCH 15/83] Fix #138: Better handling of utf-8 filenames during file import --- api/config/settings/common.py | 1 + api/funkwhale_api/common/storage.py | 12 ++++++++++++ api/tests/files/utf8-éà◌.ogg | 0 api/tests/test_import_audio_file.py | 24 ++++++++++++++++++++++++ changes/changelog.d/138.bugfix | 1 + 5 files changed, 38 insertions(+) create mode 100644 api/funkwhale_api/common/storage.py create mode 100644 api/tests/files/utf8-éà◌.ogg create mode 100644 changes/changelog.d/138.bugfix diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 077566d1c..f8bd77252 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -231,6 +231,7 @@ STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles'))) # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url STATIC_URL = env("STATIC_URL", default='/staticfiles/') +DEFAULT_FILE_STORAGE = 'funkwhale_api.common.storage.ASCIIFileSystemStorage' # See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS STATICFILES_DIRS = ( diff --git a/api/funkwhale_api/common/storage.py b/api/funkwhale_api/common/storage.py new file mode 100644 index 000000000..658ce795a --- /dev/null +++ b/api/funkwhale_api/common/storage.py @@ -0,0 +1,12 @@ +import unicodedata + +from django.core.files.storage import FileSystemStorage + + +class ASCIIFileSystemStorage(FileSystemStorage): + """ + Convert unicode characters in name to ASCII characters. + """ + def get_valid_name(self, name): + name = unicodedata.normalize('NFKD', name).encode('ascii', 'ignore') + return super().get_valid_name(name) diff --git a/api/tests/files/utf8-éà◌.ogg b/api/tests/files/utf8-éà◌.ogg new file mode 100644 index 000000000..e69de29bb diff --git a/api/tests/test_import_audio_file.py b/api/tests/test_import_audio_file.py index 4f3de27db..67263e66d 100644 --- a/api/tests/test_import_audio_file.py +++ b/api/tests/test_import_audio_file.py @@ -98,3 +98,27 @@ def test_import_files_skip_acoustid(factories, mocker): music_tasks.import_job_run.delay, import_job_id=job.pk, use_acoustid=False) + + +def test_import_files_works_with_utf8_file_name(factories, mocker): + m = mocker.patch('funkwhale_api.common.utils.on_commit') + user = factories['users.User'](username='me') + path = os.path.join(DATA_DIR, 'utf8-éà◌.ogg') + call_command( + 'import_files', + path, + username='me', + async=True, + no_acoustid=True, + interactive=False) + batch = user.imports.latest('id') + job = batch.jobs.first() + m.assert_called_once_with( + music_tasks.import_job_run.delay, + import_job_id=job.pk, + use_acoustid=False) + + +def test_storage_rename_utf_8_files(factories): + tf = factories['music.TrackFile'](audio_file__filename='été.ogg') + assert tf.audio_file.name.endswith('ete.ogg') diff --git a/changes/changelog.d/138.bugfix b/changes/changelog.d/138.bugfix new file mode 100644 index 000000000..2a8f7aeb0 --- /dev/null +++ b/changes/changelog.d/138.bugfix @@ -0,0 +1 @@ +Better handling of utf-8 filenames during file import (#138) From e99d757b5741f44c4fed2dfc1bcaafc6c243295f Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Mar 2018 15:44:48 +0200 Subject: [PATCH 16/83] Fix #120: Better error handling during file import --- api/funkwhale_api/music/metadata.py | 8 +++- .../management/commands/import_files.py | 42 ++++++++++++------- changes/changelog.d/120.bugfix | 1 + 3 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 changes/changelog.d/120.bugfix diff --git a/api/funkwhale_api/music/metadata.py b/api/funkwhale_api/music/metadata.py index 31d13d495..3748d5573 100644 --- a/api/funkwhale_api/music/metadata.py +++ b/api/funkwhale_api/music/metadata.py @@ -121,7 +121,13 @@ class Metadata(object): def __init__(self, path): self._file = mutagen.File(path) - self._conf = CONF[self.get_file_type(self._file)] + if self._file is None: + raise ValueError('Cannot parse metadata from {}'.format(path)) + ft = self.get_file_type(self._file) + try: + self._conf = CONF[ft] + except KeyError: + raise ValueError('Unsupported format {}'.format(ft)) def get_file_type(self, f): return f.__class__.__name__ diff --git a/api/funkwhale_api/providers/audiofile/management/commands/import_files.py b/api/funkwhale_api/providers/audiofile/management/commands/import_files.py index 2fa5e464c..dbc01289f 100644 --- a/api/funkwhale_api/providers/audiofile/management/commands/import_files.py +++ b/api/funkwhale_api/providers/audiofile/management/commands/import_files.py @@ -34,6 +34,13 @@ class Command(BaseCommand): default=False, help='Will launch celery tasks for each file to import instead of doing it synchronously and block the CLI', ) + parser.add_argument( + '--exit', '-x', + action='store_true', + dest='exit_on_failure', + default=False, + help='use this flag to disable error catching', + ) parser.add_argument( '--no-acoustid', action='store_true', @@ -106,20 +113,27 @@ class Command(BaseCommand): async = options['async'] import_handler = tasks.import_job_run.delay if async else tasks.import_job_run for path in matching: - job = batch.jobs.create( - source='file://' + path, - ) - name = os.path.basename(path) - with open(path, 'rb') as f: - job.audio_file.save(name, File(f)) - - job.save() try: - utils.on_commit( - import_handler, - import_job_id=job.pk, - use_acoustid=not options['no_acoustid']) + self.stdout.write(message.format(path)) + self.import_file(path, batch, import_handler, options) except Exception as e: - self.stdout.write('Error: {}'.format(e)) - + if options['exit_on_failure']: + raise + m = 'Error while importing {}: {} {}'.format( + path, e.__class__.__name__, e) + self.stderr.write(m) return batch + + def import_file(self, path, batch, import_handler, options): + job = batch.jobs.create( + source='file://' + path, + ) + name = os.path.basename(path) + with open(path, 'rb') as f: + job.audio_file.save(name, File(f)) + + job.save() + utils.on_commit( + import_handler, + import_job_id=job.pk, + use_acoustid=not options['no_acoustid']) diff --git a/changes/changelog.d/120.bugfix b/changes/changelog.d/120.bugfix new file mode 100644 index 000000000..b7d9ef066 --- /dev/null +++ b/changes/changelog.d/120.bugfix @@ -0,0 +1 @@ +Better error handling during file import (#120) From fce5ee0e7fbac196e5a1e40c29fb446085ca0988 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Mar 2018 17:17:33 +0200 Subject: [PATCH 17/83] Renamed listening end_date to creation_date --- api/funkwhale_api/history/admin.py | 2 +- .../migrations/0002_auto_20180325_1433.py | 22 +++++++++++++++++++ api/funkwhale_api/history/models.py | 5 +++-- api/funkwhale_api/history/serializers.py | 4 ++-- api/tests/history/test_activity.py | 2 +- 5 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 api/funkwhale_api/history/migrations/0002_auto_20180325_1433.py diff --git a/api/funkwhale_api/history/admin.py b/api/funkwhale_api/history/admin.py index 6d0480e73..5ddfb8998 100644 --- a/api/funkwhale_api/history/admin.py +++ b/api/funkwhale_api/history/admin.py @@ -4,7 +4,7 @@ from . import models @admin.register(models.Listening) class ListeningAdmin(admin.ModelAdmin): - list_display = ['track', 'end_date', 'user', 'session_key'] + list_display = ['track', 'creation_date', 'user', 'session_key'] search_fields = ['track__name', 'user__username'] list_select_related = [ 'user', diff --git a/api/funkwhale_api/history/migrations/0002_auto_20180325_1433.py b/api/funkwhale_api/history/migrations/0002_auto_20180325_1433.py new file mode 100644 index 000000000..d83dbb0a4 --- /dev/null +++ b/api/funkwhale_api/history/migrations/0002_auto_20180325_1433.py @@ -0,0 +1,22 @@ +# Generated by Django 2.0.3 on 2018-03-25 14:33 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('history', '0001_initial'), + ] + + operations = [ + migrations.AlterModelOptions( + name='listening', + options={'ordering': ('-creation_date',)}, + ), + migrations.RenameField( + model_name='listening', + old_name='end_date', + new_name='creation_date', + ), + ] diff --git a/api/funkwhale_api/history/models.py b/api/funkwhale_api/history/models.py index 56310ddc0..762d5bf7b 100644 --- a/api/funkwhale_api/history/models.py +++ b/api/funkwhale_api/history/models.py @@ -6,7 +6,8 @@ from funkwhale_api.music.models import Track class Listening(models.Model): - end_date = models.DateTimeField(default=timezone.now, null=True, blank=True) + creation_date = models.DateTimeField( + default=timezone.now, null=True, blank=True) track = models.ForeignKey( Track, related_name="listenings", on_delete=models.CASCADE) user = models.ForeignKey( @@ -18,7 +19,7 @@ class Listening(models.Model): session_key = models.CharField(max_length=100, null=True, blank=True) class Meta: - ordering = ('-end_date',) + ordering = ('-creation_date',) def save(self, **kwargs): if not self.user and not self.session_key: diff --git a/api/funkwhale_api/history/serializers.py b/api/funkwhale_api/history/serializers.py index 7a2280cea..8fe6fa6e0 100644 --- a/api/funkwhale_api/history/serializers.py +++ b/api/funkwhale_api/history/serializers.py @@ -12,7 +12,7 @@ class ListeningActivitySerializer(activity_serializers.ModelSerializer): type = serializers.SerializerMethodField() object = TrackActivitySerializer(source='track') actor = UserActivitySerializer(source='user') - published = serializers.DateTimeField(source='end_date') + published = serializers.DateTimeField(source='creation_date') class Meta: model = models.Listening @@ -36,7 +36,7 @@ class ListeningSerializer(serializers.ModelSerializer): class Meta: model = models.Listening - fields = ('id', 'user', 'session_key', 'track', 'end_date') + fields = ('id', 'user', 'session_key', 'track', 'creation_date') def create(self, validated_data): diff --git a/api/tests/history/test_activity.py b/api/tests/history/test_activity.py index b5ab07b82..04000604b 100644 --- a/api/tests/history/test_activity.py +++ b/api/tests/history/test_activity.py @@ -23,7 +23,7 @@ def test_activity_listening_serializer(factories): "id": listening.get_activity_url(), "actor": actor, "object": TrackActivitySerializer(listening.track).data, - "published": field.to_representation(listening.end_date), + "published": field.to_representation(listening.creation_date), } data = serializers.ListeningActivitySerializer(listening).data From 2d4003c8c4496396d6364fad42bc20e9ef746759 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Mar 2018 17:17:51 +0200 Subject: [PATCH 18/83] anonymoususer test fixture --- api/tests/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 62bc5ada6..06d225a0f 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -3,6 +3,7 @@ import tempfile import shutil import pytest +from django.contrib.auth.models import AnonymousUser from django.core.cache import cache as django_cache from dynamic_preferences.registries import global_preferences_registry @@ -66,6 +67,11 @@ def logged_in_client(db, factories, client): delattr(client, 'user') +@pytest.fixture +def anonymous_user(): + return AnonymousUser() + + @pytest.fixture def api_client(client): return APIClient() From 122c39075a8258b89557c8f1087c578c6540cde8 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Mar 2018 17:18:22 +0200 Subject: [PATCH 19/83] Fixed privacy issue in get_privacy_query utils function --- api/funkwhale_api/common/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/funkwhale_api/common/fields.py b/api/funkwhale_api/common/fields.py index ef9f840dc..1a18b5f27 100644 --- a/api/funkwhale_api/common/fields.py +++ b/api/funkwhale_api/common/fields.py @@ -22,6 +22,6 @@ def privacy_level_query(user, lookup_field='privacy_level'): return models.Q(**{ '{}__in'.format(lookup_field): [ - 'me', 'followers', 'instance', 'everyone' + 'followers', 'instance', 'everyone' ] }) From 1f2e14b20e725c6db881c3619eeccc20eb49cfa1 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Mar 2018 17:18:44 +0200 Subject: [PATCH 20/83] Use anonymous_user fixture in other tests --- api/tests/common/test_permissions.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api/tests/common/test_permissions.py b/api/tests/common/test_permissions.py index b5c5160f8..f04f12e0b 100644 --- a/api/tests/common/test_permissions.py +++ b/api/tests/common/test_permissions.py @@ -2,7 +2,6 @@ import pytest from rest_framework.views import APIView -from django.contrib.auth.models import AnonymousUser from django.http import Http404 from funkwhale_api.common import permissions @@ -19,24 +18,26 @@ def test_owner_permission_owner_field_ok(nodb_factories, api_request): assert check is True -def test_owner_permission_owner_field_not_ok(nodb_factories, api_request): +def test_owner_permission_owner_field_not_ok( + anonymous_user, nodb_factories, api_request): playlist = nodb_factories['playlists.Playlist']() view = APIView.as_view() permission = permissions.OwnerPermission() request = api_request.get('/') - setattr(request, 'user', AnonymousUser()) + setattr(request, 'user', anonymous_user) with pytest.raises(Http404): permission.has_object_permission(request, view, playlist) -def test_owner_permission_read_only(nodb_factories, api_request): +def test_owner_permission_read_only( + anonymous_user, nodb_factories, api_request): playlist = nodb_factories['playlists.Playlist']() view = APIView.as_view() setattr(view, 'owner_checks', ['write']) permission = permissions.OwnerPermission() request = api_request.get('/') - setattr(request, 'user', AnonymousUser()) + setattr(request, 'user', anonymous_user) check = permission.has_object_permission(request, view, playlist) assert check is True From 18d8baae344649d079086cc128aced119d0f55cd Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Mar 2018 17:24:08 +0200 Subject: [PATCH 21/83] API Views/serializers/tests for activity (#141) --- api/config/api_urls.py | 2 + api/funkwhale_api/activity/serializers.py | 14 +++++ api/funkwhale_api/activity/utils.py | 64 +++++++++++++++++++++++ api/funkwhale_api/activity/views.py | 20 +++++++ api/tests/activity/test_serializers.py | 17 ++++++ api/tests/activity/test_utils.py | 21 ++++++++ api/tests/activity/test_views.py | 18 +++++++ 7 files changed, 156 insertions(+) create mode 100644 api/funkwhale_api/activity/utils.py create mode 100644 api/funkwhale_api/activity/views.py create mode 100644 api/tests/activity/test_serializers.py create mode 100644 api/tests/activity/test_utils.py create mode 100644 api/tests/activity/test_views.py diff --git a/api/config/api_urls.py b/api/config/api_urls.py index ff6db0d06..cab6805b6 100644 --- a/api/config/api_urls.py +++ b/api/config/api_urls.py @@ -1,5 +1,6 @@ from rest_framework import routers from django.conf.urls import include, url +from funkwhale_api.activity import views as activity_views from funkwhale_api.instance import views as instance_views from funkwhale_api.music import views from funkwhale_api.playlists import views as playlists_views @@ -10,6 +11,7 @@ from dynamic_preferences.users.viewsets import UserPreferencesViewSet router = routers.SimpleRouter() router.register(r'settings', GlobalPreferencesViewSet, base_name='settings') +router.register(r'activity', activity_views.ActivityViewSet, 'activity') router.register(r'tags', views.TagViewSet, 'tags') router.register(r'tracks', views.TrackViewSet, 'tracks') router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles') diff --git a/api/funkwhale_api/activity/serializers.py b/api/funkwhale_api/activity/serializers.py index 325d1e820..fd9b185cf 100644 --- a/api/funkwhale_api/activity/serializers.py +++ b/api/funkwhale_api/activity/serializers.py @@ -1,5 +1,7 @@ from rest_framework import serializers +from funkwhale_api.activity import record + class ModelSerializer(serializers.ModelSerializer): id = serializers.CharField(source='get_activity_url') @@ -8,3 +10,15 @@ class ModelSerializer(serializers.ModelSerializer): def get_url(self, obj): return self.get_id(obj) + + +class AutoSerializer(serializers.Serializer): + """ + A serializer that will automatically use registered activity serializers + to serialize an henerogeneous list of objects (favorites, listenings, etc.) + """ + def to_representation(self, instance): + serializer = record.registry[instance._meta.label]['serializer']( + instance + ) + return serializer.data diff --git a/api/funkwhale_api/activity/utils.py b/api/funkwhale_api/activity/utils.py new file mode 100644 index 000000000..46336930e --- /dev/null +++ b/api/funkwhale_api/activity/utils.py @@ -0,0 +1,64 @@ +from django.db import models + +from funkwhale_api.common import fields +from funkwhale_api.favorites.models import TrackFavorite +from funkwhale_api.history.models import Listening + + +def combined_recent(limit, **kwargs): + datetime_field = kwargs.pop('datetime_field', 'creation_date') + source_querysets = { + qs.model._meta.label: qs for qs in kwargs.pop('querysets') + } + querysets = { + k: qs.annotate( + __type=models.Value( + qs.model._meta.label, output_field=models.CharField() + ) + ).values('pk', datetime_field, '__type') + for k, qs in source_querysets.items() + } + _qs_list = list(querysets.values()) + union_qs = _qs_list[0].union(*_qs_list[1:]) + records = [] + for row in union_qs.order_by('-{}'.format(datetime_field))[:limit]: + records.append({ + 'type': row['__type'], + 'when': row[datetime_field], + 'pk': row['pk'] + }) + # Now we bulk-load each object type in turn + to_load = {} + for record in records: + to_load.setdefault(record['type'], []).append(record['pk']) + fetched = {} + + for key, pks in to_load.items(): + for item in source_querysets[key].filter(pk__in=pks): + fetched[(key, item.pk)] = item + + # Annotate 'records' with loaded objects + for record in records: + record['object'] = fetched[(record['type'], record['pk'])] + return records + + +def get_activity(user, limit=20): + query = fields.privacy_level_query( + user, lookup_field='user__privacy_level') + querysets = [ + Listening.objects.filter(query).select_related( + 'track', + 'user', + 'track__artist', + 'track__album__artist', + ), + TrackFavorite.objects.filter(query).select_related( + 'track', + 'user', + 'track__artist', + 'track__album__artist', + ), + ] + records = combined_recent(limit=limit, querysets=querysets) + return [r['object'] for r in records] diff --git a/api/funkwhale_api/activity/views.py b/api/funkwhale_api/activity/views.py new file mode 100644 index 000000000..e66de1ccf --- /dev/null +++ b/api/funkwhale_api/activity/views.py @@ -0,0 +1,20 @@ +from rest_framework import viewsets +from rest_framework.response import Response + +from funkwhale_api.common.permissions import ConditionalAuthentication +from funkwhale_api.favorites.models import TrackFavorite + +from . import serializers +from . import utils + + +class ActivityViewSet(viewsets.GenericViewSet): + + serializer_class = serializers.AutoSerializer + permission_classes = [ConditionalAuthentication] + queryset = TrackFavorite.objects.none() + + def list(self, request, *args, **kwargs): + activity = utils.get_activity(user=request.user) + serializer = self.serializer_class(activity, many=True) + return Response({'results': serializer.data}, status=200) diff --git a/api/tests/activity/test_serializers.py b/api/tests/activity/test_serializers.py new file mode 100644 index 000000000..792fa74b9 --- /dev/null +++ b/api/tests/activity/test_serializers.py @@ -0,0 +1,17 @@ +from funkwhale_api.activity import serializers +from funkwhale_api.favorites.serializers import TrackFavoriteActivitySerializer +from funkwhale_api.history.serializers import \ + ListeningActivitySerializer + + +def test_autoserializer(factories): + favorite = factories['favorites.TrackFavorite']() + listening = factories['history.Listening']() + objects = [favorite, listening] + serializer = serializers.AutoSerializer(objects, many=True) + expected = [ + TrackFavoriteActivitySerializer(favorite).data, + ListeningActivitySerializer(listening).data, + ] + + assert serializer.data == expected diff --git a/api/tests/activity/test_utils.py b/api/tests/activity/test_utils.py new file mode 100644 index 000000000..43bb45df8 --- /dev/null +++ b/api/tests/activity/test_utils.py @@ -0,0 +1,21 @@ +from funkwhale_api.activity import utils + + +def test_get_activity(factories): + user = factories['users.User']() + listening = factories['history.Listening']() + favorite = factories['favorites.TrackFavorite']() + + objects = list(utils.get_activity(user)) + assert objects == [favorite, listening] + + +def test_get_activity_honors_privacy_level(factories, anonymous_user): + listening = factories['history.Listening'](user__privacy_level='me') + favorite1 = factories['favorites.TrackFavorite']( + user__privacy_level='everyone') + favorite2 = factories['favorites.TrackFavorite']( + user__privacy_level='instance') + + objects = list(utils.get_activity(anonymous_user)) + assert objects == [favorite1] diff --git a/api/tests/activity/test_views.py b/api/tests/activity/test_views.py new file mode 100644 index 000000000..bdc3c6339 --- /dev/null +++ b/api/tests/activity/test_views.py @@ -0,0 +1,18 @@ +from django.urls import reverse + +from funkwhale_api.activity import serializers +from funkwhale_api.activity import utils + + +def test_activity_view(factories, api_client, settings, anonymous_user): + settings.API_AUTHENTICATION_REQUIRED = False + favorite = factories['favorites.TrackFavorite']( + user__privacy_level='everyone') + listening = factories['history.Listening']() + url = reverse('api:v1:activity-list') + objects = utils.get_activity(anonymous_user) + serializer = serializers.AutoSerializer(objects, many=True) + response = api_client.get(url) + + assert response.status_code == 200 + assert response.data['results'] == serializer.data From 2384f761b1efe9e9fa40f5b767691a09db0aee67 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Mar 2018 17:36:02 +0200 Subject: [PATCH 22/83] Now fetch activity from API on first timeline display (#141) --- front/src/App.vue | 26 ---------------- front/src/store/instance.js | 3 ++ front/src/views/instance/Timeline.vue | 43 ++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/front/src/App.vue b/front/src/App.vue index bff52e97e..e8ab18694 100644 --- a/front/src/App.vue +++ b/front/src/App.vue @@ -36,9 +36,6 @@ diff --git a/front/src/store/instance.js b/front/src/store/instance.js index 2436eab07..245acaf03 100644 --- a/front/src/store/instance.js +++ b/front/src/store/instance.js @@ -43,6 +43,9 @@ export default { if (state.events.length > state.maxEvents) { state.events = state.events.slice(0, state.maxEvents) } + }, + events: (state, value) => { + state.events = value } }, actions: { diff --git a/front/src/views/instance/Timeline.vue b/front/src/views/instance/Timeline.vue index b959c25d6..8ffcd9758 100644 --- a/front/src/views/instance/Timeline.vue +++ b/front/src/views/instance/Timeline.vue @@ -1,7 +1,10 @@