From 23d3893f01626a9c4dc198f71cabefcecfb7bac0 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Wed, 18 Mar 2020 15:52:23 +0100 Subject: [PATCH] See #170: subsonic API for podcasts --- .gitlab-ci.yml | 2 +- api/funkwhale_api/audio/models.py | 9 ++ api/funkwhale_api/common/utils.py | 4 + api/funkwhale_api/subsonic/serializers.py | 64 +++++++- api/funkwhale_api/subsonic/views.py | 178 +++++++++++++++++++++- api/setup.cfg | 2 + api/tests/subsonic/test_serializers.py | 52 +++++++ api/tests/subsonic/test_views.py | 150 +++++++++++++++++- 8 files changed, 452 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 97cdf7683..9d241b539 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -135,7 +135,7 @@ test_api: only: - branches before_script: - - apk add make git gcc python3-dev + - apk add make git gcc python3-dev musl-dev - cd api - pip3 install -r requirements/base.txt - pip3 install -r requirements/local.txt diff --git a/api/funkwhale_api/audio/models.py b/api/funkwhale_api/audio/models.py index 38e023c4a..d2ebe5fe1 100644 --- a/api/funkwhale_api/audio/models.py +++ b/api/funkwhale_api/audio/models.py @@ -31,6 +31,15 @@ class ChannelQuerySet(models.QuerySet): return self.filter(query) return self.exclude(query) + def subscribed(self, actor): + if not actor: + return self.none() + + subscriptions = actor.emitted_follows.filter( + approved=True, target__channel__isnull=False + ) + return self.filter(actor__in=subscriptions.values_list("target", flat=True)) + class Channel(models.Model): uuid = models.UUIDField(default=uuid.uuid4, unique=True) diff --git a/api/funkwhale_api/common/utils.py b/api/funkwhale_api/common/utils.py index 917bc13b7..7d750ff22 100644 --- a/api/funkwhale_api/common/utils.py +++ b/api/funkwhale_api/common/utils.py @@ -295,6 +295,8 @@ def clean_html(html, permissive=False): def render_html(text, content_type, permissive=False): + if not text: + return "" rendered = render_markdown(text) if content_type == "text/html": rendered = text @@ -307,6 +309,8 @@ def render_html(text, content_type, permissive=False): def render_plain_text(html): + if not html: + return "" return bleach.clean(html, tags=[], strip=True) diff --git a/api/funkwhale_api/subsonic/serializers.py b/api/funkwhale_api/subsonic/serializers.py index 7b6e37686..f45b54fac 100644 --- a/api/funkwhale_api/subsonic/serializers.py +++ b/api/funkwhale_api/subsonic/serializers.py @@ -102,7 +102,7 @@ def get_track_data(album, track, upload): "id": track.pk, "isDir": "false", "title": track.title, - "album": album.title, + "album": album.title if album else "", "artist": album.artist.name, "track": track.position or 1, "discNumber": track.disc_number or 1, @@ -118,18 +118,20 @@ def get_track_data(album, track, upload): "path": get_track_path(track, upload.extension or "mp3"), "duration": upload.duration or 0, "created": to_subsonic_date(track.creation_date), - "albumId": album.pk, - "artistId": album.artist.pk, + "albumId": album.pk if album else "", + "artistId": album.artist.pk if album else track.artist.pk, "type": "music", } - if track.album.attachment_cover_id: - data["coverArt"] = "al-{}".format(track.album.id) + if album and album.attachment_cover_id: + data["coverArt"] = "al-{}".format(album.id) if upload.bitrate: data["bitrate"] = int(upload.bitrate / 1000) if upload.size: data["size"] = upload.size if album.release_date: data["year"] = album.release_date.year + else: + data["year"] = track.creation_date.year return data @@ -287,7 +289,7 @@ def get_user_detail_data(user): "adminRole": "false", "settingsRole": "false", "commentRole": "false", - "podcastRole": "false", + "podcastRole": "true", "coverArtRole": "false", "shareRole": "false", "uploadRole": "true", @@ -319,3 +321,53 @@ def get_genre_data(tag): "albumCount": getattr(tag, "_albums_count", 0), "value": tag.name, } + + +def get_channel_data(channel, uploads): + data = { + "id": str(channel.uuid), + "url": channel.get_rss_url(), + "title": channel.artist.name, + "description": channel.artist.description.as_plain_text + if channel.artist.description + else "", + "coverArt": "at-{}".format(channel.artist.attachment_cover.uuid) + if channel.artist.attachment_cover + else "", + "originalImageUrl": channel.artist.attachment_cover.url + if channel.artist.attachment_cover + else "", + "status": "completed", + } + if uploads: + data["episode"] = [ + get_channel_episode_data(upload, channel.uuid) for upload in uploads + ] + + return data + + +def get_channel_episode_data(upload, channel_id): + return { + "id": str(upload.uuid), + "channelId": str(channel_id), + "streamId": upload.track.id, + "title": upload.track.title, + "description": upload.track.description.as_plain_text + if upload.track.description + else "", + "coverArt": "at-{}".format(upload.track.attachment_cover.uuid) + if upload.track.attachment_cover + else "", + "isDir": "false", + "year": upload.track.creation_date.year, + "publishDate": upload.track.creation_date.isoformat(), + "created": upload.track.creation_date.isoformat(), + "genre": "Podcast", + "size": upload.size if upload.size else "", + "duration": upload.duration if upload.duration else "", + "bitrate": upload.bitrate / 1000 if upload.bitrate else "", + "contentType": upload.mimetype or "audio/mpeg", + "suffix": upload.extension or "mp3", + "status": "completed", + } diff --git a/api/funkwhale_api/subsonic/views.py b/api/funkwhale_api/subsonic/views.py index e1d4d10b7..e7c619a79 100644 --- a/api/funkwhale_api/subsonic/views.py +++ b/api/funkwhale_api/subsonic/views.py @@ -6,7 +6,8 @@ import functools from django.conf import settings from django.contrib.contenttypes.models import ContentType -from django.db.models import Count, Q +from django.db import transaction +from django.db.models import Count, Prefetch, Q from django.utils import timezone from rest_framework import exceptions from rest_framework import permissions as rest_permissions @@ -16,12 +17,17 @@ from rest_framework.serializers import ValidationError import funkwhale_api from funkwhale_api.activity import record +from funkwhale_api.audio import models as audio_models +from funkwhale_api.audio import serializers as audio_serializers +from funkwhale_api.audio import views as audio_views from funkwhale_api.common import ( fields, preferences, + models as common_models, utils as common_utils, tasks as common_tasks, ) +from funkwhale_api.federation import models as federation_models from funkwhale_api.favorites.models import TrackFavorite from funkwhale_api.moderation import filters as moderation_filters from funkwhale_api.music import models as music_models @@ -101,6 +107,22 @@ def get_playlist_qs(request): return qs.order_by("-creation_date") +def requires_channels(f): + @functools.wraps(f) + def inner(*args, **kwargs): + if not preferences.get("audio__channels_enabled"): + payload = { + "error": { + "code": 0, + "message": "Channels / podcasts are disabled on this pod", + } + } + return response.Response(payload, status=405) + return f(*args, **kwargs) + + return inner + + class SubsonicViewSet(viewsets.GenericViewSet): content_negotiation_class = negotiation.SubsonicContentNegociation authentication_classes = [authentication.SubsonicAuthentication] @@ -752,6 +774,14 @@ class SubsonicViewSet(viewsets.GenericViewSet): {"error": {"code": 70, "message": "cover art not found."}} ) attachment = album.attachment_cover + elif id.startswith("at-"): + try: + attachment_id = id.replace("at-", "") + attachment = common_models.Attachment.objects.get(uuid=attachment_id) + except (TypeError, ValueError, music_models.Album.DoesNotExist): + return response.Response( + {"error": {"code": 70, "message": "cover art not found."}} + ) else: return response.Response( {"error": {"code": 70, "message": "cover art not found."}} @@ -810,3 +840,149 @@ class SubsonicViewSet(viewsets.GenericViewSet): "genres": {"genre": [serializers.get_genre_data(tag) for tag in queryset]} } return response.Response(data) + + # podcast related views + @action( + detail=False, + methods=["get", "post"], + url_name="create_podcast_channel", + url_path="createPodcastChannel", + ) + @requires_channels + @transaction.atomic + def create_podcast_channel(self, request, *args, **kwargs): + data = request.GET or request.POST + serializer = audio_serializers.RssSubscribeSerializer(data=data) + if not serializer.is_valid(): + return response.Response({"error": {"code": 0, "message": "invalid url"}}) + channel = ( + audio_models.Channel.objects.filter( + rss_url=serializer.validated_data["url"], + ) + .order_by("id") + .first() + ) + if not channel: + # try to retrieve the channel via its URL and create it + try: + channel, uploads = audio_serializers.get_channel_from_rss_url( + serializer.validated_data["url"] + ) + except audio_serializers.FeedFetchException as e: + return response.Response( + { + "error": { + "code": 0, + "message": "Error while fetching url: {}".format(e), + } + } + ) + + subscription = federation_models.Follow(actor=request.user.actor) + subscription.fid = subscription.get_federation_id() + audio_views.SubscriptionsViewSet.queryset.get_or_create( + target=channel.actor, + actor=request.user.actor, + defaults={ + "approved": True, + "fid": subscription.fid, + "uuid": subscription.uuid, + }, + ) + return response.Response({"status": "ok"}) + + @action( + detail=False, + methods=["get", "post"], + url_name="delete_podcast_channel", + url_path="deletePodcastChannel", + ) + @requires_channels + @find_object( + audio_models.Channel.objects.all().select_related("actor"), + model_field="uuid", + field="id", + cast=str, + ) + def delete_podcast_channel(self, request, *args, **kwargs): + channel = kwargs.pop("obj") + actor = request.user.actor + actor.emitted_follows.filter(target=channel.actor).delete() + return response.Response({"status": "ok"}) + + @action( + detail=False, + methods=["get", "post"], + url_name="get_podcasts", + url_path="getPodcasts", + ) + @requires_channels + def get_podcasts(self, request, *args, **kwargs): + data = request.GET or request.POST + id = data.get("id") + channels = audio_models.Channel.objects.subscribed(request.user.actor) + if id: + channels = channels.filter(uuid=id) + channels = channels.select_related( + "artist__attachment_cover", "artist__description", "library", "actor" + ) + uploads_qs = ( + music_models.Upload.objects.playable_by(request.user.actor) + .select_related("track__attachment_cover", "track__description",) + .order_by("-track__creation_date") + ) + + if data.get("includeEpisodes", "true") == "true": + channels = channels.prefetch_related( + Prefetch( + "library__uploads", + queryset=uploads_qs, + to_attr="_prefetched_uploads", + ) + ) + + data = { + "podcasts": { + "channel": [ + serializers.get_channel_data( + channel, getattr(channel.library, "_prefetched_uploads", []) + ) + for channel in channels + ] + }, + } + return response.Response(data) + + @action( + detail=False, + methods=["get", "post"], + url_name="get_newest_podcasts", + url_path="getNewestPodcasts", + ) + @requires_channels + def get_newest_podcasts(self, request, *args, **kwargs): + data = request.GET or request.POST + try: + count = int(data["count"]) + except (TypeError, KeyError, ValueError): + count = 20 + channels = audio_models.Channel.objects.subscribed(request.user.actor) + uploads = ( + music_models.Upload.objects.playable_by(request.user.actor) + .filter(library__channel__in=channels) + .select_related( + "track__attachment_cover", "track__description", "library__channel" + ) + .order_by("-track__creation_date") + ) + data = { + "newestPodcasts": { + "episode": [ + serializers.get_channel_episode_data( + upload, upload.library.channel.uuid + ) + for upload in uploads[:count] + ] + } + } + return response.Response(data) diff --git a/api/setup.cfg b/api/setup.cfg index 8872573e9..44718f388 100644 --- a/api/setup.cfg +++ b/api/setup.cfg @@ -33,3 +33,5 @@ env = PROXY_MEDIA=true MUSIC_USE_DENORMALIZATION=true EXTERNAL_MEDIA_PROXY_ENABLED=true + DISABLE_PASSWORD_VALIDATORS=false + DISABLE_PASSWORD_VALIDATORS=false diff --git a/api/tests/subsonic/test_serializers.py b/api/tests/subsonic/test_serializers.py index 4da84ec35..14495ec95 100644 --- a/api/tests/subsonic/test_serializers.py +++ b/api/tests/subsonic/test_serializers.py @@ -302,3 +302,55 @@ def test_scrobble_serializer(factories): assert listening.user == user assert listening.track == track + + +def test_channel_serializer(factories): + description = factories["common.Content"]() + channel = factories["audio.Channel"](external=True, artist__description=description) + upload = factories["music.Upload"]( + playable=True, library=channel.library, duration=42 + ) + + expected = { + "id": str(channel.uuid), + "url": channel.rss_url, + "title": channel.artist.name, + "description": description.as_plain_text, + "coverArt": "at-{}".format(channel.artist.attachment_cover.uuid), + "originalImageUrl": channel.artist.attachment_cover.url, + "status": "completed", + "episode": [serializers.get_channel_episode_data(upload, channel.uuid)], + } + data = serializers.get_channel_data(channel, [upload]) + assert data == expected + + +def test_channel_episode_serializer(factories): + description = factories["common.Content"]() + channel = factories["audio.Channel"]() + track = factories["music.Track"](description=description, artist=channel.artist) + upload = factories["music.Upload"]( + playable=True, track=track, bitrate=128000, duration=42 + ) + + expected = { + "id": str(upload.uuid), + "channelId": str(channel.uuid), + "streamId": upload.track.id, + "title": track.title, + "description": description.as_plain_text, + "coverArt": "at-{}".format(track.attachment_cover.uuid), + "isDir": "false", + "year": track.creation_date.year, + "created": track.creation_date.isoformat(), + "publishDate": track.creation_date.isoformat(), + "genre": "Podcast", + "size": upload.size, + "duration": upload.duration, + "bitrate": upload.bitrate / 1000, + "contentType": upload.mimetype, + "suffix": upload.extension, + "status": "completed", + } + data = serializers.get_channel_episode_data(upload, channel.uuid) + assert data == expected diff --git a/api/tests/subsonic/test_views.py b/api/tests/subsonic/test_views.py index 24c66273b..8b576e1e2 100644 --- a/api/tests/subsonic/test_views.py +++ b/api/tests/subsonic/test_views.py @@ -731,6 +731,19 @@ def test_get_cover_art_album(factories, logged_in_api_client): ).decode("utf-8") +def test_get_cover_art_attachment(factories, logged_in_api_client): + attachment = factories["common.Attachment"]() + url = reverse("api:subsonic-get_cover_art") + assert url.endswith("getCoverArt") is True + response = logged_in_api_client.get(url, {"id": "at-{}".format(attachment.uuid)}) + + assert response.status_code == 200 + assert response["Content-Type"] == "" + assert response["X-Accel-Redirect"] == music_views.get_file_path( + attachment.file + ).decode("utf-8") + + def test_get_avatar(factories, logged_in_api_client): user = factories["users.User"]() url = reverse("api:subsonic-get_avatar") @@ -776,7 +789,7 @@ def test_get_user(f, db, logged_in_api_client, factories): "settingsRole": "false", "playlistRole": "true", "commentRole": "false", - "podcastRole": "false", + "podcastRole": "true", "streamRole": "true", "jukeboxRole": "true", "coverArtRole": "false", @@ -787,3 +800,138 @@ def test_get_user(f, db, logged_in_api_client, factories): ], } } + + +def test_create_podcast_channel(logged_in_api_client, factories, mocker): + channel = factories["audio.Channel"](external=True) + rss_url = "https://rss.url/" + get_channel_from_rss_url = mocker.patch( + "funkwhale_api.audio.serializers.get_channel_from_rss_url", + return_value=(channel, []), + ) + actor = logged_in_api_client.user.create_actor() + url = reverse("api:subsonic-create_podcast_channel") + assert url.endswith("createPodcastChannel") is True + response = logged_in_api_client.get(url, {"f": "json", "url": rss_url}) + assert response.status_code == 200 + assert response.data == {"status": "ok"} + + subscription = actor.emitted_follows.get(target=channel.actor) + assert subscription.approved is True + get_channel_from_rss_url.assert_called_once_with(rss_url) + + +def test_delete_podcast_channel(logged_in_api_client, factories, mocker): + actor = logged_in_api_client.user.create_actor() + channel = factories["audio.Channel"](external=True) + subscription = factories["federation.Follow"](actor=actor, target=channel.actor) + other_subscription = factories["federation.Follow"](target=channel.actor) + url = reverse("api:subsonic-delete_podcast_channel") + assert url.endswith("deletePodcastChannel") is True + response = logged_in_api_client.get(url, {"f": "json", "id": channel.uuid}) + assert response.status_code == 200 + assert response.data == {"status": "ok"} + other_subscription.refresh_from_db() + with pytest.raises(subscription.DoesNotExist): + subscription.refresh_from_db() + + +def test_get_podcasts(logged_in_api_client, factories, mocker): + actor = logged_in_api_client.user.create_actor() + channel = factories["audio.Channel"]( + external=True, library__privacy_level="everyone" + ) + upload1 = factories["music.Upload"]( + playable=True, + track__artist=channel.artist, + library=channel.library, + bitrate=128000, + duration=42, + ) + upload2 = factories["music.Upload"]( + playable=True, + track__artist=channel.artist, + library=channel.library, + bitrate=256000, + duration=43, + ) + factories["federation.Follow"](actor=actor, target=channel.actor, approved=True) + factories["music.Upload"](import_status="pending", track__artist=channel.artist) + factories["audio.Channel"](external=True) + factories["federation.Follow"]() + url = reverse("api:subsonic-get_podcasts") + assert url.endswith("getPodcasts") is True + response = logged_in_api_client.get(url, {"f": "json"}) + assert response.status_code == 200 + assert response.data == { + "podcasts": { + "channel": [serializers.get_channel_data(channel, [upload2, upload1])], + } + } + + +def test_get_podcasts_by_id(logged_in_api_client, factories, mocker): + actor = logged_in_api_client.user.create_actor() + channel1 = factories["audio.Channel"]( + external=True, library__privacy_level="everyone" + ) + channel2 = factories["audio.Channel"]( + external=True, library__privacy_level="everyone" + ) + upload1 = factories["music.Upload"]( + playable=True, + track__artist=channel1.artist, + library=channel1.library, + bitrate=128000, + duration=42, + ) + factories["music.Upload"]( + playable=True, + track__artist=channel2.artist, + library=channel2.library, + bitrate=256000, + duration=43, + ) + factories["federation.Follow"](actor=actor, target=channel1.actor, approved=True) + factories["federation.Follow"](actor=actor, target=channel2.actor, approved=True) + url = reverse("api:subsonic-get_podcasts") + assert url.endswith("getPodcasts") is True + response = logged_in_api_client.get(url, {"f": "json", "id": channel1.uuid}) + assert response.status_code == 200 + assert response.data == { + "podcasts": {"channel": [serializers.get_channel_data(channel1, [upload1])]} + } + + +def test_get_newest_podcasts(logged_in_api_client, factories, mocker): + actor = logged_in_api_client.user.create_actor() + channel = factories["audio.Channel"]( + external=True, library__privacy_level="everyone" + ) + upload1 = factories["music.Upload"]( + playable=True, + track__artist=channel.artist, + library=channel.library, + bitrate=128000, + duration=42, + ) + upload2 = factories["music.Upload"]( + playable=True, + track__artist=channel.artist, + library=channel.library, + bitrate=256000, + duration=43, + ) + factories["federation.Follow"](actor=actor, target=channel.actor, approved=True) + url = reverse("api:subsonic-get_newest_podcasts") + assert url.endswith("getNewestPodcasts") is True + response = logged_in_api_client.get(url, {"f": "json"}) + assert response.status_code == 200 + assert response.data == { + "newestPodcasts": { + "episode": [ + serializers.get_channel_episode_data(upload, channel.uuid) + for upload in [upload2, upload1] + ], + } + }