From 9f1da096212dee5fed3432279670667aa26c8922 Mon Sep 17 00:00:00 2001 From: Agate Date: Mon, 4 May 2020 11:20:47 +0200 Subject: [PATCH 1/6] Fixed broken docs build on master --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3f787fec..a089db796 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,7 +203,7 @@ pages: - cd docs - apt-get update - apt-get install -y graphviz - - pip install sphinx sphinx_rtd_theme + - pip install sphinx sphinx_rtd_theme django-environ django script: - ./build_docs.sh cache: From e981f005dc2eb9447c0cccb1fd7fa4f01be02f90 Mon Sep 17 00:00:00 2001 From: Tony Wasserka Date: Sat, 25 Apr 2020 11:58:52 +0200 Subject: [PATCH 2/6] Fix spelling of "occurrences" --- CHANGELOG | 4 ++-- .../federation/management/commands/fix_federation_ids.py | 2 +- docs/admin/url.rst | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 569dbfe36..28205caea 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1952,11 +1952,11 @@ existing instance, do a backup of your ``.env`` and ``docker-compose.yml`` files - Ensure MUSIC_DIRECTORY_SERVE_PATH is uncommented and set to the absolute path on your host were your music files are stored (``/srv/funkwhale/data/music`` by default) - Edit your docker-compose.yml file to reflect the changes: - - Search for volumes (there should be two occurences) that contains ``/app/funkwhale_api/media`` on the right side, and + - Search for volumes (there should be two occurrences) that contains ``/app/funkwhale_api/media`` on the right side, and replace the whole line with ``- "${MEDIA_ROOT}:${MEDIA_ROOT}"`` - Search for a volume that contains ``/app/staticfiles`` on the right side, and replace the whole line with ``- "${STATIC_ROOT}:${STATIC_ROOT}"`` - - If you use in-place import, search for volumes (there should be two occurences) that contains ``/music:ro`` on the right side, and + - If you use in-place import, search for volumes (there should be two occurrences) that contains ``/music:ro`` on the right side, and replace the whole line with ``- "${MUSIC_DIRECTORY_SERVE_PATH}:${MUSIC_DIRECTORY_PATH}:ro"`` In the end, the ``volumes`` directives of your containers should look like that:: diff --git a/api/funkwhale_api/federation/management/commands/fix_federation_ids.py b/api/funkwhale_api/federation/management/commands/fix_federation_ids.py index e53eb62b4..764a9c1e4 100644 --- a/api/funkwhale_api/federation/management/commands/fix_federation_ids.py +++ b/api/funkwhale_api/federation/management/commands/fix_federation_ids.py @@ -78,7 +78,7 @@ class Command(BaseCommand): if total: self.stdout.write( self.style.WARNING( - "Will replace {} found occurences of '{}' by '{}':".format( + "Will replace {} found occurrences of '{}' by '{}':".format( total, old_prefix, new_prefix ) ) diff --git a/docs/admin/url.rst b/docs/admin/url.rst index daa11a805..0e37a6208 100644 --- a/docs/admin/url.rst +++ b/docs/admin/url.rst @@ -22,7 +22,7 @@ Example output: python manage.py fix_federation_ids https://old-url https://new-url --no-dry-run --no-input # Output - Will replace 108 found occurences of 'https://old-url' by 'https://new-url': + Will replace 108 found occurrences of 'https://old-url' by 'https://new-url': - 20 music.Artist - 13 music.Album From e8efa4213a4495cc760106c81889b9a80ba856ae Mon Sep 17 00:00:00 2001 From: Agate Date: Mon, 4 May 2020 12:02:08 +0200 Subject: [PATCH 3/6] Fix #1085: Make URL-building logic more resilient against reverse proxy misconfiguration --- api/config/settings/common.py | 7 ++++ api/funkwhale_api/common/apps.py | 2 + api/funkwhale_api/common/utils.py | 25 ++++++++++++ api/tests/common/test_utils.py | 61 ++++++++++++++++++++++++++++ changes/changelog.d/1085.enhancement | 1 + 5 files changed, 96 insertions(+) create mode 100644 changes/changelog.d/1085.enhancement diff --git a/api/config/settings/common.py b/api/config/settings/common.py index b55cfe84a..675d3e8ce 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -1302,3 +1302,10 @@ PODCASTS_RSS_FEED_MAX_ITEMS = env.int("PODCASTS_RSS_FEED_MAX_ITEMS", default=250 """ Maximum number of RSS items to load in each podcast feed. """ + +IGNORE_FORWARDED_HOST_AND_PROTO = env.bool( + "IGNORE_FORWARDED_HOST_AND_PROTO", default=True +) +""" +Use :attr:`FUNKWHALE_HOSTNAME` and :attr:`FUNKWHALE_PROTOCOL ` instead of request header. +""" diff --git a/api/funkwhale_api/common/apps.py b/api/funkwhale_api/common/apps.py index cd671be29..7d94695a1 100644 --- a/api/funkwhale_api/common/apps.py +++ b/api/funkwhale_api/common/apps.py @@ -1,6 +1,7 @@ from django.apps import AppConfig, apps from . import mutations +from . import utils class CommonConfig(AppConfig): @@ -11,3 +12,4 @@ class CommonConfig(AppConfig): app_names = [app.name for app in apps.app_configs.values()] mutations.registry.autodiscover(app_names) + utils.monkey_patch_request_build_absolute_uri() diff --git a/api/funkwhale_api/common/utils.py b/api/funkwhale_api/common/utils.py index b721bdf7b..0a38ef05f 100644 --- a/api/funkwhale_api/common/utils.py +++ b/api/funkwhale_api/common/utils.py @@ -1,6 +1,7 @@ import datetime from django.core.files.base import ContentFile +from django.http import request from django.utils.deconstruct import deconstructible import bleach.sanitizer @@ -433,3 +434,27 @@ def update_modification_date(obj, field="modification_date", date=None): obj.__class__.objects.filter(pk=obj.pk).update(**{field: date}) return date + + +def monkey_patch_request_build_absolute_uri(): + """ + Since we have FUNKWHALE_HOSTNAME and PROTOCOL hardcoded in settings, we can + override django's multisite logic which can break when reverse proxy aren't configured + properly. + """ + builtin_scheme = request.HttpRequest.scheme + + def scheme(self): + if settings.IGNORE_FORWARDED_HOST_AND_PROTO: + return settings.FUNKWHALE_PROTOCOL + return builtin_scheme.fget(self) + + builtin_get_host = request.HttpRequest.get_host + + def get_host(self): + if settings.IGNORE_FORWARDED_HOST_AND_PROTO: + return settings.FUNKWHALE_HOSTNAME + return builtin_get_host(self) + + request.HttpRequest.scheme = property(scheme) + request.HttpRequest.get_host = get_host diff --git a/api/tests/common/test_utils.py b/api/tests/common/test_utils.py index af2b25207..ffe9eae3b 100644 --- a/api/tests/common/test_utils.py +++ b/api/tests/common/test_utils.py @@ -197,3 +197,64 @@ def test_attach_file_content(factories, r_mock): assert new_attachment.file.read() == b"content" assert new_attachment.url is None assert new_attachment.mimetype == data["mimetype"] + + +@pytest.mark.parametrize( + "ignore, hostname, protocol, meta, path, expected", + [ + ( + False, + "test.hostname", + "http", + { + "HTTP_X_FORWARDED_HOST": "real.hostname", + "HTTP_X_FORWARDED_PROTO": "https", + }, + "/hello", + "https://real.hostname/hello", + ), + ( + False, + "test.hostname", + "http", + { + "HTTP_X_FORWARDED_HOST": "real.hostname", + "HTTP_X_FORWARDED_PROTO": "http", + }, + "/hello", + "http://real.hostname/hello", + ), + ( + True, + "test.hostname", + "http", + { + "HTTP_X_FORWARDED_HOST": "real.hostname", + "HTTP_X_FORWARDED_PROTO": "https", + }, + "/hello", + "http://test.hostname/hello", + ), + ( + True, + "test.hostname", + "https", + { + "HTTP_X_FORWARDED_HOST": "real.hostname", + "HTTP_X_FORWARDED_PROTO": "http", + }, + "/hello", + "https://test.hostname/hello", + ), + ], +) +def test_monkey_patch_request_build_absolute_uri( + ignore, hostname, protocol, meta, path, expected, fake_request, settings +): + settings.IGNORE_FORWARDED_HOST_AND_PROTO = ignore + settings.ALLOWED_HOSTS = "*" + settings.FUNKWHALE_HOSTNAME = hostname + settings.FUNKWHALE_PROTOCOL = protocol + request = fake_request.get("/", **meta) + + assert request.build_absolute_uri(path) == expected diff --git a/changes/changelog.d/1085.enhancement b/changes/changelog.d/1085.enhancement new file mode 100644 index 000000000..49cbee6f8 --- /dev/null +++ b/changes/changelog.d/1085.enhancement @@ -0,0 +1 @@ +Make URL-building logic more resilient against reverse proxy misconfiguration (#1085) From 4364f9b4d13b1ad230a7a8e200aa2e97722bc3f8 Mon Sep 17 00:00:00 2001 From: Agate Date: Mon, 4 May 2020 12:13:58 +0200 Subject: [PATCH 4/6] Fix #1096: Fixed wrong album and track count in admin artist API --- api/funkwhale_api/manage/views.py | 4 ++-- changes/changelog.d/1096.bugfix | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changes/changelog.d/1096.bugfix diff --git a/api/funkwhale_api/manage/views.py b/api/funkwhale_api/manage/views.py index 0f0f16ce0..adb7128e2 100644 --- a/api/funkwhale_api/manage/views.py +++ b/api/funkwhale_api/manage/views.py @@ -84,8 +84,8 @@ class ManageArtistViewSet( music_models.Artist.objects.all() .order_by("-id") .select_related("attributed_to", "attachment_cover", "channel") - .annotate(_tracks_count=Count("tracks")) - .annotate(_albums_count=Count("albums")) + .annotate(_tracks_count=Count("tracks", distinct=True)) + .annotate(_albums_count=Count("albums", distinct=True)) .prefetch_related(music_views.TAG_PREFETCH) ) serializer_class = serializers.ManageArtistSerializer diff --git a/changes/changelog.d/1096.bugfix b/changes/changelog.d/1096.bugfix new file mode 100644 index 000000000..81ffb284d --- /dev/null +++ b/changes/changelog.d/1096.bugfix @@ -0,0 +1 @@ +Fixed wrong album and track count in admin artist API (#1096) From 7b81d94e91c604b07ba936817bbd5d80d3a88f7a Mon Sep 17 00:00:00 2001 From: Agate Date: Mon, 4 May 2020 13:15:40 +0200 Subject: [PATCH 5/6] Fix #1082: issue when displaying starred tracks on subsonic --- api/funkwhale_api/subsonic/serializers.py | 2 +- changes/changelog.d/1082.bugfix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/changelog.d/1082.bugfix diff --git a/api/funkwhale_api/subsonic/serializers.py b/api/funkwhale_api/subsonic/serializers.py index fa6bc4475..d7042718a 100644 --- a/api/funkwhale_api/subsonic/serializers.py +++ b/api/funkwhale_api/subsonic/serializers.py @@ -130,7 +130,7 @@ def get_track_data(album, track, upload): data["bitrate"] = int(upload.bitrate / 1000) if upload.size: data["size"] = upload.size - if album.release_date: + if album and album.release_date: data["year"] = album.release_date.year else: data["year"] = track.creation_date.year diff --git a/changes/changelog.d/1082.bugfix b/changes/changelog.d/1082.bugfix new file mode 100644 index 000000000..b99f0c1f0 --- /dev/null +++ b/changes/changelog.d/1082.bugfix @@ -0,0 +1 @@ +Fixed issue when displaying starred tracks on subsonic (#1082) From 85bc8d93e87a73baee6d11e6028e73c45e96a83b Mon Sep 17 00:00:00 2001 From: Agate Date: Mon, 4 May 2020 13:52:25 +0200 Subject: [PATCH 6/6] Fix #1093: mimetype detection issue that broke transcoding on certain tracks --- .../music/management/commands/fix_uploads.py | 36 +++++++++++++++---- api/funkwhale_api/music/utils.py | 2 ++ api/tests/music/test_views.py | 1 + changes/changelog.d/1093.bugfix | 1 + 4 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 changes/changelog.d/1093.bugfix diff --git a/api/funkwhale_api/music/management/commands/fix_uploads.py b/api/funkwhale_api/music/management/commands/fix_uploads.py index 94f8dd21c..582a837c4 100644 --- a/api/funkwhale_api/music/management/commands/fix_uploads.py +++ b/api/funkwhale_api/music/management/commands/fix_uploads.py @@ -16,20 +16,44 @@ class Command(BaseCommand): default=False, help="Do not execute anything", ) + parser.add_argument( + "--mimetypes", + action="store_true", + dest="mimetypes", + default=True, + help="Check and fix mimetypes", + ) + parser.add_argument( + "--audio-data", + action="store_true", + dest="data", + default=False, + help="Check and fix bitrate and duration, can be really slow because it needs to access files", + ) + parser.add_argument( + "--size", + action="store_true", + dest="size", + default=False, + help="Check and fix file size, can be really slow because it needs to access files", + ) def handle(self, *args, **options): if options["dry_run"]: self.stdout.write("Dry-run on, will not commit anything") - self.fix_mimetypes(**options) - self.fix_file_data(**options) - self.fix_file_size(**options) + if options["mimetypes"]: + self.fix_mimetypes(**options) + if options["data"]: + self.fix_file_data(**options) + if options["size"]: + self.fix_file_size(**options) @transaction.atomic def fix_mimetypes(self, dry_run, **kwargs): self.stdout.write("Fixing missing mimetypes...") - matching = models.Upload.objects.filter(source__startswith="file://").exclude( - mimetype__startswith="audio/" - ) + matching = models.Upload.objects.filter( + Q(source__startswith="file://") | Q(source__startswith="upload://") + ).exclude(mimetype__startswith="audio/") self.stdout.write( "[mimetypes] {} entries found with bad or no mimetype".format( matching.count() diff --git a/api/funkwhale_api/music/utils.py b/api/funkwhale_api/music/utils.py index 14f245aaa..64a7c24f8 100644 --- a/api/funkwhale_api/music/utils.py +++ b/api/funkwhale_api/music/utils.py @@ -22,6 +22,8 @@ def guess_mimetype(f): mt, _ = mimetypes.guess_type(f.name) if mt: t = mt + else: + t = EXTENSION_TO_MIMETYPE.get(f.name.split(".")[-1]) return t diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py index db39f571f..3d79c0eba 100644 --- a/api/tests/music/test_views.py +++ b/api/tests/music/test_views.py @@ -824,6 +824,7 @@ def test_user_can_create_draft_upload( assert upload.source == "upload://test" assert upload.import_reference == "test" assert upload.import_status == "draft" + assert upload.mimetype == "audio/ogg" assert upload.track is None m.assert_not_called() diff --git a/changes/changelog.d/1093.bugfix b/changes/changelog.d/1093.bugfix new file mode 100644 index 000000000..1d9a15e59 --- /dev/null +++ b/changes/changelog.d/1093.bugfix @@ -0,0 +1 @@ +Fixed mimetype detection issue that broke transcoding on some tracks (#1093). Run ``python manage.py fix_uploads --mimetypes`` to set proper mimetypes on existing uploads.