diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3f787fec..a089db796 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,7 +203,7 @@ pages: - cd docs - apt-get update - apt-get install -y graphviz - - pip install sphinx sphinx_rtd_theme + - pip install sphinx sphinx_rtd_theme django-environ django script: - ./build_docs.sh cache: diff --git a/api/config/settings/common.py b/api/config/settings/common.py index b55cfe84a..675d3e8ce 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -1302,3 +1302,10 @@ PODCASTS_RSS_FEED_MAX_ITEMS = env.int("PODCASTS_RSS_FEED_MAX_ITEMS", default=250 """ Maximum number of RSS items to load in each podcast feed. """ + +IGNORE_FORWARDED_HOST_AND_PROTO = env.bool( + "IGNORE_FORWARDED_HOST_AND_PROTO", default=True +) +""" +Use :attr:`FUNKWHALE_HOSTNAME` and :attr:`FUNKWHALE_PROTOCOL ` instead of request header. +""" diff --git a/api/funkwhale_api/common/apps.py b/api/funkwhale_api/common/apps.py index cd671be29..7d94695a1 100644 --- a/api/funkwhale_api/common/apps.py +++ b/api/funkwhale_api/common/apps.py @@ -1,6 +1,7 @@ from django.apps import AppConfig, apps from . import mutations +from . import utils class CommonConfig(AppConfig): @@ -11,3 +12,4 @@ class CommonConfig(AppConfig): app_names = [app.name for app in apps.app_configs.values()] mutations.registry.autodiscover(app_names) + utils.monkey_patch_request_build_absolute_uri() diff --git a/api/funkwhale_api/common/utils.py b/api/funkwhale_api/common/utils.py index b721bdf7b..0a38ef05f 100644 --- a/api/funkwhale_api/common/utils.py +++ b/api/funkwhale_api/common/utils.py @@ -1,6 +1,7 @@ import datetime from django.core.files.base import ContentFile +from django.http import request from django.utils.deconstruct import deconstructible import bleach.sanitizer @@ -433,3 +434,27 @@ def update_modification_date(obj, field="modification_date", date=None): obj.__class__.objects.filter(pk=obj.pk).update(**{field: date}) return date + + +def monkey_patch_request_build_absolute_uri(): + """ + Since we have FUNKWHALE_HOSTNAME and PROTOCOL hardcoded in settings, we can + override django's multisite logic which can break when reverse proxy aren't configured + properly. + """ + builtin_scheme = request.HttpRequest.scheme + + def scheme(self): + if settings.IGNORE_FORWARDED_HOST_AND_PROTO: + return settings.FUNKWHALE_PROTOCOL + return builtin_scheme.fget(self) + + builtin_get_host = request.HttpRequest.get_host + + def get_host(self): + if settings.IGNORE_FORWARDED_HOST_AND_PROTO: + return settings.FUNKWHALE_HOSTNAME + return builtin_get_host(self) + + request.HttpRequest.scheme = property(scheme) + request.HttpRequest.get_host = get_host diff --git a/api/funkwhale_api/manage/views.py b/api/funkwhale_api/manage/views.py index 0f0f16ce0..adb7128e2 100644 --- a/api/funkwhale_api/manage/views.py +++ b/api/funkwhale_api/manage/views.py @@ -84,8 +84,8 @@ class ManageArtistViewSet( music_models.Artist.objects.all() .order_by("-id") .select_related("attributed_to", "attachment_cover", "channel") - .annotate(_tracks_count=Count("tracks")) - .annotate(_albums_count=Count("albums")) + .annotate(_tracks_count=Count("tracks", distinct=True)) + .annotate(_albums_count=Count("albums", distinct=True)) .prefetch_related(music_views.TAG_PREFETCH) ) serializer_class = serializers.ManageArtistSerializer diff --git a/api/funkwhale_api/music/management/commands/fix_uploads.py b/api/funkwhale_api/music/management/commands/fix_uploads.py index 94f8dd21c..582a837c4 100644 --- a/api/funkwhale_api/music/management/commands/fix_uploads.py +++ b/api/funkwhale_api/music/management/commands/fix_uploads.py @@ -16,20 +16,44 @@ class Command(BaseCommand): default=False, help="Do not execute anything", ) + parser.add_argument( + "--mimetypes", + action="store_true", + dest="mimetypes", + default=True, + help="Check and fix mimetypes", + ) + parser.add_argument( + "--audio-data", + action="store_true", + dest="data", + default=False, + help="Check and fix bitrate and duration, can be really slow because it needs to access files", + ) + parser.add_argument( + "--size", + action="store_true", + dest="size", + default=False, + help="Check and fix file size, can be really slow because it needs to access files", + ) def handle(self, *args, **options): if options["dry_run"]: self.stdout.write("Dry-run on, will not commit anything") - self.fix_mimetypes(**options) - self.fix_file_data(**options) - self.fix_file_size(**options) + if options["mimetypes"]: + self.fix_mimetypes(**options) + if options["data"]: + self.fix_file_data(**options) + if options["size"]: + self.fix_file_size(**options) @transaction.atomic def fix_mimetypes(self, dry_run, **kwargs): self.stdout.write("Fixing missing mimetypes...") - matching = models.Upload.objects.filter(source__startswith="file://").exclude( - mimetype__startswith="audio/" - ) + matching = models.Upload.objects.filter( + Q(source__startswith="file://") | Q(source__startswith="upload://") + ).exclude(mimetype__startswith="audio/") self.stdout.write( "[mimetypes] {} entries found with bad or no mimetype".format( matching.count() diff --git a/api/funkwhale_api/music/utils.py b/api/funkwhale_api/music/utils.py index 14f245aaa..64a7c24f8 100644 --- a/api/funkwhale_api/music/utils.py +++ b/api/funkwhale_api/music/utils.py @@ -22,6 +22,8 @@ def guess_mimetype(f): mt, _ = mimetypes.guess_type(f.name) if mt: t = mt + else: + t = EXTENSION_TO_MIMETYPE.get(f.name.split(".")[-1]) return t diff --git a/api/funkwhale_api/subsonic/serializers.py b/api/funkwhale_api/subsonic/serializers.py index fa6bc4475..d7042718a 100644 --- a/api/funkwhale_api/subsonic/serializers.py +++ b/api/funkwhale_api/subsonic/serializers.py @@ -130,7 +130,7 @@ def get_track_data(album, track, upload): data["bitrate"] = int(upload.bitrate / 1000) if upload.size: data["size"] = upload.size - if album.release_date: + if album and album.release_date: data["year"] = album.release_date.year else: data["year"] = track.creation_date.year diff --git a/api/tests/common/test_utils.py b/api/tests/common/test_utils.py index af2b25207..ffe9eae3b 100644 --- a/api/tests/common/test_utils.py +++ b/api/tests/common/test_utils.py @@ -197,3 +197,64 @@ def test_attach_file_content(factories, r_mock): assert new_attachment.file.read() == b"content" assert new_attachment.url is None assert new_attachment.mimetype == data["mimetype"] + + +@pytest.mark.parametrize( + "ignore, hostname, protocol, meta, path, expected", + [ + ( + False, + "test.hostname", + "http", + { + "HTTP_X_FORWARDED_HOST": "real.hostname", + "HTTP_X_FORWARDED_PROTO": "https", + }, + "/hello", + "https://real.hostname/hello", + ), + ( + False, + "test.hostname", + "http", + { + "HTTP_X_FORWARDED_HOST": "real.hostname", + "HTTP_X_FORWARDED_PROTO": "http", + }, + "/hello", + "http://real.hostname/hello", + ), + ( + True, + "test.hostname", + "http", + { + "HTTP_X_FORWARDED_HOST": "real.hostname", + "HTTP_X_FORWARDED_PROTO": "https", + }, + "/hello", + "http://test.hostname/hello", + ), + ( + True, + "test.hostname", + "https", + { + "HTTP_X_FORWARDED_HOST": "real.hostname", + "HTTP_X_FORWARDED_PROTO": "http", + }, + "/hello", + "https://test.hostname/hello", + ), + ], +) +def test_monkey_patch_request_build_absolute_uri( + ignore, hostname, protocol, meta, path, expected, fake_request, settings +): + settings.IGNORE_FORWARDED_HOST_AND_PROTO = ignore + settings.ALLOWED_HOSTS = "*" + settings.FUNKWHALE_HOSTNAME = hostname + settings.FUNKWHALE_PROTOCOL = protocol + request = fake_request.get("/", **meta) + + assert request.build_absolute_uri(path) == expected diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py index db39f571f..3d79c0eba 100644 --- a/api/tests/music/test_views.py +++ b/api/tests/music/test_views.py @@ -824,6 +824,7 @@ def test_user_can_create_draft_upload( assert upload.source == "upload://test" assert upload.import_reference == "test" assert upload.import_status == "draft" + assert upload.mimetype == "audio/ogg" assert upload.track is None m.assert_not_called() diff --git a/changes/changelog.d/1082.bugfix b/changes/changelog.d/1082.bugfix new file mode 100644 index 000000000..b99f0c1f0 --- /dev/null +++ b/changes/changelog.d/1082.bugfix @@ -0,0 +1 @@ +Fixed issue when displaying starred tracks on subsonic (#1082) diff --git a/changes/changelog.d/1085.enhancement b/changes/changelog.d/1085.enhancement new file mode 100644 index 000000000..49cbee6f8 --- /dev/null +++ b/changes/changelog.d/1085.enhancement @@ -0,0 +1 @@ +Make URL-building logic more resilient against reverse proxy misconfiguration (#1085) diff --git a/changes/changelog.d/1093.bugfix b/changes/changelog.d/1093.bugfix new file mode 100644 index 000000000..1d9a15e59 --- /dev/null +++ b/changes/changelog.d/1093.bugfix @@ -0,0 +1 @@ +Fixed mimetype detection issue that broke transcoding on some tracks (#1093). Run ``python manage.py fix_uploads --mimetypes`` to set proper mimetypes on existing uploads. diff --git a/changes/changelog.d/1096.bugfix b/changes/changelog.d/1096.bugfix new file mode 100644 index 000000000..81ffb284d --- /dev/null +++ b/changes/changelog.d/1096.bugfix @@ -0,0 +1 @@ +Fixed wrong album and track count in admin artist API (#1096)