Merge branch 'master' into develop

.gitlab-ci.yml

@@ -203,7 +203,7 @@ pages:
     - cd docs
     - apt-get update
     - apt-get install -y graphviz
-    - pip install sphinx sphinx_rtd_theme
+    - pip install sphinx sphinx_rtd_theme django-environ django
   script:
     - ./build_docs.sh
   cache:

api/config/settings/common.py

@@ -1302,3 +1302,10 @@ PODCASTS_RSS_FEED_MAX_ITEMS = env.int("PODCASTS_RSS_FEED_MAX_ITEMS", default=250
 """
 Maximum number of RSS items to load in each podcast feed.
 """
+
+IGNORE_FORWARDED_HOST_AND_PROTO = env.bool(
+    "IGNORE_FORWARDED_HOST_AND_PROTO", default=True
+)
+"""
+Use :attr:`FUNKWHALE_HOSTNAME` and :attr:`FUNKWHALE_PROTOCOL ` instead of request header.
+"""

api/funkwhale_api/common/apps.py

@@ -1,6 +1,7 @@
 from django.apps import AppConfig, apps
 
 from . import mutations
+from . import utils
 
 
 class CommonConfig(AppConfig):
@@ -11,3 +12,4 @@ class CommonConfig(AppConfig):
 
         app_names = [app.name for app in apps.app_configs.values()]
         mutations.registry.autodiscover(app_names)
+        utils.monkey_patch_request_build_absolute_uri()

api/funkwhale_api/common/utils.py

@@ -1,6 +1,7 @@
 import datetime
 
 from django.core.files.base import ContentFile
+from django.http import request
 from django.utils.deconstruct import deconstructible
 
 import bleach.sanitizer
@@ -433,3 +434,27 @@ def update_modification_date(obj, field="modification_date", date=None):
         obj.__class__.objects.filter(pk=obj.pk).update(**{field: date})
 
     return date
+
+
+def monkey_patch_request_build_absolute_uri():
+    """
+    Since we have FUNKWHALE_HOSTNAME and PROTOCOL hardcoded in settings, we can
+    override django's multisite logic which can break when reverse proxy aren't configured
+    properly.
+    """
+    builtin_scheme = request.HttpRequest.scheme
+
+    def scheme(self):
+        if settings.IGNORE_FORWARDED_HOST_AND_PROTO:
+            return settings.FUNKWHALE_PROTOCOL
+        return builtin_scheme.fget(self)
+
+    builtin_get_host = request.HttpRequest.get_host
+
+    def get_host(self):
+        if settings.IGNORE_FORWARDED_HOST_AND_PROTO:
+            return settings.FUNKWHALE_HOSTNAME
+        return builtin_get_host(self)
+
+    request.HttpRequest.scheme = property(scheme)
+    request.HttpRequest.get_host = get_host

api/funkwhale_api/manage/views.py

@@ -84,8 +84,8 @@ class ManageArtistViewSet(
         music_models.Artist.objects.all()
         .order_by("-id")
         .select_related("attributed_to", "attachment_cover", "channel")
-        .annotate(_tracks_count=Count("tracks"))
+        .annotate(_tracks_count=Count("tracks", distinct=True))
-        .annotate(_albums_count=Count("albums"))
+        .annotate(_albums_count=Count("albums", distinct=True))
         .prefetch_related(music_views.TAG_PREFETCH)
     )
     serializer_class = serializers.ManageArtistSerializer

api/funkwhale_api/music/management/commands/fix_uploads.py

@@ -16,20 +16,44 @@ class Command(BaseCommand):
             default=False,
             help="Do not execute anything",
         )
+        parser.add_argument(
+            "--mimetypes",
+            action="store_true",
+            dest="mimetypes",
+            default=True,
+            help="Check and fix mimetypes",
+        )
+        parser.add_argument(
+            "--audio-data",
+            action="store_true",
+            dest="data",
+            default=False,
+            help="Check and fix bitrate and duration, can be really slow because it needs to access files",
+        )
+        parser.add_argument(
+            "--size",
+            action="store_true",
+            dest="size",
+            default=False,
+            help="Check and fix file size, can be really slow because it needs to access files",
+        )
 
     def handle(self, *args, **options):
         if options["dry_run"]:
             self.stdout.write("Dry-run on, will not commit anything")
+        if options["mimetypes"]:
             self.fix_mimetypes(**options)
+        if options["data"]:
             self.fix_file_data(**options)
+        if options["size"]:
             self.fix_file_size(**options)
 
     @transaction.atomic
     def fix_mimetypes(self, dry_run, **kwargs):
         self.stdout.write("Fixing missing mimetypes...")
-        matching = models.Upload.objects.filter(source__startswith="file://").exclude(
+        matching = models.Upload.objects.filter(
-            mimetype__startswith="audio/"
+            Q(source__startswith="file://") | Q(source__startswith="upload://")
-        )
+        ).exclude(mimetype__startswith="audio/")
         self.stdout.write(
             "[mimetypes] {} entries found with bad or no mimetype".format(
                 matching.count()

api/funkwhale_api/music/utils.py

@@ -22,6 +22,8 @@ def guess_mimetype(f):
         mt, _ = mimetypes.guess_type(f.name)
         if mt:
             t = mt
+        else:
+            t = EXTENSION_TO_MIMETYPE.get(f.name.split(".")[-1])
     return t
 
 

api/funkwhale_api/subsonic/serializers.py

@@ -130,7 +130,7 @@ def get_track_data(album, track, upload):
         data["bitrate"] = int(upload.bitrate / 1000)
     if upload.size:
         data["size"] = upload.size
-    if album.release_date:
+    if album and album.release_date:
         data["year"] = album.release_date.year
     else:
         data["year"] = track.creation_date.year

api/tests/common/test_utils.py

@@ -197,3 +197,64 @@ def test_attach_file_content(factories, r_mock):
     assert new_attachment.file.read() == b"content"
     assert new_attachment.url is None
     assert new_attachment.mimetype == data["mimetype"]
+
+
+@pytest.mark.parametrize(
+    "ignore, hostname, protocol, meta, path, expected",
+    [
+        (
+            False,
+            "test.hostname",
+            "http",
+            {
+                "HTTP_X_FORWARDED_HOST": "real.hostname",
+                "HTTP_X_FORWARDED_PROTO": "https",
+            },
+            "/hello",
+            "https://real.hostname/hello",
+        ),
+        (
+            False,
+            "test.hostname",
+            "http",
+            {
+                "HTTP_X_FORWARDED_HOST": "real.hostname",
+                "HTTP_X_FORWARDED_PROTO": "http",
+            },
+            "/hello",
+            "http://real.hostname/hello",
+        ),
+        (
+            True,
+            "test.hostname",
+            "http",
+            {
+                "HTTP_X_FORWARDED_HOST": "real.hostname",
+                "HTTP_X_FORWARDED_PROTO": "https",
+            },
+            "/hello",
+            "http://test.hostname/hello",
+        ),
+        (
+            True,
+            "test.hostname",
+            "https",
+            {
+                "HTTP_X_FORWARDED_HOST": "real.hostname",
+                "HTTP_X_FORWARDED_PROTO": "http",
+            },
+            "/hello",
+            "https://test.hostname/hello",
+        ),
+    ],
+)
+def test_monkey_patch_request_build_absolute_uri(
+    ignore, hostname, protocol, meta, path, expected, fake_request, settings
+):
+    settings.IGNORE_FORWARDED_HOST_AND_PROTO = ignore
+    settings.ALLOWED_HOSTS = "*"
+    settings.FUNKWHALE_HOSTNAME = hostname
+    settings.FUNKWHALE_PROTOCOL = protocol
+    request = fake_request.get("/", **meta)
+
+    assert request.build_absolute_uri(path) == expected

api/tests/music/test_views.py

@@ -824,6 +824,7 @@ def test_user_can_create_draft_upload(
     assert upload.source == "upload://test"
     assert upload.import_reference == "test"
     assert upload.import_status == "draft"
+    assert upload.mimetype == "audio/ogg"
     assert upload.track is None
     m.assert_not_called()
 

changes/changelog.d/1082.bugfix

@@ -0,0 +1 @@
+Fixed issue when displaying starred tracks on subsonic (#1082)

changes/changelog.d/1085.enhancement

@@ -0,0 +1 @@
+Make URL-building logic more resilient against reverse proxy misconfiguration (#1085)

changes/changelog.d/1093.bugfix

@@ -0,0 +1 @@
+Fixed mimetype detection issue that broke transcoding on some tracks (#1093). Run ``python manage.py fix_uploads --mimetypes`` to set proper mimetypes on existing uploads.

changes/changelog.d/1096.bugfix

@@ -0,0 +1 @@
+Fixed wrong album and track count in admin artist API (#1096)