Cherry-pick resolved

.gitlab-ci.yml

@@ -231,7 +231,7 @@ test_api:
   image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-api:$PYTHON_VERSION
   parallel:
     matrix:
-      - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11"]
+      - PYTHON_VERSION: ["3.10", "3.11", "3.12", "3.13"]
   services:
     - name: postgres:15-alpine
       command:
@@ -508,3 +508,24 @@ docker:
     name: docker_metadata_${CI_COMMIT_REF_NAME}
     paths:
       - metadata.json
+
+package:
+  stage: publish
+  needs:
+    - job: build_metadata
+      artifacts: true
+    - job: build_api
+      artifacts: true
+    - job: build_front
+      artifacts: true
+  #   - job: build_tauri
+  #     artifacts: true
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+
+  image: $CI_REGISTRY/funkwhale/ci/python:3.11
+  variables:
+    <<: *keep_git_files_permissions
+  script:
+    - make package
+    - scripts/ci-upload-packages.sh

api/Dockerfile.alpine

@@ -0,0 +1,120 @@
+FROM alpine:3.21 AS requirements
+
+RUN set -eux; \
+  apk add --no-cache \
+  poetry \
+  py3-cryptography \
+  py3-pip \
+  python3
+
+COPY pyproject.toml poetry.lock /
+RUN set -eux; \
+  poetry export --without-hashes --extras typesense > requirements.txt; \
+  poetry export --without-hashes --with dev > dev-requirements.txt;
+
+FROM alpine:3.21 AS builder
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ARG PIP_NO_CACHE_DIR=1
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
+
+RUN set -eux; \
+  apk add --no-cache \
+  cargo \
+  curl \
+  gcc \
+  g++ \
+  git \
+  jpeg-dev \
+  libffi-dev \
+  libldap \
+  libxml2-dev \
+  libxslt-dev \
+  make \
+  musl-dev \
+  openldap-dev \
+  openssl-dev \
+  postgresql-dev \
+  zlib-dev \
+  py3-cryptography \
+  py3-lxml \
+  py3-pillow \
+  py3-psycopg2 \
+  py3-watchfiles \
+  python3-dev
+
+# Create virtual env
+RUN python3 -m venv --system-site-packages /venv
+ENV PATH="/venv/bin:$PATH"
+
+COPY --from=requirements /requirements.txt /requirements.txt
+COPY --from=requirements /dev-requirements.txt /dev-requirements.txt
+
+RUN --mount=type=cache,target=~/.cache/pip; \
+  set -eux; \
+  pip3 install --upgrade pip; \
+  pip3 install setuptools wheel; \
+  # Currently we are unable to relieably build rust-based packages on armv7. This
+  # is why we need to use the packages shipped by Alpine Linux.
+  # Since poetry does not allow in-place dependency pinning, we need
+  # to install the deps using pip.
+  grep -Ev 'cryptography|lxml|pillow|psycopg2|watchfiles' /requirements.txt \
+  | pip3 install -r /dev/stdin \
+  cryptography \
+  lxml \
+  pillow \
+  psycopg2 \
+  watchfiles
+
+ARG install_dev_deps=0
+RUN --mount=type=cache,target=~/.cache/pip; \
+  set -eux; \
+  if [ "$install_dev_deps" = "1" ] ; then \
+    grep -Ev 'cryptography|lxml|pillow|psycopg2|watchfiles' /dev-requirements.txt \
+    | pip3 install -r /dev/stdin \
+    cryptography \
+    lxml \
+    pillow \
+    psycopg2 \
+    watchfiles; \
+  fi
+
+FROM alpine:3.21 AS production
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ARG PIP_NO_CACHE_DIR=1
+
+RUN set -eux; \
+  apk add --no-cache \
+  bash \
+  ffmpeg \
+  gettext \
+  jpeg-dev \
+  libldap \
+  libmagic \
+  libpq \
+  libxml2 \
+  libxslt \
+  py3-cryptography \
+  py3-lxml \
+  py3-pillow \
+  py3-psycopg2 \
+  py3-watchfiles \
+  python3 \
+  tzdata
+
+COPY --from=builder /venv /venv
+ENV PATH="/venv/bin:$PATH"
+
+COPY . /app
+WORKDIR /app
+
+RUN --mount=type=cache,target=~/.cache/pip; \
+  set -eux; \
+  pip3 install --no-deps --editable .
+
+ENV IS_DOCKER_SETUP=true
+
+CMD ["./docker/server.sh"]

api/funkwhale_api/music/models.py

@@ -6,7 +6,6 @@ import urllib.parse
 import uuid
 
 import arrow
-import pydub
 from django.conf import settings
 from django.contrib.contenttypes.fields import GenericRelation
 from django.contrib.postgres.indexes import GinIndex
@@ -850,6 +849,12 @@ class Upload(models.Model):
         if self.source and self.source.startswith("file://"):
             return open(self.source.replace("file://", "", 1), "rb")
 
+    def get_audio_file_path(self):
+        if self.audio_file:
+            return self.audio_file.path
+        if self.source and self.source.startswith("file://"):
+            return self.source.replace("file://", "", 1)
+
     def get_audio_data(self):
         audio_file = self.get_audio_file()
         if not audio_file:
@@ -863,13 +868,47 @@ class Upload(models.Model):
             "size": self.get_file_size(),
         }
 
-    def get_audio_segment(self):
+    def get_quality(self):
-        input = self.get_audio_file()
+        extension_to_mimetypes = utils.get_extension_to_mimetype_dict()
-        if not input:
-            return
 
-        audio = pydub.AudioSegment.from_file(input)
+        if not self.bitrate and self.mimetype not in list(
-        return audio
+            itertools.chain(
+                extension_to_mimetypes["aiff"],
+                extension_to_mimetypes["aif"],
+                extension_to_mimetypes["flac"],
+            )
+        ):
+            return 1
+
+        bitrate_limits = {
+            "mp3": {192: 0, 256: 1, 320: 2},
+            "ogg": {96: 0, 192: 1, 256: 2},
+            "aac": {96: 0, 128: 1, 288: 2},
+            "m4a": {96: 0, 128: 1, 288: 2},
+            "opus": {
+                96: 0,
+                128: 1,
+                160: 2,
+            },
+        }
+
+        for ext in bitrate_limits:
+            if self.mimetype in extension_to_mimetypes[ext]:
+                for limit, quality in sorted(bitrate_limits[ext].items()):
+                    if int(self.bitrate) <= limit:
+                        return quality
+
+                # opus higher tham 160
+                return 3
+
+        if self.mimetype in list(
+            itertools.chain(
+                extension_to_mimetypes["aiff"],
+                extension_to_mimetypes["aif"],
+                extension_to_mimetypes["flac"],
+            )
+        ):
+            return 3
 
     def save(self, **kwargs):
         if not self.mimetype:
@@ -951,8 +990,8 @@ class Upload(models.Model):
         )
         version.audio_file.save(new_name, f)
         utils.transcode_audio(
-            audio=self.get_audio_segment(),
+            audio_file_path=self.get_audio_file_path(),
-            output=version.audio_file,
+            output_path=version.audio_file.path,
             output_format=utils.MIMETYPE_TO_EXTENSION[mimetype],
             bitrate=str(bitrate),
         )

api/funkwhale_api/music/utils.py

@@ -4,10 +4,10 @@ import pathlib
 
 import magic
 import mutagen
-import pydub
 from django.conf import settings
 from django.core.cache import cache
 from django.db.models import F
+from ffmpeg import FFmpeg
 
 from funkwhale_api.common import throttling
 from funkwhale_api.common.search import get_fts_query  # noqa
@@ -102,15 +102,10 @@ def get_actor_from_request(request):
     return actor
 
 
-def transcode_file(input, output, input_format=None, output_format="mp3", **kwargs):
+def transcode_audio(audio_file_path, output_path, output_format="mp3", **kwargs):
-    with input.open("rb"):
+    FFmpeg().input(audio_file_path).output(
-        audio = pydub.AudioSegment.from_file(input, format=input_format)
+        output_path, format=output_format, **kwargs
-    return transcode_audio(audio, output, output_format, **kwargs)
+    ).option("y").execute()
-
-
-def transcode_audio(audio, output, output_format, **kwargs):
-    with output.open("wb"):
-        return audio.export(output, format=output_format, **kwargs)
 
 
 def increment_downloads_count(upload, user, wsgi_request):

api/pyproject.toml

@@ -25,29 +25,29 @@ exclude = ["tests"]
 funkwhale-manage = 'funkwhale_api.main:main'
 
 [tool.poetry.dependencies]
-python = "^3.8,<3.12"
+python = "^3.10,<3.13"
 
 # Django
 dj-rest-auth = "5.0.2"
 django = "4.2.9"
 django-allauth = "0.55.2"
 django-cache-memoize = "0.1.10"
-django-cacheops = "==6.1"
+django-cacheops = "==7.0.2"
-django-cleanup = "==6.0.0"
+django-cleanup = "==8.1.0"
-django-cors-headers = "==3.13.0"
+django-cors-headers = "==4.3.1"
 django-dynamic-preferences = "==1.14.0"
-django-environ = "==0.10.0"
+django-environ = "==0.11.2"
-django-filter = "==22.1"
+django-filter = "==23.5"
 django-oauth-toolkit = "2.2.0"
 django-redis = "==5.2.0"
 django-storages = "==1.13.2"
-django-versatileimagefield = "==2.2"
+django-versatileimagefield = "==3.1"
 djangorestframework = "==3.14.0"
 drf-spectacular = "==0.26.5"
 markdown = "==3.4.4"
 persisting-theory = "==1.0"
-psycopg2 = "==2.9.9"
+psycopg2-binary = "==2.9.9"
-redis = "==4.5.5"
+redis = "==5.0.1"
 
 # Django LDAP
 django-auth-ldap = "==4.1.0"
@@ -58,69 +58,70 @@ channels = { extras = ["daphne"], version = "==4.0.0" }
 channels-redis = "==4.1.0"
 
 # Celery
-kombu = "==5.2.4"
+kombu = "5.3.4"
-celery = "==5.2.7"
+celery = "5.3.6"
 
 # Deployment
-gunicorn = "==20.1.0"
+gunicorn = "==21.2.0"
 uvicorn = { version = "==0.20.0", extras = ["standard"] }
 
 # Libs
-aiohttp = "==3.8.6"
+aiohttp = "3.9.1"
 arrow = "==1.2.3"
 backports-zoneinfo = { version = "==0.2.1", python = "<3.9" }
-bleach = "==5.0.1"
+bleach = "==6.1.0"
 boto3 = "==1.26.161"
 click = "==8.1.7"
-cryptography = "==38.0.4"
+cryptography = "==41.0.7"
-feedparser = "==6.0.11"
+feedparser = "==6.0.10"
+liblistenbrainz = "==0.5.5"
 musicbrainzngs = "==0.7.1"
 mutagen = "==1.46.0"
-pillow = "==9.3.0"
+pillow = "==10.2.0"
 pydub = "==0.25.1"
-pyld = "==2.0.4"
+pyld = "==2.0.3"
 python-magic = "==0.4.27"
-requests = "==2.28.2"
+requests = "==2.31.0"
 requests-http-message-signatures = "==0.3.1"
 sentry-sdk = "==1.19.1"
-watchdog = "==2.2.1"
+watchdog = "==4.0.0"
-troi = { git = "https://github.com/metabrainz/troi-recommendation-playground.git", tag = "v-2023-10-30.0"}
+troi = "==2024.1.26.0"
-lb-matching-tools = { git = "https://github.com/metabrainz/listenbrainz-matching-tools.git", branch = "main"}
+lb-matching-tools = "==2024.1.25.0rc1"
-unidecode = "==1.3.8"
+unidecode = "==1.3.7"
-pycountry = "22.3.5"
+pycountry = "23.12.11"
 
 # Typesense
 typesense = { version = "==0.15.1", optional = true }
 
 # Dependencies pinning
-ipython = "==7.34.0"
+ipython = "==8.12.3"
 pluralizer = "==1.2.0"
-service-identity = "==21.1.0"
+service-identity = "==24.1.0"
 unicode-slugify = "==0.1.5"
 
 [tool.poetry.group.dev.dependencies]
 aioresponses = "==0.7.6"
 asynctest = "==0.13.0"
-black = "==23.3.0"
+black = "==24.1.1"
-coverage = { version = "==6.5.0", extras = ["toml"] }
+coverage = { version = "==7.4.1", extras = ["toml"] }
 debugpy = "==1.6.7.post1"
 django-coverage-plugin = "==3.0.0"
-django-debug-toolbar = "==3.8.1"
+django-debug-toolbar = "==4.2.0"
 factory-boy = "==3.2.1"
-faker = "==15.3.4"
+faker = "==23.2.1"
 flake8 = "==3.9.2"
 ipdb = "==0.13.13"
-pytest = "==7.4.4"
+pytest = "==8.0.0"
 pytest-asyncio = "==0.21.0"
-prompt-toolkit = "==3.0.43"
+prompt-toolkit = "==3.0.41"
 pytest-cov = "==4.0.0"
 pytest-django = "==4.5.2"
-pytest-env = "==0.8.2"
+pytest-env = "==1.1.3"
 pytest-mock = "==3.10.0"
 pytest-randomly = "==3.12.0"
-pytest-sugar = "==0.9.7"
+pytest-sugar = "==1.0.0"
 requests-mock = "==1.10.0"
-pylint = "==2.17.7"
+pylint = "==3.0.3"
 pylint-django = "==2.5.5"
 django-extensions = "==3.2.3"
 
@@ -128,7 +129,7 @@ django-extensions = "==3.2.3"
 typesense = ["typesense"]
 
 [build-system]
-requires = ["poetry-core==1.8.1"]
+requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.pylint.master]
@@ -158,28 +159,29 @@ python_files = [
 testpaths = ["tests"]
 addopts = "-p no:warnings"
 env = [
-  "SECRET_KEY=test",
-  "EMAIL_CONFIG=consolemail://",
   "CELERY_BROKER_URL=memory://",
   "CELERY_TASK_ALWAYS_EAGER=True",
-  "FUNKWHALE_HOSTNAME_SUFFIX=",
-  "FUNKWHALE_HOSTNAME_PREFIX=",
-  "FUNKWHALE_HOSTNAME=test.federation",
-  "FEDERATION_HOSTNAME=test.federation",
-  "FUNKWHALE_URL=https://test.federation",
-  "DEBUG_TOOLBAR_ENABLED=False",
-  "DEBUG=False",
-  "WEAK_PASSWORDS=True",
   "CREATE_IMAGE_THUMBNAILS=False",
-  "FORCE_HTTPS_URLS=False",
+  "DEBUG=False",
-  "FUNKWHALE_SPA_HTML_ROOT=http://noop/",
+  "DEBUG_TOOLBAR_ENABLED=False",
-  "PROXY_MEDIA=true",
+  "DISABLE_PASSWORD_VALIDATORS=false",
-  "MUSIC_USE_DENORMALIZATION=true",
+  "DISABLE_PASSWORD_VALIDATORS=false",
+  "EMAIL_CONFIG=consolemail://",
   "EXTERNAL_MEDIA_PROXY_ENABLED=true",
-  "DISABLE_PASSWORD_VALIDATORS=false",
+  "FEDERATION_HOSTNAME=test.federation",
-  "DISABLE_PASSWORD_VALIDATORS=false",
+  "FORCE_HTTPS_URLS=False",
+  "FUNKWHALE_HOSTNAME=test.federation",
+  "FUNKWHALE_HOSTNAME_PREFIX=",
+  "FUNKWHALE_HOSTNAME_SUFFIX=",
   "FUNKWHALE_PLUGINS=",
+  "FUNKWHALE_SPA_HTML_ROOT=http://noop/",
+  "FUNKWHALE_URL=https://test.federation",
   "MUSIC_DIRECTORY_PATH=/music",
+  "MUSIC_USE_DENORMALIZATION=true",
+  "PROXY_MEDIA=true",
+  "SECRET_KEY=test",
+  "TYPESENSE_API_KEY=apikey",
+  "WEAK_PASSWORDS=True",
 ]
 
 [tool.coverage.run]

api/tests/music/test_utils.py

@@ -1,6 +1,5 @@
 import os
 import pathlib
-import tempfile
 
 import pytest
 
@@ -114,25 +113,6 @@ def test_get_dirs_and_files(path, expected, tmpdir):
     assert utils.browse_dir(root_path, path) == expected
 
 
-@pytest.mark.parametrize(
-    "name, expected",
-    [
-        ("sample.flac", {"bitrate": 128000, "length": 0}),
-        ("test.mp3", {"bitrate": 16000, "length": 268}),
-        ("test.ogg", {"bitrate": 128000, "length": 1}),
-        ("test.opus", {"bitrate": 128000, "length": 1}),
-    ],
-)
-def test_transcode_file(name, expected):
-    path = pathlib.Path(os.path.join(DATA_DIR, name))
-    with tempfile.NamedTemporaryFile() as dest:
-        utils.transcode_file(path, pathlib.Path(dest.name))
-        with open(dest.name, "rb") as f:
-            result = {k: round(v) for k, v in utils.get_audio_file_data(f).items()}
-
-            assert result == expected
-
-
 def test_custom_s3_domain(factories, settings):
     """See #2220"""
     settings.AWS_S3_CUSTOM_DOMAIN = "my.custom.domain.tld"

api/tests/music/test_views.py

@@ -620,8 +620,6 @@ def test_listen_transcode_in_place(
         source="file://" + os.path.join(DATA_DIR, "test.ogg"),
     )
 
-    assert upload.get_audio_segment()
-
     url = reverse("api:v1:listen-detail", kwargs={"uuid": upload.track.uuid})
     handle_serve = mocker.spy(views, "handle_serve")
     response = logged_in_api_client.get(url, {"to": "mp3"})

changes/changelog.d/2282.enhancement

@@ -0,0 +1 @@
+Drop python 3.8 and 3.9 (#2282)