MirrorMirror
/
funkwhale
mirror of https://dev.funkwhale.audio/funkwhale/funkwhale.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cherry-pick resolved

This commit is contained in:
petitminion 2025-01-13 21:47:18 +00:00 committed by Petitminion
parent 9f57111866
commit 5884f1467e
9 changed files with 2178 additions and 1746 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
.gitlab-ci.yml
View File

@ -231,7 +231,7 @@ test_api:
image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-api:$PYTHON_VERSION
parallel:
matrix:
- PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11"]
- PYTHON_VERSION: ["3.10", "3.11", "3.12", "3.13"]
services:
- name: postgres:15-alpine
command:
@ -508,3 +508,24 @@ docker:
name: docker_metadata_${CI_COMMIT_REF_NAME}
paths:
- metadata.json
package:
stage: publish
needs:
- job: build_metadata
artifacts: true
- job: build_api
artifacts: true
- job: build_front
artifacts: true
# - job: build_tauri
# artifacts: true
rules:
- if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
image: $CI_REGISTRY/funkwhale/ci/python:3.11
variables:
<<: *keep_git_files_permissions
script:
- make package
- scripts/ci-upload-packages.sh

120
api/Dockerfile.alpine Normal file
View File

@ -0,0 +1,120 @@
FROM alpine:3.21 AS requirements
RUN set -eux; \
apk add --no-cache \
poetry \
py3-cryptography \
py3-pip \
python3
COPY pyproject.toml poetry.lock /
RUN set -eux; \
poetry export --without-hashes --extras typesense > requirements.txt; \
poetry export --without-hashes --with dev > dev-requirements.txt;
FROM alpine:3.21 AS builder
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ARG PIP_NO_CACHE_DIR=1
ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
RUN set -eux; \
apk add --no-cache \
cargo \
curl \
gcc \
g++ \
git \
jpeg-dev \
libffi-dev \
libldap \
libxml2-dev \
libxslt-dev \
make \
musl-dev \
openldap-dev \
openssl-dev \
postgresql-dev \
zlib-dev \
py3-cryptography \
py3-lxml \
py3-pillow \
py3-psycopg2 \
py3-watchfiles \
python3-dev
# Create virtual env
RUN python3 -m venv --system-site-packages /venv
ENV PATH="/venv/bin:$PATH"
COPY --from=requirements /requirements.txt /requirements.txt
COPY --from=requirements /dev-requirements.txt /dev-requirements.txt
RUN --mount=type=cache,target=~/.cache/pip; \
set -eux; \
pip3 install --upgrade pip; \
pip3 install setuptools wheel; \
# Currently we are unable to relieably build rust-based packages on armv7. This
# is why we need to use the packages shipped by Alpine Linux.
# Since poetry does not allow in-place dependency pinning, we need
# to install the deps using pip.
grep -Ev 'cryptography|lxml|pillow|psycopg2|watchfiles' /requirements.txt \
| pip3 install -r /dev/stdin \
cryptography \
lxml \
pillow \
psycopg2 \
watchfiles
ARG install_dev_deps=0
RUN --mount=type=cache,target=~/.cache/pip; \
set -eux; \
if [ "$install_dev_deps" = "1" ] ; then \
grep -Ev 'cryptography|lxml|pillow|psycopg2|watchfiles' /dev-requirements.txt \
| pip3 install -r /dev/stdin \
cryptography \
lxml \
pillow \
psycopg2 \
watchfiles; \
fi
FROM alpine:3.21 AS production
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ARG PIP_NO_CACHE_DIR=1
RUN set -eux; \
apk add --no-cache \
bash \
ffmpeg \
gettext \
jpeg-dev \
libldap \
libmagic \
libpq \
libxml2 \
libxslt \
py3-cryptography \
py3-lxml \
py3-pillow \
py3-psycopg2 \
py3-watchfiles \
python3 \
tzdata
COPY --from=builder /venv /venv
ENV PATH="/venv/bin:$PATH"
COPY . /app
WORKDIR /app
RUN --mount=type=cache,target=~/.cache/pip; \
set -eux; \
pip3 install --no-deps --editable .
ENV IS_DOCKER_SETUP=true
CMD ["./docker/server.sh"]

57
api/funkwhale_api/music/models.py
View File

@ -6,7 +6,6 @@ import urllib.parse
import uuid
import arrow
import pydub
from django.conf import settings
from django.contrib.contenttypes.fields import GenericRelation
from django.contrib.postgres.indexes import GinIndex
@ -850,6 +849,12 @@ class Upload(models.Model):
if self.source and self.source.startswith("file://"):
return open(self.source.replace("file://", "", 1), "rb")
def get_audio_file_path(self):
if self.audio_file:
return self.audio_file.path
if self.source and self.source.startswith("file://"):
return self.source.replace("file://", "", 1)
def get_audio_data(self):
audio_file = self.get_audio_file()
if not audio_file:
@ -863,13 +868,47 @@ class Upload(models.Model):
"size": self.get_file_size(),
}
def get_audio_segment(self):
input = self.get_audio_file()
if not input:
return
def get_quality(self):
extension_to_mimetypes = utils.get_extension_to_mimetype_dict()
audio = pydub.AudioSegment.from_file(input)
return audio
if not self.bitrate and self.mimetype not in list(
itertools.chain(
extension_to_mimetypes["aiff"],
extension_to_mimetypes["aif"],
extension_to_mimetypes["flac"],
)
):
return 1
bitrate_limits = {
"mp3": {192: 0, 256: 1, 320: 2},
"ogg": {96: 0, 192: 1, 256: 2},
"aac": {96: 0, 128: 1, 288: 2},
"m4a": {96: 0, 128: 1, 288: 2},
"opus": {
96: 0,
128: 1,
160: 2,
},
}
for ext in bitrate_limits:
if self.mimetype in extension_to_mimetypes[ext]:
for limit, quality in sorted(bitrate_limits[ext].items()):
if int(self.bitrate) <= limit:
return quality
# opus higher tham 160
return 3
if self.mimetype in list(
itertools.chain(
extension_to_mimetypes["aiff"],
extension_to_mimetypes["aif"],
extension_to_mimetypes["flac"],
)
):
return 3
def save(self, **kwargs):
if not self.mimetype:
@ -951,8 +990,8 @@ class Upload(models.Model):
)
version.audio_file.save(new_name, f)
utils.transcode_audio(
audio=self.get_audio_segment(),
output=version.audio_file,
audio_file_path=self.get_audio_file_path(),
output_path=version.audio_file.path,
output_format=utils.MIMETYPE_TO_EXTENSION[mimetype],
bitrate=str(bitrate),
)

15
api/funkwhale_api/music/utils.py
View File

@ -4,10 +4,10 @@ import pathlib
import magic
import mutagen
import pydub
from django.conf import settings
from django.core.cache import cache
from django.db.models import F
from ffmpeg import FFmpeg
from funkwhale_api.common import throttling
from funkwhale_api.common.search import get_fts_query # noqa
@ -102,15 +102,10 @@ def get_actor_from_request(request):
return actor
def transcode_file(input, output, input_format=None, output_format="mp3", **kwargs):
with input.open("rb"):
audio = pydub.AudioSegment.from_file(input, format=input_format)
return transcode_audio(audio, output, output_format, **kwargs)
def transcode_audio(audio, output, output_format, **kwargs):
with output.open("wb"):
return audio.export(output, format=output_format, **kwargs)
def transcode_audio(audio_file_path, output_path, output_format="mp3", **kwargs):
FFmpeg().input(audio_file_path).output(
output_path, format=output_format, **kwargs
).option("y").execute()
def increment_downloads_count(upload, user, wsgi_request):

3580
api/poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

106
api/pyproject.toml
View File

@ -25,29 +25,29 @@ exclude = ["tests"]
funkwhale-manage = 'funkwhale_api.main:main'
[tool.poetry.dependencies]
python = "^3.8,<3.12"
python = "^3.10,<3.13"
# Django
dj-rest-auth = "5.0.2"
django = "4.2.9"
django-allauth = "0.55.2"
django-cache-memoize = "0.1.10"
django-cacheops = "==6.1"
django-cleanup = "==6.0.0"
django-cors-headers = "==3.13.0"
django-cacheops = "==7.0.2"
django-cleanup = "==8.1.0"
django-cors-headers = "==4.3.1"
django-dynamic-preferences = "==1.14.0"
django-environ = "==0.10.0"
django-filter = "==22.1"
django-environ = "==0.11.2"
django-filter = "==23.5"
django-oauth-toolkit = "2.2.0"
django-redis = "==5.2.0"
django-storages = "==1.13.2"
django-versatileimagefield = "==2.2"
django-versatileimagefield = "==3.1"
djangorestframework = "==3.14.0"
drf-spectacular = "==0.26.5"
markdown = "==3.4.4"
persisting-theory = "==1.0"
psycopg2 = "==2.9.9"
redis = "==4.5.5"
psycopg2-binary = "==2.9.9"
redis = "==5.0.1"
# Django LDAP
django-auth-ldap = "==4.1.0"
@ -58,69 +58,70 @@ channels = { extras = ["daphne"], version = "==4.0.0" }
channels-redis = "==4.1.0"
# Celery
kombu = "==5.2.4"
celery = "==5.2.7"
kombu = "5.3.4"
celery = "5.3.6"
# Deployment
gunicorn = "==20.1.0"
gunicorn = "==21.2.0"
uvicorn = { version = "==0.20.0", extras = ["standard"] }
# Libs
aiohttp = "==3.8.6"
aiohttp = "3.9.1"
arrow = "==1.2.3"
backports-zoneinfo = { version = "==0.2.1", python = "<3.9" }
bleach = "==5.0.1"
bleach = "==6.1.0"
boto3 = "==1.26.161"
click = "==8.1.7"
cryptography = "==38.0.4"
feedparser = "==6.0.11"
cryptography = "==41.0.7"
feedparser = "==6.0.10"
liblistenbrainz = "==0.5.5"
musicbrainzngs = "==0.7.1"
mutagen = "==1.46.0"
pillow = "==9.3.0"
pillow = "==10.2.0"
pydub = "==0.25.1"
pyld = "==2.0.4"
pyld = "==2.0.3"
python-magic = "==0.4.27"
requests = "==2.28.2"
requests = "==2.31.0"
requests-http-message-signatures = "==0.3.1"
sentry-sdk = "==1.19.1"
watchdog = "==2.2.1"
troi = { git = "https://github.com/metabrainz/troi-recommendation-playground.git", tag = "v-2023-10-30.0"}
lb-matching-tools = { git = "https://github.com/metabrainz/listenbrainz-matching-tools.git", branch = "main"}
unidecode = "==1.3.8"
pycountry = "22.3.5"
watchdog = "==4.0.0"
troi = "==2024.1.26.0"
lb-matching-tools = "==2024.1.25.0rc1"
unidecode = "==1.3.7"
pycountry = "23.12.11"
# Typesense
typesense = { version = "==0.15.1", optional = true }
# Dependencies pinning
ipython = "==7.34.0"
ipython = "==8.12.3"
pluralizer = "==1.2.0"
service-identity = "==21.1.0"
service-identity = "==24.1.0"
unicode-slugify = "==0.1.5"
[tool.poetry.group.dev.dependencies]
aioresponses = "==0.7.6"
asynctest = "==0.13.0"
black = "==23.3.0"
coverage = { version = "==6.5.0", extras = ["toml"] }
black = "==24.1.1"
coverage = { version = "==7.4.1", extras = ["toml"] }
debugpy = "==1.6.7.post1"
django-coverage-plugin = "==3.0.0"
django-debug-toolbar = "==3.8.1"
django-debug-toolbar = "==4.2.0"
factory-boy = "==3.2.1"
faker = "==15.3.4"
faker = "==23.2.1"
flake8 = "==3.9.2"
ipdb = "==0.13.13"
pytest = "==7.4.4"
pytest = "==8.0.0"
pytest-asyncio = "==0.21.0"
prompt-toolkit = "==3.0.43"
prompt-toolkit = "==3.0.41"
pytest-cov = "==4.0.0"
pytest-django = "==4.5.2"
pytest-env = "==0.8.2"
pytest-env = "==1.1.3"
pytest-mock = "==3.10.0"
pytest-randomly = "==3.12.0"
pytest-sugar = "==0.9.7"
pytest-sugar = "==1.0.0"
requests-mock = "==1.10.0"
pylint = "==2.17.7"
pylint = "==3.0.3"
pylint-django = "==2.5.5"
django-extensions = "==3.2.3"
@ -128,7 +129,7 @@ django-extensions = "==3.2.3"
typesense = ["typesense"]
[build-system]
requires = ["poetry-core==1.8.1"]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.pylint.master]
@ -158,28 +159,29 @@ python_files = [
testpaths = ["tests"]
addopts = "-p no:warnings"
env = [
"SECRET_KEY=test",
"EMAIL_CONFIG=consolemail://",
"CELERY_BROKER_URL=memory://",
"CELERY_TASK_ALWAYS_EAGER=True",
"FUNKWHALE_HOSTNAME_SUFFIX=",
"FUNKWHALE_HOSTNAME_PREFIX=",
"FUNKWHALE_HOSTNAME=test.federation",
"FEDERATION_HOSTNAME=test.federation",
"FUNKWHALE_URL=https://test.federation",
"DEBUG_TOOLBAR_ENABLED=False",
"DEBUG=False",
"WEAK_PASSWORDS=True",
"CREATE_IMAGE_THUMBNAILS=False",
"FORCE_HTTPS_URLS=False",
"FUNKWHALE_SPA_HTML_ROOT=http://noop/",
"PROXY_MEDIA=true",
"MUSIC_USE_DENORMALIZATION=true",
"DEBUG=False",
"DEBUG_TOOLBAR_ENABLED=False",
"DISABLE_PASSWORD_VALIDATORS=false",
"DISABLE_PASSWORD_VALIDATORS=false",
"EMAIL_CONFIG=consolemail://",
"EXTERNAL_MEDIA_PROXY_ENABLED=true",
"DISABLE_PASSWORD_VALIDATORS=false",
"DISABLE_PASSWORD_VALIDATORS=false",
"FEDERATION_HOSTNAME=test.federation",
"FORCE_HTTPS_URLS=False",
"FUNKWHALE_HOSTNAME=test.federation",
"FUNKWHALE_HOSTNAME_PREFIX=",
"FUNKWHALE_HOSTNAME_SUFFIX=",
"FUNKWHALE_PLUGINS=",
"FUNKWHALE_SPA_HTML_ROOT=http://noop/",
"FUNKWHALE_URL=https://test.federation",
"MUSIC_DIRECTORY_PATH=/music",
"MUSIC_USE_DENORMALIZATION=true",
"PROXY_MEDIA=true",
"SECRET_KEY=test",
"TYPESENSE_API_KEY=apikey",
"WEAK_PASSWORDS=True",
]
[tool.coverage.run]

20
api/tests/music/test_utils.py
View File

@ -1,6 +1,5 @@
import os
import pathlib
import tempfile
import pytest
@ -114,25 +113,6 @@ def test_get_dirs_and_files(path, expected, tmpdir):
assert utils.browse_dir(root_path, path) == expected
@pytest.mark.parametrize(
"name, expected",
[
("sample.flac", {"bitrate": 128000, "length": 0}),
("test.mp3", {"bitrate": 16000, "length": 268}),
("test.ogg", {"bitrate": 128000, "length": 1}),
("test.opus", {"bitrate": 128000, "length": 1}),
],
)
def test_transcode_file(name, expected):
path = pathlib.Path(os.path.join(DATA_DIR, name))
with tempfile.NamedTemporaryFile() as dest:
utils.transcode_file(path, pathlib.Path(dest.name))
with open(dest.name, "rb") as f:
result = {k: round(v) for k, v in utils.get_audio_file_data(f).items()}
assert result == expected
def test_custom_s3_domain(factories, settings):
"""See #2220"""
settings.AWS_S3_CUSTOM_DOMAIN = "my.custom.domain.tld"

2
api/tests/music/test_views.py
View File

@ -620,8 +620,6 @@ def test_listen_transcode_in_place(
source="file://" + os.path.join(DATA_DIR, "test.ogg"),
)
assert upload.get_audio_segment()
url = reverse("api:v1:listen-detail", kwargs={"uuid": upload.track.uuid})
handle_serve = mocker.spy(views, "handle_serve")
response = logged_in_api_client.get(url, {"to": "mp3"})

1
changes/changelog.d/2282.enhancement Normal file
View File

@ -0,0 +1 @@
Drop python 3.8 and 3.9 (#2282)