From 78ae09e165be43e52b5ffb90007ea77007730bdc Mon Sep 17 00:00:00 2001 From: jon r Date: Fri, 21 Feb 2025 12:37:54 +0100 Subject: [PATCH 01/28] chore(compose): refactor django environment next to its manifest --- compose.yml | 33 --------------------------------- compose/app.django.yml | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/compose.yml b/compose.yml index 55560113a..1402abccf 100644 --- a/compose.yml +++ b/compose.yml @@ -4,39 +4,6 @@ networks: external: true x-django: &django - environment: - - DEBUG - - DEFAULT_FROM_EMAIL - - - DJANGO_SETTINGS_MODULE - - DJANGO_SECRET_KEY - - - EXTERNAL_REQUESTS_VERIFY_SSL - - - "FORCE_HTTPS_URLS=${FORCE_HTTPS_URLS:-False}" - - FUNKWHALE_PROTOCOL - - "FUNKWHALE_HOSTNAME=${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}" - - - DATABASE_URL - - CACHE_URL - - EMAIL_CONFIG - - TYPESENSE_API_KEY - - - "STATIC_URL=${FUNKWHALE_PROTOCOL}://${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}/static/" - - "MEDIA_URL=${FUNKWHALE_PROTOCOL}://${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}/media/" - - - STATIC_ROOT - - MEDIA_ROOT - - - FUNKWHALE_SPA_HTML_ROOT - - LDAP_ENABLED - - BROWSABLE_API_ENABLED - - "MUSIC_DIRECTORY_PATH=${MUSIC_DIRECTORY_PATH:-/music}" - - - C_FORCE_ROOT - - PYTHONDONTWRITEBYTECODE - - PYTHONTRACEMALLOC - dns: 172.17.0.1 dns_search: funkwhale.test diff --git a/compose/app.django.yml b/compose/app.django.yml index 2b2b80f0e..8757be4db 100644 --- a/compose/app.django.yml +++ b/compose/app.django.yml @@ -1,5 +1,7 @@ x-django: &django image: funkwhale-api + networks: + - internal volumes: - ../api:/app - ../.env:/app/.env @@ -13,8 +15,38 @@ x-django: &django condition: service_healthy redis: condition: service_healthy - networks: - - internal + environment: + - DEBUG + - DEFAULT_FROM_EMAIL + + - DJANGO_SETTINGS_MODULE + - DJANGO_SECRET_KEY + + - EXTERNAL_REQUESTS_VERIFY_SSL + + - "FORCE_HTTPS_URLS=${FORCE_HTTPS_URLS:-False}" + - FUNKWHALE_PROTOCOL + - "FUNKWHALE_HOSTNAME=${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}" + + - DATABASE_URL + - CACHE_URL + - EMAIL_CONFIG + - TYPESENSE_API_KEY + + - "STATIC_URL=${FUNKWHALE_PROTOCOL}://${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}/static/" + - "MEDIA_URL=${FUNKWHALE_PROTOCOL}://${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}/media/" + + - STATIC_ROOT + - MEDIA_ROOT + + - FUNKWHALE_SPA_HTML_ROOT + - LDAP_ENABLED + - BROWSABLE_API_ENABLED + - "MUSIC_DIRECTORY_PATH=${MUSIC_DIRECTORY_PATH:-/music}" + + - C_FORCE_ROOT + - PYTHONDONTWRITEBYTECODE + - PYTHONTRACEMALLOC services: api: From 1be743c5e0473deae5dc27b9a0bc7917e68fbc92 Mon Sep 17 00:00:00 2001 From: jon r Date: Mon, 24 Feb 2025 19:37:38 +0100 Subject: [PATCH 02/28] chore(changelog): add fragment for #2406 --- changes/changelog.d/2406.chore | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/changelog.d/2406.chore diff --git a/changes/changelog.d/2406.chore b/changes/changelog.d/2406.chore new file mode 100644 index 000000000..55f997bf2 --- /dev/null +++ b/changes/changelog.d/2406.chore @@ -0,0 +1 @@ +Refactor Django Compose environment (#2406) From 0b4e7dac41c84ac8701cbd6991a26f2de98dc19c Mon Sep 17 00:00:00 2001 From: jon r Date: Tue, 25 Feb 2025 01:02:20 +0100 Subject: [PATCH 03/28] feat: add .editorconfig --- .editorconfig | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..383809536 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = false +insert_final_newline = true From 0c3d732791f083233a0018ca8e832011a1ae7d1e Mon Sep 17 00:00:00 2001 From: jon r Date: Tue, 25 Feb 2025 01:04:43 +0100 Subject: [PATCH 04/28] chore(nginx): consistent use of _HOST suffix --- .env.example | 2 +- .gitpod/docker-compose.yml | 2 +- compose/etc/nginx/conf.dev | 2 +- templates/nginx.conf.j2 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index e8a58128f..cc083684e 100644 --- a/.env.example +++ b/.env.example @@ -50,7 +50,7 @@ NGINX_MAX_BODY_SIZE=10G FUNKWHALE_API_HOST=api FUNKWHALE_API_PORT=5000 -FUNKWHALE_FRONT_IP=front +FUNKWHALE_FRONT_HOST=front FUNKWHALE_FRONT_PORT=${VUE_PORT} # postgres diff --git a/.gitpod/docker-compose.yml b/.gitpod/docker-compose.yml index 62b5f5930..271786a1b 100644 --- a/.gitpod/docker-compose.yml +++ b/.gitpod/docker-compose.yml @@ -30,7 +30,7 @@ services: - "FUNKWHALE_API_IP=host.docker.internal" - "FUNKWHALE_API_HOST=host.docker.internal" - "FUNKWHALE_API_PORT=5000" - - "FUNKWHALE_FRONT_IP=host.docker.internal" + - "FUNKWHALE_FRONT_HOST=host.docker.internal" - "FUNKWHALE_FRONT_PORT=8080" - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-host.docker.internal}" - "FUNKWHALE_PROTOCOL=https" diff --git a/compose/etc/nginx/conf.dev b/compose/etc/nginx/conf.dev index 40af155ae..e867e2277 100644 --- a/compose/etc/nginx/conf.dev +++ b/compose/etc/nginx/conf.dev @@ -4,7 +4,7 @@ upstream funkwhale-api { } upstream funkwhale-front { - server ${FUNKWHALE_FRONT_IP}:${FUNKWHALE_FRONT_PORT}; + server ${FUNKWHALE_FRONT_HOST}:${FUNKWHALE_FRONT_PORT}; } # Required for websocket support. diff --git a/templates/nginx.conf.j2 b/templates/nginx.conf.j2 index 503d7abcb..8af7dbc4b 100644 --- a/templates/nginx.conf.j2 +++ b/templates/nginx.conf.j2 @@ -19,7 +19,7 @@ upstream funkwhale-api { {% if config.proxy_frontend %} upstream funkwhale-front { - server ${FUNKWHALE_FRONT_IP}:${FUNKWHALE_FRONT_PORT}; + server ${FUNKWHALE_FRONT_HOST}:${FUNKWHALE_FRONT_PORT}; } {% endif %} From afceb5bf3be036202d2840f97d7474114fe093d7 Mon Sep 17 00:00:00 2001 From: jon r Date: Tue, 25 Feb 2025 01:09:10 +0100 Subject: [PATCH 05/28] chore(compose): dependencies, environments and healthchecks --- compose.yml | 45 ++++++++++++------------------------------ compose/app.django.yml | 12 +++-------- compose/app.nginx.yml | 28 +++++++++++++++++++++----- compose/app.vue.yml | 17 ++++++++++++++++ front/Dockerfile.dev | 3 +++ 5 files changed, 59 insertions(+), 46 deletions(-) create mode 100644 compose/app.vue.yml diff --git a/compose.yml b/compose.yml index 1402abccf..2b5e2f9ff 100644 --- a/compose.yml +++ b/compose.yml @@ -4,26 +4,19 @@ networks: external: true x-django: &django + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy dns: 172.17.0.1 dns_search: funkwhale.test services: front: - build: - context: ./front - dockerfile: Dockerfile.dev - ports: - - "${VUE_PORT:-8080}:${VUE_PORT:-8080}" - environment: - - HOST - - VUE_PORT - volumes: - - "./front:/app" - - "/app/node_modules" - - "./po:/po" - networks: - - internal - command: "yarn dev --host" + extends: + file: ./compose/app.vue.yml + service: app api: extends: @@ -41,26 +34,14 @@ services: extends: file: ./compose/app.nginx.yml service: nginx - environment: - - "MUSIC_DIRECTORY_PATH=${MUSIC_DIRECTORY_PATH:-/music}" - - "FUNKWHALE_HOSTNAME=${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}" - - - FUNKWHALE_PROTOCOL - - - FUNKWHALE_API_HOST - - FUNKWHALE_API_PORT - - - FUNKWHALE_FRONT_IP - - FUNKWHALE_FRONT_PORT - - - NGINX_MAX_BODY_SIZE - - - STATIC_ROOT - - "MEDIA_ROOT=${MEDIA_ROOT:-/data/media}" + depends_on: + front: + condition: service_healthy + api: + condition: service_healthy networks: - web - internal - labels: - "traefik.enable=true" diff --git a/compose/app.django.yml b/compose/app.django.yml index 8757be4db..74423d1dc 100644 --- a/compose/app.django.yml +++ b/compose/app.django.yml @@ -55,14 +55,11 @@ services: context: ../api dockerfile: Dockerfile.debian healthcheck: - test: - [ - "CMD-SHELL", - 'docker compose logs api | grep -q "Uvicorn running on" || exit 0', - ] - interval: 3s + test: 'curl -o /dev/null -s -w "%{http_code}" http://localhost:5000/api/v1 | grep "301" || exit 1' + interval: 10s timeout: 5s retries: 3 + start_period: 60s command: > sh -c " funkwhale-manage collectstatic --no-input && @@ -76,6 +73,3 @@ services: pip install watchdog[watchmedo] && watchmedo auto-restart --patterns="*.py" --recursive -- celery -A funkwhale_api.taskapp worker -l debug -B --concurrency=${CELERYD_CONCURRENCY} ' - depends_on: - api: - condition: service_healthy diff --git a/compose/app.nginx.yml b/compose/app.nginx.yml index 86cca2bf0..6000b3491 100644 --- a/compose/app.nginx.yml +++ b/compose/app.nginx.yml @@ -1,9 +1,24 @@ services: nginx: image: nginx - depends_on: - - api - - front + networks: + - internal + environment: + - "MUSIC_DIRECTORY_PATH=${MUSIC_DIRECTORY_PATH:-/music}" + - "FUNKWHALE_HOSTNAME=${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}" + + - FUNKWHALE_PROTOCOL + + - FUNKWHALE_API_HOST + - FUNKWHALE_API_PORT + + - FUNKWHALE_FRONT_HOST + - FUNKWHALE_FRONT_PORT + + - NGINX_MAX_BODY_SIZE + + - STATIC_ROOT + - "MEDIA_ROOT=${MEDIA_ROOT:-/data/media}" volumes: - "${MUSIC_DIRECTORY_SERVE_PATH:-../.state/music}:${MUSIC_DIRECTORY_PATH:-/music}:ro" @@ -14,5 +29,8 @@ services: - ../.state/staticfiles:/usr/share/nginx/html/staticfiles:ro - ../.state/media:/protected/media:ro - ../.state/${COMPOSE_PROJECT_NAME:-funkwhale}/media:/data/media:ro - networks: - - internal + healthcheck: + test: 'curl -o /dev/null -s -w "%{http_code}" http://localhost:80/ | grep "200" || exit 1' + interval: 5s + timeout: 3s + retries: 3 diff --git a/compose/app.vue.yml b/compose/app.vue.yml new file mode 100644 index 000000000..758a0cb0c --- /dev/null +++ b/compose/app.vue.yml @@ -0,0 +1,17 @@ +services: + app: + image: funkwhale-app + build: + context: ../front + dockerfile: Dockerfile.dev + command: "yarn dev --host" + ports: + - "${VUE_PORT:-8080}:${VUE_PORT:-8080}" + networks: + - internal + volumes: + - "../front:/app" + - "/app/node_modules" + environment: + - HOST + - VUE_PORT diff --git a/front/Dockerfile.dev b/front/Dockerfile.dev index f280b65ae..3605f0691 100644 --- a/front/Dockerfile.dev +++ b/front/Dockerfile.dev @@ -12,3 +12,6 @@ RUN yarn install COPY . . CMD ["yarn", "serve"] + +HEALTHCHECK --start-period=30s --interval=10s --timeout=5s \ + CMD wget --no-verbose --tries=1 --spider http://localhost:8080/ || exit 1 From 9f7f6ba8bfeb09251f72038f3031b692045e36c1 Mon Sep 17 00:00:00 2001 From: jon r Date: Tue, 25 Feb 2025 01:12:19 +0100 Subject: [PATCH 06/28] fix(settings/local): do not force by default, but specify explicitly Adds tini as PID 1 to Django containers for them to react to signals. (init: true) --- .env.example | 1 + api/config/settings/local.py | 2 +- compose/app.django.yml | 8 ++------ 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index cc083684e..51977f5db 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ # api + celeryworker DEBUG=True +FORCE=True DEFAULT_FROM_EMAIL=hello@funkwhale.test FUNKWHALE_DOMAIN=funkwhale.test diff --git a/api/config/settings/local.py b/api/config/settings/local.py index b1674f46d..5e85ff5cc 100644 --- a/api/config/settings/local.py +++ b/api/config/settings/local.py @@ -154,4 +154,4 @@ REST_FRAMEWORK.update( ) # allows makemigrations and superuser creation -FORCE = env("FORCE", default=1) +FORCE = env("FORCE", default=0) diff --git a/compose/app.django.yml b/compose/app.django.yml index 74423d1dc..7c1ec7a7a 100644 --- a/compose/app.django.yml +++ b/compose/app.django.yml @@ -1,22 +1,18 @@ x-django: &django image: funkwhale-api + init: true networks: - internal volumes: - ../api:/app - - ../.env:/app/.env - "${MUSIC_DIRECTORY_SERVE_PATH:-../.state/music}:/music:ro" - "../.state/plugins:/srv/funkwhale/plugins" - "../.state/staticfiles:/staticfiles" - "../.state/media:/protected/media" - "../.state/${COMPOSE_PROJECT_NAME:-funkwhale}/media:/data/media" - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy environment: - DEBUG + - FORCE - DEFAULT_FROM_EMAIL - DJANGO_SETTINGS_MODULE From 354bb24dc38560a1764ac851e11365c29a578dab Mon Sep 17 00:00:00 2001 From: jon r Date: Tue, 25 Feb 2025 22:25:17 +0100 Subject: [PATCH 07/28] feat(editorconfig): trim trailing whitespace --- .editorconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 383809536..c6c8b3621 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,5 +5,5 @@ indent_style = space indent_size = 2 end_of_line = lf charset = utf-8 -trim_trailing_whitespace = false +trim_trailing_whitespace = true insert_final_newline = true From 0bc581044a49f3424fc61a2e990aff0cacb7c23c Mon Sep 17 00:00:00 2001 From: jon r Date: Tue, 25 Feb 2025 22:37:45 +0100 Subject: [PATCH 08/28] chore(api/settings): move FORCE from local to common --- api/config/settings/common.py | 2 ++ api/config/settings/local.py | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 776d73c2e..e3765ed67 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -38,6 +38,8 @@ Available levels: IS_DOCKER_SETUP = env.bool("IS_DOCKER_SETUP", False) +# allows makemigrations and superuser creation +FORCE = env.bool("FORCE", False) if env("FUNKWHALE_SENTRY_DSN", default=None) is not None: import sentry_sdk diff --git a/api/config/settings/local.py b/api/config/settings/local.py index 5e85ff5cc..70fb5a21e 100644 --- a/api/config/settings/local.py +++ b/api/config/settings/local.py @@ -152,6 +152,3 @@ REST_FRAMEWORK.update( ], } ) - -# allows makemigrations and superuser creation -FORCE = env("FORCE", default=0) From 034a7b75cd4148bb613bc1cfe38e1b63a2365eb3 Mon Sep 17 00:00:00 2001 From: jon r Date: Wed, 26 Feb 2025 00:26:19 +0100 Subject: [PATCH 09/28] chore(api/settings): remove depreciated IS_DOCKER_SETUP variable This configuration was for a very old Docker version that still supported linking containers together, which allowed for service discovery via the environment. This is not present in Docker anymore for many years now, so removing. --- api/config/settings/common.py | 44 ++++------------------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index e3765ed67..634e74235 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -1,6 +1,5 @@ import logging.config import sys -import warnings from collections import OrderedDict from urllib.parse import urlparse, urlsplit @@ -36,8 +35,6 @@ Available levels: """ -IS_DOCKER_SETUP = env.bool("IS_DOCKER_SETUP", False) - # allows makemigrations and superuser creation FORCE = env.bool("FORCE", False) @@ -395,31 +392,6 @@ vars().update(EMAIL_CONFIG) # ------------------------------------------------------------------------------ # See: https://docs.djangoproject.com/en/dev/ref/settings/#databases -# The `_database_url_docker` variable will only by used as default for DATABASE_URL -# in the context of a docker deployment. -_database_url_docker = None -if IS_DOCKER_SETUP and env.str("DATABASE_URL", None) is None: - warnings.warn( - DeprecationWarning( - "the automatically generated 'DATABASE_URL' configuration in the docker " - "setup is deprecated, please configure either the 'DATABASE_URL' " - "environment variable or the 'DATABASE_HOST', 'DATABASE_USER' and " - "'DATABASE_PASSWORD' environment variables instead" - ) - ) - _DOCKER_DATABASE_HOST = "postgres" - _DOCKER_DATABASE_PORT = 5432 - _DOCKER_DATABASE_USER = env.str("POSTGRES_ENV_POSTGRES_USER", "postgres") - _DOCKER_DATABASE_PASSWORD = env.str("POSTGRES_ENV_POSTGRES_PASSWORD", "") - _DOCKER_DATABASE_NAME = _DOCKER_DATABASE_USER - - _database_url_docker = ( - f"postgres:" - f"//{_DOCKER_DATABASE_USER}:{_DOCKER_DATABASE_PASSWORD}" - f"@{_DOCKER_DATABASE_HOST}:{_DOCKER_DATABASE_PORT}" - f"/{_DOCKER_DATABASE_NAME}" - ) - DATABASE_HOST = env.str("DATABASE_HOST", "localhost") """ The hostname of the PostgreSQL server. Defaults to ``localhost``. @@ -442,8 +414,7 @@ The name of the PostgreSQL database. Defaults to ``funkwhale``. """ DATABASE_URL = env.db( "DATABASE_URL", - _database_url_docker # This is only set in the context of a docker deployment. - or ( + ( f"postgres:" f"//{DATABASE_USER}:{DATABASE_PASSWORD}" f"@{DATABASE_HOST}:{DATABASE_PORT}" @@ -840,11 +811,7 @@ if AUTH_LDAP_ENABLED: # SLUGLIFIER AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify" -CACHE_URL_DEFAULT = "redis://127.0.0.1:6379/0" -if IS_DOCKER_SETUP: - CACHE_URL_DEFAULT = "redis://redis:6379/0" - -CACHE_URL = env.str("CACHE_URL", default=CACHE_URL_DEFAULT) +CACHE_URL = env.str("CACHE_URL", default="redis://127.0.0.1:6379/0") """ The URL of your redis server. For example: @@ -1529,10 +1496,7 @@ TYPESENSE_PROTOCOL = env("TYPESENSE_PROTOCOL", default="http") """Typesense listening protocol""" TYPESENSE_HOST = env( "TYPESENSE_HOST", - default="typesense" if IS_DOCKER_SETUP else "localhost", + default="localhost", ) -""" -Typesense hostname. Defaults to `localhost` on non-Docker deployments and to `typesense` on -Docker deployments. -""" +"""Typesense hostname. Defaults to `localhost`.""" TYPESENSE_NUM_TYPO = env("TYPESENSE_NUM_TYPO", default=5) From 99f3f36ed587e6bc20bb717226631d1a589ac0b0 Mon Sep 17 00:00:00 2001 From: jon r Date: Wed, 26 Feb 2025 00:28:43 +0100 Subject: [PATCH 10/28] feat(compose/api): move healthcheck to Dockerfile + update entrypoint.sh --- .env.example | 4 ---- api/.dockerignore | 2 +- api/Dockerfile.alpine | 7 ++++-- api/Dockerfile.debian | 7 +++++- api/docker/server.sh | 13 ----------- api/entrypoint.sh | 49 ++++++++++++++++++++++++++++++++++++++++++ compose/app.django.yml | 18 ++-------------- front/Dockerfile.dev | 2 +- 8 files changed, 64 insertions(+), 38 deletions(-) delete mode 100755 api/docker/server.sh create mode 100755 api/entrypoint.sh diff --git a/.env.example b/.env.example index 51977f5db..4a247e3d2 100644 --- a/.env.example +++ b/.env.example @@ -27,10 +27,6 @@ FUNKWHALE_SPA_HTML_ROOT=http://nginx/ LDAP_ENABLED=False BROWSABLE_API_ENABLED=True -# celeryworker - -CELERYD_CONCURRENCY=0 - # api + nginx STATIC_ROOT=/staticfiles diff --git a/api/.dockerignore b/api/.dockerignore index 729de9825..dc2e56f7a 100644 --- a/api/.dockerignore +++ b/api/.dockerignore @@ -1,8 +1,8 @@ # Exclude everything and allow only the necessary files * -!/docker/ !/config/ !/funkwhale_api/ +!/entrypoint.sh !/manage.py !/poetry.lock !/pyproject.toml diff --git a/api/Dockerfile.alpine b/api/Dockerfile.alpine index 6a5599b08..7b08b651d 100644 --- a/api/Dockerfile.alpine +++ b/api/Dockerfile.alpine @@ -103,6 +103,7 @@ ARG PIP_NO_CACHE_DIR=1 RUN set -eux; \ apk add --no-cache \ bash \ + curl \ ffmpeg \ gettext \ jpeg-dev \ @@ -132,6 +133,8 @@ RUN --mount=type=cache,target=~/.cache/pip; \ set -eux; \ pip3 install --no-deps --editable . -ENV IS_DOCKER_SETUP=true +ADD --chown=0:0 --chmod=u+x ./entrypoint.sh / +ENTRYPOINT [ "/entrypoint.sh" ] -CMD ["./docker/server.sh"] +HEALTHCHECK --start-period=60s --interval=10s --timeout=5s --retries=3 \ + CMD curl -o /dev/null -s -w "%{http_code}" http://localhost:5000/api/v1 | grep "301" || exit 1 diff --git a/api/Dockerfile.debian b/api/Dockerfile.debian index 8e4de64bb..4b3a561c2 100644 --- a/api/Dockerfile.debian +++ b/api/Dockerfile.debian @@ -51,6 +51,7 @@ ENV PATH="/venv/bin:$PATH" RUN --mount=type=cache,target=/var/lib/apt/lists \ apt update; \ apt install -y \ + curl \ ffmpeg \ gettext \ libjpeg-dev \ @@ -68,4 +69,8 @@ WORKDIR /app COPY . /app RUN poetry install --extras typesense -CMD ["./docker/server.sh"] +ADD --chown=0:0 --chmod=u+x ./entrypoint.sh / +ENTRYPOINT [ "/entrypoint.sh" ] + +HEALTHCHECK --start-period=60s --interval=10s --timeout=5s --retries=3 \ + CMD curl -o /dev/null -s -w "%{http_code}" http://localhost:5000/api/v1 | grep "301" || exit 1 diff --git a/api/docker/server.sh b/api/docker/server.sh deleted file mode 100755 index e4aebcb98..000000000 --- a/api/docker/server.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -set -eux - -funkwhale-manage collectstatic --noinput -funkwhale-manage migrate - -# shellcheck disable=SC2086 -exec gunicorn config.asgi:application \ - --workers "${FUNKWHALE_WEB_WORKERS-1}" \ - --worker-class uvicorn.workers.UvicornWorker \ - --bind 0.0.0.0:"${FUNKWHALE_API_PORT}" \ - ${GUNICORN_ARGS-} diff --git a/api/entrypoint.sh b/api/entrypoint.sh new file mode 100755 index 000000000..830ea9594 --- /dev/null +++ b/api/entrypoint.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env sh + +set -eux + +case "${1}" in + gunicorn) + # shellcheck disable=SC2086 + exec gunicorn config.asgi:application \ + --workers "${FUNKWHALE_WEB_WORKERS:-1}" \ + --worker-class uvicorn.workers.UvicornWorker \ + --bind 0.0.0.0:"${FUNKWHALE_API_PORT}" \ + ${GUNICORN_ARGS:-} + ;; + migrate) + funkwhale-manage migrate + ;; + collectstatic) + funkwhale-manage collectstatic --noinput + ;; + uvicorn) + exec uvicorn \ + --reload config.asgi:application \ + --host 0.0.0.0 \ + --port 5000 \ + --reload-dir config/ \ + --reload-dir funkwhale_api/ + ;; + develop) + ${0} migrate + ${0} collectstatic + ${0} uvicorn + ;; + develop-celery) + export CELERYD_CONCURRENCY=0 + watchmedo auto-restart \ + --patterns="*.py" \ + --recursive \ + -- \ + celery \ + -A funkwhale_api.taskapp worker \ + -B \ + -l debug \ + -s /tmp/celerybeat-schedule \ + --concurrency=${CELERYD_CONCURRENCY} + ;; + *) + exec "${@}" + ;; +esac diff --git a/compose/app.django.yml b/compose/app.django.yml index 7c1ec7a7a..3452c1dfb 100644 --- a/compose/app.django.yml +++ b/compose/app.django.yml @@ -50,22 +50,8 @@ services: build: context: ../api dockerfile: Dockerfile.debian - healthcheck: - test: 'curl -o /dev/null -s -w "%{http_code}" http://localhost:5000/api/v1 | grep "301" || exit 1' - interval: 10s - timeout: 5s - retries: 3 - start_period: 60s - command: > - sh -c " - funkwhale-manage collectstatic --no-input && - uvicorn --reload config.asgi:application --host 0.0.0.0 --port 5000 --reload-dir config/ --reload-dir funkwhale_api/ - " + command: develop celeryworker: <<: *django - command: > - sh -c ' - pip install watchdog[watchmedo] && - watchmedo auto-restart --patterns="*.py" --recursive -- celery -A funkwhale_api.taskapp worker -l debug -B --concurrency=${CELERYD_CONCURRENCY} - ' + command: develop-celery diff --git a/front/Dockerfile.dev b/front/Dockerfile.dev index 3605f0691..bd9d14299 100644 --- a/front/Dockerfile.dev +++ b/front/Dockerfile.dev @@ -11,7 +11,7 @@ RUN yarn install COPY . . -CMD ["yarn", "serve"] +CMD [ "yarn", "serve" ] HEALTHCHECK --start-period=30s --interval=10s --timeout=5s \ CMD wget --no-verbose --tries=1 --spider http://localhost:8080/ || exit 1 From 559c339f6a6c5a44fde7bf51c87abb0dcefa2e01 Mon Sep 17 00:00:00 2001 From: jon r Date: Thu, 27 Feb 2025 01:59:49 +0100 Subject: [PATCH 11/28] fix(Dockerfile): chmod octal notation for Docker --- api/Dockerfile.alpine | 2 +- api/Dockerfile.debian | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/Dockerfile.alpine b/api/Dockerfile.alpine index 7b08b651d..29d8e15a3 100644 --- a/api/Dockerfile.alpine +++ b/api/Dockerfile.alpine @@ -133,7 +133,7 @@ RUN --mount=type=cache,target=~/.cache/pip; \ set -eux; \ pip3 install --no-deps --editable . -ADD --chown=0:0 --chmod=u+x ./entrypoint.sh / +ADD --chown=0:0 --chmod=0755 ./entrypoint.sh / ENTRYPOINT [ "/entrypoint.sh" ] HEALTHCHECK --start-period=60s --interval=10s --timeout=5s --retries=3 \ diff --git a/api/Dockerfile.debian b/api/Dockerfile.debian index 4b3a561c2..3e8c6d28a 100644 --- a/api/Dockerfile.debian +++ b/api/Dockerfile.debian @@ -69,7 +69,7 @@ WORKDIR /app COPY . /app RUN poetry install --extras typesense -ADD --chown=0:0 --chmod=u+x ./entrypoint.sh / +ADD --chown=0:0 --chmod=0755 ./entrypoint.sh / ENTRYPOINT [ "/entrypoint.sh" ] HEALTHCHECK --start-period=60s --interval=10s --timeout=5s --retries=3 \ From 9e91ec3d961bd5d20112e60d26803177504bd289 Mon Sep 17 00:00:00 2001 From: jon r Date: Sat, 15 Mar 2025 16:50:27 +0100 Subject: [PATCH 12/28] chore(compose): call frontend application app --- .env.example | 2 +- compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 4a247e3d2..a56a491ab 100644 --- a/.env.example +++ b/.env.example @@ -47,7 +47,7 @@ NGINX_MAX_BODY_SIZE=10G FUNKWHALE_API_HOST=api FUNKWHALE_API_PORT=5000 -FUNKWHALE_FRONT_HOST=front +FUNKWHALE_FRONT_HOST=app FUNKWHALE_FRONT_PORT=${VUE_PORT} # postgres diff --git a/compose.yml b/compose.yml index 2b5e2f9ff..b184ab313 100644 --- a/compose.yml +++ b/compose.yml @@ -13,7 +13,7 @@ x-django: &django dns_search: funkwhale.test services: - front: + app: extends: file: ./compose/app.vue.yml service: app @@ -35,7 +35,7 @@ services: file: ./compose/app.nginx.yml service: nginx depends_on: - front: + app: condition: service_healthy api: condition: service_healthy From 119794d5db0589083c105678a6f758bba202bcfd Mon Sep 17 00:00:00 2001 From: jon r Date: Sat, 15 Mar 2025 16:51:49 +0100 Subject: [PATCH 13/28] fix(front/Dockerfile.dev): use adaptive healthcheck --- front/Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/Dockerfile.dev b/front/Dockerfile.dev index bd9d14299..b11b022ee 100644 --- a/front/Dockerfile.dev +++ b/front/Dockerfile.dev @@ -14,4 +14,4 @@ COPY . . CMD [ "yarn", "serve" ] HEALTHCHECK --start-period=30s --interval=10s --timeout=5s \ - CMD wget --no-verbose --tries=1 --spider http://localhost:8080/ || exit 1 + CMD wget --no-verbose --tries=1 --spider http://localhost:${VUE_PORT}/ || exit 1 From d9d71bccf47f263949c2a22f91c9f4eca7fc8884 Mon Sep 17 00:00:00 2001 From: jon r Date: Sat, 15 Mar 2025 16:54:58 +0100 Subject: [PATCH 14/28] chore(compose): update Typesense and add healthcheck adapted from https://github.com/typesense/typesense/issues/441#issuecomment-2452014001 --- compose/app.typesense.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/compose/app.typesense.yml b/compose/app.typesense.yml index 110fa90f4..adfec6b75 100644 --- a/compose/app.typesense.yml +++ b/compose/app.typesense.yml @@ -2,9 +2,20 @@ services: typesense: environment: - TYPESENSE_API_KEY - image: typesense/typesense:27.1 + image: typesense/typesense:28.0 networks: - internal volumes: - ../.state/${COMPOSE_PROJECT_NAME:-funkwhale}/typesense/data:/data command: --data-dir /data --api-key=$${TYPESENSE_API_KEY} --enable-cors + healthcheck: + test: + [ + "CMD", + "bash", + "-c", + "exec 3<>/dev/tcp/localhost/8108 && printf 'GET /health HTTP/1.1\\r\\nConnection: close\\r\\n\\r\\n' >&3 && head -n1 <&3 | grep '200' && exec 3>&-", + ] + interval: 10s + timeout: 5s + retries: 3 From 2bee3686834e6cc85717a621b3b5e79aca351808 Mon Sep 17 00:00:00 2001 From: jon r Date: Sat, 15 Mar 2025 17:00:13 +0100 Subject: [PATCH 15/28] fix(compose): split Django and Celery healthchecks - This also fixes a regression with Celery not finding its broker in the worker container, by adding a dedictad variable and assigning a separate Redis database /1. Some issues were complaining about corruptions in the Celery database, which might have come from reusing the same /0 database as Django cache and queue. - Renames the celeryworker container to just worker. This partly undoes 4ee6190cc196d25d4bb400aaf1d7cdfd1e3b01c2 --- .env.example | 3 ++- api/Dockerfile.alpine | 3 --- api/Dockerfile.debian | 3 --- api/entrypoint.sh | 2 +- compose.yml | 4 ++-- compose/app.django.yml | 25 +++++++++++++++++++++++-- compose/net.minio.yml | 2 +- docs/developer/setup/docker.md | 2 +- 8 files changed, 30 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index a56a491ab..8a093903e 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ -# api + celeryworker +# api + worker DEBUG=True FORCE=True DEFAULT_FROM_EMAIL=hello@funkwhale.test @@ -12,6 +12,7 @@ DJANGO_SETTINGS_MODULE=config.settings.local DATABASE_URL=postgresql://postgres@postgres/postgres CACHE_URL=redis://redis:6379/0 +CELERY_BROKER_URL=redis://redis:6379/1 EMAIL_CONFIG=smtp://mailpit.funkwhale.test:1025 FORCE_HTTPS_URLS=True diff --git a/api/Dockerfile.alpine b/api/Dockerfile.alpine index 29d8e15a3..3149650d0 100644 --- a/api/Dockerfile.alpine +++ b/api/Dockerfile.alpine @@ -135,6 +135,3 @@ RUN --mount=type=cache,target=~/.cache/pip; \ ADD --chown=0:0 --chmod=0755 ./entrypoint.sh / ENTRYPOINT [ "/entrypoint.sh" ] - -HEALTHCHECK --start-period=60s --interval=10s --timeout=5s --retries=3 \ - CMD curl -o /dev/null -s -w "%{http_code}" http://localhost:5000/api/v1 | grep "301" || exit 1 diff --git a/api/Dockerfile.debian b/api/Dockerfile.debian index 3e8c6d28a..e59e86fe0 100644 --- a/api/Dockerfile.debian +++ b/api/Dockerfile.debian @@ -71,6 +71,3 @@ RUN poetry install --extras typesense ADD --chown=0:0 --chmod=0755 ./entrypoint.sh / ENTRYPOINT [ "/entrypoint.sh" ] - -HEALTHCHECK --start-period=60s --interval=10s --timeout=5s --retries=3 \ - CMD curl -o /dev/null -s -w "%{http_code}" http://localhost:5000/api/v1 | grep "301" || exit 1 diff --git a/api/entrypoint.sh b/api/entrypoint.sh index 830ea9594..2c8a70abb 100755 --- a/api/entrypoint.sh +++ b/api/entrypoint.sh @@ -30,7 +30,7 @@ case "${1}" in ${0} collectstatic ${0} uvicorn ;; - develop-celery) + develop-worker) export CELERYD_CONCURRENCY=0 watchmedo auto-restart \ --patterns="*.py" \ diff --git a/compose.yml b/compose.yml index b184ab313..f4bdc0053 100644 --- a/compose.yml +++ b/compose.yml @@ -24,10 +24,10 @@ services: service: api <<: *django - celeryworker: + worker: extends: file: ./compose/app.django.yml - service: celeryworker + service: worker <<: *django nginx: diff --git a/compose/app.django.yml b/compose/app.django.yml index 3452c1dfb..b609da6b0 100644 --- a/compose/app.django.yml +++ b/compose/app.django.yml @@ -26,6 +26,7 @@ x-django: &django - DATABASE_URL - CACHE_URL + - CELERY_BROKER_URL - EMAIL_CONFIG - TYPESENSE_API_KEY @@ -51,7 +52,27 @@ services: context: ../api dockerfile: Dockerfile.debian command: develop + healthcheck: + test: + [ + "CMD-SHELL", + "curl -o /dev/null -s -w '%{http_code}' http://localhost:5000/api/v1 | grep '301' || exit 1", + ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 60s - celeryworker: + worker: <<: *django - command: develop-celery + command: develop-worker + healthcheck: + test: + [ + "CMD-SHELL", + "celery -A funkwhale_api.taskapp status | grep 'OK' || exit 1", + ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s diff --git a/compose/net.minio.yml b/compose/net.minio.yml index f8e180db9..ce724ab14 100644 --- a/compose/net.minio.yml +++ b/compose/net.minio.yml @@ -18,6 +18,6 @@ services: depends_on: minio: {} - celeryworker: + worker: depends_on: minio: {} diff --git a/docs/developer/setup/docker.md b/docs/developer/setup/docker.md index 283259ecc..7f32cf3f2 100644 --- a/docs/developer/setup/docker.md +++ b/docs/developer/setup/docker.md @@ -284,7 +284,7 @@ Username `funkwhale` is not permitted. You need to export COMPOSE_PROJECT_NAME t Recycle individual containers: ```sh -docker compose rm -sf api celeryworker; docker compose up -d api celeryworker +docker compose rm -sf api worker; docker compose up -d api worker ``` Once you're done with the containers, you can stop them all: From 65ec9c33fe3b59f6faf93e03d91095b2d678596b Mon Sep 17 00:00:00 2001 From: jon r Date: Sat, 15 Mar 2025 18:06:06 +0100 Subject: [PATCH 16/28] feat(compose): add temporary redirect to web container --- compose.yml | 1 + .../dynamic/{api-dashboard.yml => http.api-dashboard.yml} | 0 .../etc/traefik/dynamic/{mailpit.yml => http.mailpit.yml} | 0 compose/etc/traefik/dynamic/http.redirect-scheme.yml | 6 ++++++ 4 files changed, 7 insertions(+) rename compose/etc/traefik/dynamic/{api-dashboard.yml => http.api-dashboard.yml} (100%) rename compose/etc/traefik/dynamic/{mailpit.yml => http.mailpit.yml} (100%) create mode 100644 compose/etc/traefik/dynamic/http.redirect-scheme.yml diff --git a/compose.yml b/compose.yml index f4bdc0053..a8ef017d4 100644 --- a/compose.yml +++ b/compose.yml @@ -47,6 +47,7 @@ services: - "traefik.http.routers.test-funkwhale-${COMPOSE_PROJECT_NAME:-funkwhale}-web.rule=Host(`${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}`)" - "traefik.http.routers.test-funkwhale-${COMPOSE_PROJECT_NAME:-funkwhale}-web.entrypoints=web" + - "traefik.http.routers.test-funkwhale-${COMPOSE_PROJECT_NAME:-funkwhale}-web.middlewares=redirect-scheme@file" - "traefik.http.routers.test-funkwhale-${COMPOSE_PROJECT_NAME:-funkwhale}-webs.rule=Host(`${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}`)" - "traefik.http.routers.test-funkwhale-${COMPOSE_PROJECT_NAME:-funkwhale}-webs.entrypoints=webs" diff --git a/compose/etc/traefik/dynamic/api-dashboard.yml b/compose/etc/traefik/dynamic/http.api-dashboard.yml similarity index 100% rename from compose/etc/traefik/dynamic/api-dashboard.yml rename to compose/etc/traefik/dynamic/http.api-dashboard.yml diff --git a/compose/etc/traefik/dynamic/mailpit.yml b/compose/etc/traefik/dynamic/http.mailpit.yml similarity index 100% rename from compose/etc/traefik/dynamic/mailpit.yml rename to compose/etc/traefik/dynamic/http.mailpit.yml diff --git a/compose/etc/traefik/dynamic/http.redirect-scheme.yml b/compose/etc/traefik/dynamic/http.redirect-scheme.yml new file mode 100644 index 000000000..f9292aae4 --- /dev/null +++ b/compose/etc/traefik/dynamic/http.redirect-scheme.yml @@ -0,0 +1,6 @@ +http: + middlewares: + redirect-scheme: + redirectScheme: + scheme: https + permanent: false From 2b5bf1d82d586f5bf403a9e2562ebc7ed5493cc8 Mon Sep 17 00:00:00 2001 From: jon r Date: Sat, 15 Mar 2025 18:06:55 +0100 Subject: [PATCH 17/28] chore(compose): use canonical name for web service --- .env.example | 10 +++++----- compose.yml | 4 ++-- compose/app.nginx.yml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 8a093903e..7198773d7 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,7 @@ FUNKWHALE_DOMAIN=funkwhale.test FUNKWHALE_PROTOCOL=https DJANGO_SECRET_KEY=dev -DJANGO_ALLOWED_HOSTS=.funkwhale.test,nginx +DJANGO_ALLOWED_HOSTS=.funkwhale.test,web DJANGO_SETTINGS_MODULE=config.settings.local DATABASE_URL=postgresql://postgres@postgres/postgres @@ -24,11 +24,11 @@ PYTHONTRACEMALLOC=0 # api -FUNKWHALE_SPA_HTML_ROOT=http://nginx/ +FUNKWHALE_SPA_HTML_ROOT=${FUNKWHALE_PROTOCOL}://${FUNKWHALE_DOMAIN} LDAP_ENABLED=False BROWSABLE_API_ENABLED=True -# api + nginx +# api + web STATIC_ROOT=/staticfiles MEDIA_ROOT=/data/media @@ -36,12 +36,12 @@ MEDIA_ROOT=/data/media # api + Typesense TYPESENSE_API_KEY=apikey -# front +# app HOST=0.0.0.0 VUE_PORT=8080 -# nginx +# web NGINX_MAX_BODY_SIZE=10G diff --git a/compose.yml b/compose.yml index a8ef017d4..7c3d86814 100644 --- a/compose.yml +++ b/compose.yml @@ -30,10 +30,10 @@ services: service: worker <<: *django - nginx: + web: extends: file: ./compose/app.nginx.yml - service: nginx + service: web depends_on: app: condition: service_healthy diff --git a/compose/app.nginx.yml b/compose/app.nginx.yml index 6000b3491..1619fa6ee 100644 --- a/compose/app.nginx.yml +++ b/compose/app.nginx.yml @@ -1,5 +1,5 @@ services: - nginx: + web: image: nginx networks: - internal From 1eaff2b398d8cbf30d748149009b137e20cc8103 Mon Sep 17 00:00:00 2001 From: jon r Date: Sat, 15 Mar 2025 18:27:00 +0100 Subject: [PATCH 18/28] docs(changelog.d/2406): update MR title next to issue number --- changes/changelog.d/2406.chore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/changelog.d/2406.chore b/changes/changelog.d/2406.chore index 55f997bf2..f6532fb9d 100644 --- a/changes/changelog.d/2406.chore +++ b/changes/changelog.d/2406.chore @@ -1 +1 @@ -Refactor Django Compose environment (#2406) +Chore: refactor development environment manifests (#2406) From 17e1ed50517b094db2b8e741a8e9440b44c42780 Mon Sep 17 00:00:00 2001 From: jon r Date: Mon, 31 Mar 2025 23:57:38 +0200 Subject: [PATCH 19/28] fix(compose): use common internal address for SPA root --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 7198773d7..ef80fee8e 100644 --- a/.env.example +++ b/.env.example @@ -24,7 +24,7 @@ PYTHONTRACEMALLOC=0 # api -FUNKWHALE_SPA_HTML_ROOT=${FUNKWHALE_PROTOCOL}://${FUNKWHALE_DOMAIN} +FUNKWHALE_SPA_HTML_ROOT=http://web LDAP_ENABLED=False BROWSABLE_API_ENABLED=True From 058777912720daf88202b96efcc1d4a96d562cad Mon Sep 17 00:00:00 2001 From: jon r Date: Mon, 31 Mar 2025 23:58:26 +0200 Subject: [PATCH 20/28] feat(compose): update to use Bake, as suggested by the Compose binary --- .env.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.env.example b/.env.example index ef80fee8e..eda866304 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,5 @@ +COMPOSE_BAKE=true + # api + worker DEBUG=True FORCE=True From 22edb05a998885b142f34453b01218eef60e665a Mon Sep 17 00:00:00 2001 From: jon r Date: Tue, 1 Apr 2025 00:03:05 +0200 Subject: [PATCH 21/28] docs(developer/setup/docker): refine --- docs/developer/setup/docker.md | 41 +++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/docs/developer/setup/docker.md b/docs/developer/setup/docker.md index 7f32cf3f2..e2676f893 100644 --- a/docs/developer/setup/docker.md +++ b/docs/developer/setup/docker.md @@ -33,10 +33,13 @@ Funkwhale can be run in Docker containers for local development. You can work on :::: 6. Activate the pre-commit hook: + ```sh pre-commit install ``` + 7. Finally, initialise the environment: + ```sh cp .env.example .env ``` @@ -50,16 +53,18 @@ Funkwhale provides a `compose.yml` file following the default file naming conven To set up your Docker environment: 1. Create a network for federation support via the web proxy: + ```sh docker network create web ``` + 2. Then build the application containers. Run this command any time there are upstream changes or dependency changes to ensure you're up-to-date. ```sh docker compose build ``` -## Set up auxiliary services +## Set up network services To support ActivityPub in the local development environment, we use a combination of auxiliary services that provide DNS-based discovery, local email delivery and web/TLS termination. This also has the benefit that we can talk to @@ -85,9 +90,11 @@ The services bind to the following ports on the default Docker bridge network: 1. Create a wildcard certificate for the Common Name (CN) `funkwhale.test` and the Subject Alternative Name (SAN) `*.funkwhale.test` which will be installed into your system and browser trust stores with: + ```sh mkcert -install -cert-file compose/var/test.crt -key-file compose/var/test.key "funkwhale.test" "*.funkwhale.test" ``` + It will be used by Træefik to secure connections, which is needed for ActivityPub to work locally. @@ -318,9 +325,9 @@ Set up as many different projects as you need. Make sure the `COMPOSE_PROJECT_NAME` and `VUE_PORT` variables are unique per instance. ```sh -export COMPOSE_PROJECT_NAME=node2 +export COMPOSE_PROJECT_NAME=node1 # VUE_PORT this has to be unique for each instance -export VUE_PORT=1234 +export VUE_PORT=8081 docker compose run --rm api funkwhale-manage fw users create --superuser docker compose up -d ``` @@ -333,23 +340,29 @@ You can access your project at `https://{COMPOSE_PROJECT_NAME}.funkwhale.test`. You may as well address the different Compose projects by using ad hoc environment variables: -``` -COMPOSE_PROJECT_NAME=node1 VUE_PORT=1234 docker compose run --rm api funkwhale-manage fw users create --superuser -COMPOSE_PROJECT_NAME=node1 VUE_PORT=1234 docker compose up -d +```sh +COMPOSE_PROJECT_NAME=node1 VUE_PORT=8081 docker compose run --rm api funkwhale-manage fw users create --superuser +COMPOSE_PROJECT_NAME=node1 VUE_PORT=8081 docker compose up -d ``` The `node1` instance will be available at [https://node1.funkwhale.test](https://node1.funkwhale.test). -``` -COMPOSE_PROJECT_NAME=node2 VUE_PORT=1235 docker compose run --rm api funkwhale-manage fw users create --superuser -COMPOSE_PROJECT_NAME=node2 VUE_PORT=1235 docker compose up -d +```sh +COMPOSE_PROJECT_NAME=node2 VUE_PORT=8082 docker compose run --rm api funkwhale-manage fw users create --superuser +COMPOSE_PROJECT_NAME=node2 VUE_PORT=8082 docker compose up -d ``` The `node2` instance will be available at [https://node2.funkwhale.test](https://node2.funkwhale.test). -Proceed freely with different sets of values for `COMPOSE_PROJECT_NAME` and +Proceed freely with different sets of non-overlapping values for `COMPOSE_PROJECT_NAME` and `VUE_PORT`. +As a rule of thumb, remember to: + +- Prepend `COMPOSE_PROJECT_NAME=node1 VUE_PORT=8081` to the lifecycle commands `up` and `run` to execute containers of additional instances. +- Prepend `COMPOSE_PROJECT_NAME=node1` to any other `docker compose` command to work with the indicated instance. + +By example, this mechanic also applies to the [set up of local data for development](#set-up-local-data-for-development) above. ::: ::::{tab-set} @@ -416,11 +429,9 @@ To build the documentation locally run: docker compose -f compose.docs.yml up -d ``` -The documentation is then accessible at [https://docs.funkwhale.test](https://docs.funkwhale.test). The OpenAPI schema is available at [https://openapi.funkwhale.test](https://openapi.funkwhale.test). +The documentation is then accessible at . The OpenAPI schema is available at . The UI component library will be served at . -Fallback ports are available for the documentation at -[http://localhost:8001/](http://localhost:8001/) and for the OpenAPI schema at -[http://localhost:8002/](http://localhost:8002/). +Fallback ports are available for the documentation at , for the OpenAPI schema at and for the UI component library at . Maintain their life cycle with similar commands to those used to -[set up auxiliary services (point 2.)](#set-up-auxiliary-services). +[set up network services (point 2.)](#set-up-network-services). From 507346905f6acd68d1814eae5d1873608290f92a Mon Sep 17 00:00:00 2001 From: jon r Date: Tue, 1 Apr 2025 00:08:28 +0200 Subject: [PATCH 22/28] docs(developer/setup/docker): add Updating section + split Seeding into two parts --- docs/developer/setup/docker.md | 83 ++++++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 18 deletions(-) diff --git a/docs/developer/setup/docker.md b/docs/developer/setup/docker.md index e2676f893..eba9b3123 100644 --- a/docs/developer/setup/docker.md +++ b/docs/developer/setup/docker.md @@ -264,29 +264,20 @@ Review the configuration: docker compose config ``` -### Set up local data for development +## Set up local data for development -You can create local data to mimic a live environment. - -Add some fake data to populate the database. The following command creates 25 artists with random albums, tracks, and metadata. +You can create local data to simulate a live environment. We are providing a procedure to create fake data to populate the database. The following command creates 25 artists with random albums, tracks, and metadata. ```sh -command="from funkwhale_api.music import fake_data; fake_data.create_data()" -echo $command | docker compose run --rm -T api funkwhale-manage shell -i python +docker compose run --rm -T api \ + funkwhale-manage shell -i python \ + <<< \ + "from funkwhale_api.music import fake_data; fake_data.create_data()" ``` -This will launch a development funkwhale instance with a super user having `COMPOSE_PROJECT_NAME` as username and `funkwhale` as password. Libraries, listenings and music data will be associated with the superuser : +The generated tracks do not contain any audio and are here for testing purposes of metadata handling only. -```sh -export COMPOSE_PROJECT_NAME=node1 ; export VUE_PORT=8882 ; docker compose run --rm api funkwhale-manage migrate ; echo "from funkwhale_api.music import fake_data; fake_data.create_data(super_user_name=\"$COMPOSE_PROJECT_NAME\")" | docker compose run --rm -T api funkwhale-manage shell -i python - -``` - -```{note} -Username `funkwhale` is not permitted. You need to export COMPOSE_PROJECT_NAME to make sure it's different from `funkwhale` -``` - -### Lifecycle +## Lifecycle Recycle individual containers: @@ -319,7 +310,7 @@ instances: rm -rf .state/ ``` -### Running multiple instances +## Running multiple instances Set up as many different projects as you need. Make sure the `COMPOSE_PROJECT_NAME` and `VUE_PORT` variables are unique per instance. @@ -421,6 +412,62 @@ to learn how else you may interact directly with containers, when needed. :::: +## Updating local environments + +During development you will find yourself switching between branches and pulling new configuration from your remotes, at least from `develop` and your feature branches. + +If the `.env.example` file changed, you need to make sure all are present in your current environment `.env`. + +```sh +diff .env .env.example +``` + +In most cases when (a) changes are present and (b) you did not customise or modify the setup, then you are able to simply copy the new version. + +```sh +cp .env.example .env +``` + +In presence of customisations, you need to adapt the values manually to the example. + +If any of the `Dockerfile` manifests changed, you need to rebuild the (affected) containers. + +```sh +docker compose build +``` + +Then recreate the application containers. + +```sh +docker compose up -d --force-recreate +``` + +For the additional instances, this reads: + +```sh +COMPOSE_PROJECT_NAME=node1 docker compose build +COMPOSE_PROJECT_NAME=node1 VUE_PORT=8081 docker compose up -d --force-recreate +``` + +## Seeding additional instances + +We provide a convenience method to initialise the additional Funkwhale instances with fake seed data altogether with a super user having `COMPOSE_PROJECT_NAME` as username and `funkwhale` as password. Libraries, listenings and music data will be associated to that superuser. + +```sh +COMPOSE_PROJECT_NAME=node1 docker compose run --rm -T api \ + funkwhale-manage shell -i python \ + <<< \ + "from os import getenv; from funkwhale_api.music import fake_data; fake_data.create_data(super_user_name=getenv('FUNKWHALE_HOSTNAME').split('.')[0])" +``` + +```{note} +The username `funkwhale` is not permitted, since it violates the password constraint of not being equal to the password. Therefore you need to export the `COMPOSE_PROJECT_NAME` to make sure the method is only run in cases where it will be different from `funkwhale`. + +This step does not apply to the default instance when running `docker compose` without specifying a `COMPOSE_PROJECT_NAME`. + +In this case follow the manual steps from above. First create a super user as described in [set up application services](#set-up-application-services) and then continue with the [set up of local data for development](#set-up-local-data-for-development). +``` + ## Local documentation To build the documentation locally run: From 987980db1fca56e43d5a9d8d0335564d801b8fb7 Mon Sep 17 00:00:00 2001 From: jon r Date: Fri, 4 Apr 2025 14:14:53 +0200 Subject: [PATCH 23/28] docs(developer/setup/docker): move documentation section up to the other set up instructions [skip ci] --- docs/developer/setup/docker.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/developer/setup/docker.md b/docs/developer/setup/docker.md index eba9b3123..797e7264d 100644 --- a/docs/developer/setup/docker.md +++ b/docs/developer/setup/docker.md @@ -277,6 +277,21 @@ docker compose run --rm -T api \ The generated tracks do not contain any audio and are here for testing purposes of metadata handling only. +## Set up local documentation + +To build the documentation locally run: + +```sh +docker compose -f compose.docs.yml up -d +``` + +The documentation is then accessible at . The OpenAPI schema is available at . The UI component library will be served at . + +Fallback ports are available for the documentation at , for the OpenAPI schema at and for the UI component library at . + +Maintain their life cycle with similar commands to those used to +[set up network services (point 2.)](#set-up-network-services). + ## Lifecycle Recycle individual containers: @@ -467,18 +482,3 @@ This step does not apply to the default instance when running `docker compose` w In this case follow the manual steps from above. First create a super user as described in [set up application services](#set-up-application-services) and then continue with the [set up of local data for development](#set-up-local-data-for-development). ``` - -## Local documentation - -To build the documentation locally run: - -```sh -docker compose -f compose.docs.yml up -d -``` - -The documentation is then accessible at . The OpenAPI schema is available at . The UI component library will be served at . - -Fallback ports are available for the documentation at , for the OpenAPI schema at and for the UI component library at . - -Maintain their life cycle with similar commands to those used to -[set up network services (point 2.)](#set-up-network-services). From 93cfa3cfd3a3173a989b389469d1b2f728585be1 Mon Sep 17 00:00:00 2001 From: jon r Date: Fri, 4 Apr 2025 14:21:32 +0200 Subject: [PATCH 24/28] docs(developer/setup/docker): call fixture by its use and not the implicit location --- docs/developer/setup/docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/setup/docker.md b/docs/developer/setup/docker.md index 797e7264d..d48d43f70 100644 --- a/docs/developer/setup/docker.md +++ b/docs/developer/setup/docker.md @@ -264,7 +264,7 @@ Review the configuration: docker compose config ``` -## Set up local data for development +## Set up example data for development You can create local data to simulate a live environment. We are providing a procedure to create fake data to populate the database. The following command creates 25 artists with random albums, tracks, and metadata. From d33bfbe68e9ceb363e790243e3690d2c80eb9f6d Mon Sep 17 00:00:00 2001 From: jon r Date: Fri, 4 Apr 2025 14:22:10 +0200 Subject: [PATCH 25/28] docs(developer/setup/docker): add information about container engine lifecycle --- docs/developer/setup/docker.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/developer/setup/docker.md b/docs/developer/setup/docker.md index d48d43f70..3a866b6bb 100644 --- a/docs/developer/setup/docker.md +++ b/docs/developer/setup/docker.md @@ -325,6 +325,40 @@ instances: rm -rf .state/ ``` +### Container engine + +Your container engine, often Docker or Podman, over time will accumulate state. +This state comes in form of images, networks and containers. Due to the nature +of immutable infrastructure, assets are often replaced with newer generations. + +For images this means that old ones pile up, while new ones are being used with +current containers. + +Delete dangling images regularly. + +```sh +docker image prune +``` + +Delete all unused images, which are not used by any container, regularly. + +```sh +docker image prune -a +``` + +Delete all unused networks regularly. + +```sh +docker network prune +``` + +In case you run into trouble, as a last resort you can always reset the state of +your container engine completely. + +```sh +docker system prune -a +``` + ## Running multiple instances Set up as many different projects as you need. Make sure the From 074bff1b9ea815ec26f251af34e0ff82aa5c41da Mon Sep 17 00:00:00 2001 From: jon r Date: Fri, 4 Apr 2025 14:57:10 +0200 Subject: [PATCH 26/28] fix(compose/net): make web network mandatory --- compose.net.yml | 19 ++++++++++++++++++- compose/net.verify.yml | 1 - 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/compose.net.yml b/compose.net.yml index 560c5a45e..ab28f2f44 100644 --- a/compose.net.yml +++ b/compose.net.yml @@ -1,4 +1,11 @@ name: funkwhale-net +networks: + web: + external: true + +x-networks: &networks + - web + include: - path: compose/net.dnsmasq.yml - path: compose/net.traefik.yml @@ -12,4 +19,14 @@ include: # Comment out the following line if you have other containers # present on the docker0 network. - path: compose/net.helpers.docker0.yml - - path: compose/net.verify.yml +services: + verify-external-connectivity: + extends: + file: compose/net.verify.yml + service: verify-external-connectivity + networks: *networks + verify-internal-connectivity: + extends: + file: compose/net.verify.yml + service: verify-internal-connectivity + networks: *networks diff --git a/compose/net.verify.yml b/compose/net.verify.yml index b6e5b1caa..8a6c8eb1a 100644 --- a/compose/net.verify.yml +++ b/compose/net.verify.yml @@ -1,7 +1,6 @@ x-verify: &verify init: true image: "busybox" - network_mode: bridge dns: 172.17.0.1 dns_search: funkwhale.test From 93e09b6c96cf1ccc2eb06a98b2437e3d2d5d2461 Mon Sep 17 00:00:00 2001 From: jon r Date: Fri, 4 Apr 2025 14:59:32 +0200 Subject: [PATCH 27/28] docs(developer/setup/docker): use lifecycle tab set and exemplify more commands --- docs/developer/setup/docker.md | 90 ++++++++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 9 deletions(-) diff --git a/docs/developer/setup/docker.md b/docs/developer/setup/docker.md index 3a866b6bb..e7c904e85 100644 --- a/docs/developer/setup/docker.md +++ b/docs/developer/setup/docker.md @@ -294,19 +294,83 @@ Maintain their life cycle with similar commands to those used to ## Lifecycle +Your local Funkwhale development environment will undergo various lifecycles +during its existence. This is due to the immutable nature of containers and the +way how we selectively apply state to them. + +Make yourself familiar with the following lifecycle commands to get an +impression of the phase changes that happen in your local deployment. + +::::{tab-set} + +:::{tab-item} Application lifecycle + +Build the application images, which will be used to create containers later. +This is often also needed after switching a branch or pulling new commits. + +```sh +docker compose build +``` + +> Selectively rebuild container images. +> +> ```sh +> docker compose build api +> ``` + +Start the whole composition detached (`-d`) in the background: + +```sh +docker compose up -d +``` + +List running containers: + +```sh +docker compose ps +``` + +> You can use the `-f` flag behind the `compose` command to target alternative +> compositions: +> +> ```sh +> docker compose -f compose.net.yml ps +> docker compose -f compose.docs.yml ps -a +> ``` + Recycle individual containers: ```sh -docker compose rm -sf api worker; docker compose up -d api worker +docker compose up -d --force-recreate app api ``` -Once you're done with the containers, you can stop them all: +> Alternative, if the previous does not work as expected: +> +> ```sh +> docker compose rm -sf api worker; docker compose up -d api worker +> ``` + +Once you're done for the day, you can stop them all: ```sh docker compose stop ``` -If you want to destroy your containers, run the following: +List all containers, including expectedly or unexpectedly exited ones: + +```sh +docker compose ps -a +``` + +**The following commands are destructive.** + +Stop running containers and remove all of them, but keep the network: + +```sh +docker compose rm -sf +``` + +If you want to destroy your containers and the network, run: ```sh docker compose down @@ -318,14 +382,16 @@ Destroy all state of your containers: docker compose down --volumes ``` -Remove all state of all Funkwhale-related containers, incl. from additional -instances: +**Remove all state of all Funkwhale containers**, incl. from any additional +instance: ```sh rm -rf .state/ ``` -### Container engine +::: + +:::{tab-item} Container engine lifecycle Your container engine, often Docker or Podman, over time will accumulate state. This state comes in form of images, networks and containers. Due to the nature @@ -359,6 +425,10 @@ your container engine completely. docker system prune -a ``` +::: + +:::: + ## Running multiple instances Set up as many different projects as you need. Make sure the @@ -402,7 +472,7 @@ As a rule of thumb, remember to: - Prepend `COMPOSE_PROJECT_NAME=node1 VUE_PORT=8081` to the lifecycle commands `up` and `run` to execute containers of additional instances. - Prepend `COMPOSE_PROJECT_NAME=node1` to any other `docker compose` command to work with the indicated instance. -By example, this mechanic also applies to the [set up of local data for development](#set-up-local-data-for-development) above. +By example, this mechanic also applies to the [set up of example data for development](#set-up-example-data-for-development) above. ::: ::::{tab-set} @@ -494,10 +564,12 @@ docker compose up -d --force-recreate For the additional instances, this reads: ```sh -COMPOSE_PROJECT_NAME=node1 docker compose build COMPOSE_PROJECT_NAME=node1 VUE_PORT=8081 docker compose up -d --force-recreate ``` +The build images from the primary instance will be reused here, since they +carry the same name and are expected to be at the same revision. + ## Seeding additional instances We provide a convenience method to initialise the additional Funkwhale instances with fake seed data altogether with a super user having `COMPOSE_PROJECT_NAME` as username and `funkwhale` as password. Libraries, listenings and music data will be associated to that superuser. @@ -514,5 +586,5 @@ The username `funkwhale` is not permitted, since it violates the password constr This step does not apply to the default instance when running `docker compose` without specifying a `COMPOSE_PROJECT_NAME`. -In this case follow the manual steps from above. First create a super user as described in [set up application services](#set-up-application-services) and then continue with the [set up of local data for development](#set-up-local-data-for-development). +In this case follow the manual steps from above. First create a super user as described in [set up application services](#set-up-application-services) and then continue with the [set up of example data for development](#set-up-example-data-for-development). ``` From d43d436b89c60c4e0c9247968d71559793ccc871 Mon Sep 17 00:00:00 2001 From: jon r Date: Thu, 10 Apr 2025 11:00:51 +0200 Subject: [PATCH 28/28] Fix: use app container as SPA root --- .env.example | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index eda866304..49112eadd 100644 --- a/.env.example +++ b/.env.example @@ -24,9 +24,14 @@ C_FORCE_ROOT=true PYTHONDONTWRITEBYTECODE=true PYTHONTRACEMALLOC=0 +# app + +HOST=0.0.0.0 +VUE_PORT=8080 + # api -FUNKWHALE_SPA_HTML_ROOT=http://web +FUNKWHALE_SPA_HTML_ROOT=http://app:${VUE_PORT} LDAP_ENABLED=False BROWSABLE_API_ENABLED=True @@ -38,11 +43,6 @@ MEDIA_ROOT=/data/media # api + Typesense TYPESENSE_API_KEY=apikey -# app - -HOST=0.0.0.0 -VUE_PORT=8080 - # web NGINX_MAX_BODY_SIZE=10G