diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..c6c8b3621 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/.env.example b/.env.example index e8a58128f..49112eadd 100644 --- a/.env.example +++ b/.env.example @@ -1,16 +1,20 @@ -# api + celeryworker +COMPOSE_BAKE=true + +# api + worker DEBUG=True +FORCE=True DEFAULT_FROM_EMAIL=hello@funkwhale.test FUNKWHALE_DOMAIN=funkwhale.test FUNKWHALE_PROTOCOL=https DJANGO_SECRET_KEY=dev -DJANGO_ALLOWED_HOSTS=.funkwhale.test,nginx +DJANGO_ALLOWED_HOSTS=.funkwhale.test,web DJANGO_SETTINGS_MODULE=config.settings.local DATABASE_URL=postgresql://postgres@postgres/postgres CACHE_URL=redis://redis:6379/0 +CELERY_BROKER_URL=redis://redis:6379/1 EMAIL_CONFIG=smtp://mailpit.funkwhale.test:1025 FORCE_HTTPS_URLS=True @@ -20,17 +24,18 @@ C_FORCE_ROOT=true PYTHONDONTWRITEBYTECODE=true PYTHONTRACEMALLOC=0 +# app + +HOST=0.0.0.0 +VUE_PORT=8080 + # api -FUNKWHALE_SPA_HTML_ROOT=http://nginx/ +FUNKWHALE_SPA_HTML_ROOT=http://app:${VUE_PORT} LDAP_ENABLED=False BROWSABLE_API_ENABLED=True -# celeryworker - -CELERYD_CONCURRENCY=0 - -# api + nginx +# api + web STATIC_ROOT=/staticfiles MEDIA_ROOT=/data/media @@ -38,19 +43,14 @@ MEDIA_ROOT=/data/media # api + Typesense TYPESENSE_API_KEY=apikey -# front - -HOST=0.0.0.0 -VUE_PORT=8080 - -# nginx +# web NGINX_MAX_BODY_SIZE=10G FUNKWHALE_API_HOST=api FUNKWHALE_API_PORT=5000 -FUNKWHALE_FRONT_IP=front +FUNKWHALE_FRONT_HOST=app FUNKWHALE_FRONT_PORT=${VUE_PORT} # postgres diff --git a/.gitpod/docker-compose.yml b/.gitpod/docker-compose.yml index 62b5f5930..271786a1b 100644 --- a/.gitpod/docker-compose.yml +++ b/.gitpod/docker-compose.yml @@ -30,7 +30,7 @@ services: - "FUNKWHALE_API_IP=host.docker.internal" - "FUNKWHALE_API_HOST=host.docker.internal" - "FUNKWHALE_API_PORT=5000" - - "FUNKWHALE_FRONT_IP=host.docker.internal" + - "FUNKWHALE_FRONT_HOST=host.docker.internal" - "FUNKWHALE_FRONT_PORT=8080" - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-host.docker.internal}" - "FUNKWHALE_PROTOCOL=https" diff --git a/api/.dockerignore b/api/.dockerignore index 729de9825..dc2e56f7a 100644 --- a/api/.dockerignore +++ b/api/.dockerignore @@ -1,8 +1,8 @@ # Exclude everything and allow only the necessary files * -!/docker/ !/config/ !/funkwhale_api/ +!/entrypoint.sh !/manage.py !/poetry.lock !/pyproject.toml diff --git a/api/Dockerfile.alpine b/api/Dockerfile.alpine index 6a5599b08..3149650d0 100644 --- a/api/Dockerfile.alpine +++ b/api/Dockerfile.alpine @@ -103,6 +103,7 @@ ARG PIP_NO_CACHE_DIR=1 RUN set -eux; \ apk add --no-cache \ bash \ + curl \ ffmpeg \ gettext \ jpeg-dev \ @@ -132,6 +133,5 @@ RUN --mount=type=cache,target=~/.cache/pip; \ set -eux; \ pip3 install --no-deps --editable . -ENV IS_DOCKER_SETUP=true - -CMD ["./docker/server.sh"] +ADD --chown=0:0 --chmod=0755 ./entrypoint.sh / +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/api/Dockerfile.debian b/api/Dockerfile.debian index 8e4de64bb..e59e86fe0 100644 --- a/api/Dockerfile.debian +++ b/api/Dockerfile.debian @@ -51,6 +51,7 @@ ENV PATH="/venv/bin:$PATH" RUN --mount=type=cache,target=/var/lib/apt/lists \ apt update; \ apt install -y \ + curl \ ffmpeg \ gettext \ libjpeg-dev \ @@ -68,4 +69,5 @@ WORKDIR /app COPY . /app RUN poetry install --extras typesense -CMD ["./docker/server.sh"] +ADD --chown=0:0 --chmod=0755 ./entrypoint.sh / +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 776d73c2e..634e74235 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -1,6 +1,5 @@ import logging.config import sys -import warnings from collections import OrderedDict from urllib.parse import urlparse, urlsplit @@ -36,8 +35,8 @@ Available levels: """ -IS_DOCKER_SETUP = env.bool("IS_DOCKER_SETUP", False) - +# allows makemigrations and superuser creation +FORCE = env.bool("FORCE", False) if env("FUNKWHALE_SENTRY_DSN", default=None) is not None: import sentry_sdk @@ -393,31 +392,6 @@ vars().update(EMAIL_CONFIG) # ------------------------------------------------------------------------------ # See: https://docs.djangoproject.com/en/dev/ref/settings/#databases -# The `_database_url_docker` variable will only by used as default for DATABASE_URL -# in the context of a docker deployment. -_database_url_docker = None -if IS_DOCKER_SETUP and env.str("DATABASE_URL", None) is None: - warnings.warn( - DeprecationWarning( - "the automatically generated 'DATABASE_URL' configuration in the docker " - "setup is deprecated, please configure either the 'DATABASE_URL' " - "environment variable or the 'DATABASE_HOST', 'DATABASE_USER' and " - "'DATABASE_PASSWORD' environment variables instead" - ) - ) - _DOCKER_DATABASE_HOST = "postgres" - _DOCKER_DATABASE_PORT = 5432 - _DOCKER_DATABASE_USER = env.str("POSTGRES_ENV_POSTGRES_USER", "postgres") - _DOCKER_DATABASE_PASSWORD = env.str("POSTGRES_ENV_POSTGRES_PASSWORD", "") - _DOCKER_DATABASE_NAME = _DOCKER_DATABASE_USER - - _database_url_docker = ( - f"postgres:" - f"//{_DOCKER_DATABASE_USER}:{_DOCKER_DATABASE_PASSWORD}" - f"@{_DOCKER_DATABASE_HOST}:{_DOCKER_DATABASE_PORT}" - f"/{_DOCKER_DATABASE_NAME}" - ) - DATABASE_HOST = env.str("DATABASE_HOST", "localhost") """ The hostname of the PostgreSQL server. Defaults to ``localhost``. @@ -440,8 +414,7 @@ The name of the PostgreSQL database. Defaults to ``funkwhale``. """ DATABASE_URL = env.db( "DATABASE_URL", - _database_url_docker # This is only set in the context of a docker deployment. - or ( + ( f"postgres:" f"//{DATABASE_USER}:{DATABASE_PASSWORD}" f"@{DATABASE_HOST}:{DATABASE_PORT}" @@ -838,11 +811,7 @@ if AUTH_LDAP_ENABLED: # SLUGLIFIER AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify" -CACHE_URL_DEFAULT = "redis://127.0.0.1:6379/0" -if IS_DOCKER_SETUP: - CACHE_URL_DEFAULT = "redis://redis:6379/0" - -CACHE_URL = env.str("CACHE_URL", default=CACHE_URL_DEFAULT) +CACHE_URL = env.str("CACHE_URL", default="redis://127.0.0.1:6379/0") """ The URL of your redis server. For example: @@ -1527,10 +1496,7 @@ TYPESENSE_PROTOCOL = env("TYPESENSE_PROTOCOL", default="http") """Typesense listening protocol""" TYPESENSE_HOST = env( "TYPESENSE_HOST", - default="typesense" if IS_DOCKER_SETUP else "localhost", + default="localhost", ) -""" -Typesense hostname. Defaults to `localhost` on non-Docker deployments and to `typesense` on -Docker deployments. -""" +"""Typesense hostname. Defaults to `localhost`.""" TYPESENSE_NUM_TYPO = env("TYPESENSE_NUM_TYPO", default=5) diff --git a/api/config/settings/local.py b/api/config/settings/local.py index b1674f46d..70fb5a21e 100644 --- a/api/config/settings/local.py +++ b/api/config/settings/local.py @@ -152,6 +152,3 @@ REST_FRAMEWORK.update( ], } ) - -# allows makemigrations and superuser creation -FORCE = env("FORCE", default=1) diff --git a/api/docker/server.sh b/api/docker/server.sh deleted file mode 100755 index e4aebcb98..000000000 --- a/api/docker/server.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -set -eux - -funkwhale-manage collectstatic --noinput -funkwhale-manage migrate - -# shellcheck disable=SC2086 -exec gunicorn config.asgi:application \ - --workers "${FUNKWHALE_WEB_WORKERS-1}" \ - --worker-class uvicorn.workers.UvicornWorker \ - --bind 0.0.0.0:"${FUNKWHALE_API_PORT}" \ - ${GUNICORN_ARGS-} diff --git a/api/entrypoint.sh b/api/entrypoint.sh new file mode 100755 index 000000000..2c8a70abb --- /dev/null +++ b/api/entrypoint.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env sh + +set -eux + +case "${1}" in + gunicorn) + # shellcheck disable=SC2086 + exec gunicorn config.asgi:application \ + --workers "${FUNKWHALE_WEB_WORKERS:-1}" \ + --worker-class uvicorn.workers.UvicornWorker \ + --bind 0.0.0.0:"${FUNKWHALE_API_PORT}" \ + ${GUNICORN_ARGS:-} + ;; + migrate) + funkwhale-manage migrate + ;; + collectstatic) + funkwhale-manage collectstatic --noinput + ;; + uvicorn) + exec uvicorn \ + --reload config.asgi:application \ + --host 0.0.0.0 \ + --port 5000 \ + --reload-dir config/ \ + --reload-dir funkwhale_api/ + ;; + develop) + ${0} migrate + ${0} collectstatic + ${0} uvicorn + ;; + develop-worker) + export CELERYD_CONCURRENCY=0 + watchmedo auto-restart \ + --patterns="*.py" \ + --recursive \ + -- \ + celery \ + -A funkwhale_api.taskapp worker \ + -B \ + -l debug \ + -s /tmp/celerybeat-schedule \ + --concurrency=${CELERYD_CONCURRENCY} + ;; + *) + exec "${@}" + ;; +esac diff --git a/changes/changelog.d/2406.chore b/changes/changelog.d/2406.chore new file mode 100644 index 000000000..f6532fb9d --- /dev/null +++ b/changes/changelog.d/2406.chore @@ -0,0 +1 @@ +Chore: refactor development environment manifests (#2406) diff --git a/compose.net.yml b/compose.net.yml index 560c5a45e..ab28f2f44 100644 --- a/compose.net.yml +++ b/compose.net.yml @@ -1,4 +1,11 @@ name: funkwhale-net +networks: + web: + external: true + +x-networks: &networks + - web + include: - path: compose/net.dnsmasq.yml - path: compose/net.traefik.yml @@ -12,4 +19,14 @@ include: # Comment out the following line if you have other containers # present on the docker0 network. - path: compose/net.helpers.docker0.yml - - path: compose/net.verify.yml +services: + verify-external-connectivity: + extends: + file: compose/net.verify.yml + service: verify-external-connectivity + networks: *networks + verify-internal-connectivity: + extends: + file: compose/net.verify.yml + service: verify-internal-connectivity + networks: *networks diff --git a/compose.yml b/compose.yml index 55560113a..7c3d86814 100644 --- a/compose.yml +++ b/compose.yml @@ -4,59 +4,19 @@ networks: external: true x-django: &django - environment: - - DEBUG - - DEFAULT_FROM_EMAIL - - - DJANGO_SETTINGS_MODULE - - DJANGO_SECRET_KEY - - - EXTERNAL_REQUESTS_VERIFY_SSL - - - "FORCE_HTTPS_URLS=${FORCE_HTTPS_URLS:-False}" - - FUNKWHALE_PROTOCOL - - "FUNKWHALE_HOSTNAME=${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}" - - - DATABASE_URL - - CACHE_URL - - EMAIL_CONFIG - - TYPESENSE_API_KEY - - - "STATIC_URL=${FUNKWHALE_PROTOCOL}://${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}/static/" - - "MEDIA_URL=${FUNKWHALE_PROTOCOL}://${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}/media/" - - - STATIC_ROOT - - MEDIA_ROOT - - - FUNKWHALE_SPA_HTML_ROOT - - LDAP_ENABLED - - BROWSABLE_API_ENABLED - - "MUSIC_DIRECTORY_PATH=${MUSIC_DIRECTORY_PATH:-/music}" - - - C_FORCE_ROOT - - PYTHONDONTWRITEBYTECODE - - PYTHONTRACEMALLOC - + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy dns: 172.17.0.1 dns_search: funkwhale.test services: - front: - build: - context: ./front - dockerfile: Dockerfile.dev - ports: - - "${VUE_PORT:-8080}:${VUE_PORT:-8080}" - environment: - - HOST - - VUE_PORT - volumes: - - "./front:/app" - - "/app/node_modules" - - "./po:/po" - networks: - - internal - command: "yarn dev --host" + app: + extends: + file: ./compose/app.vue.yml + service: app api: extends: @@ -64,41 +24,30 @@ services: service: api <<: *django - celeryworker: + worker: extends: file: ./compose/app.django.yml - service: celeryworker + service: worker <<: *django - nginx: + web: extends: file: ./compose/app.nginx.yml - service: nginx - environment: - - "MUSIC_DIRECTORY_PATH=${MUSIC_DIRECTORY_PATH:-/music}" - - "FUNKWHALE_HOSTNAME=${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}" - - - FUNKWHALE_PROTOCOL - - - FUNKWHALE_API_HOST - - FUNKWHALE_API_PORT - - - FUNKWHALE_FRONT_IP - - FUNKWHALE_FRONT_PORT - - - NGINX_MAX_BODY_SIZE - - - STATIC_ROOT - - "MEDIA_ROOT=${MEDIA_ROOT:-/data/media}" + service: web + depends_on: + app: + condition: service_healthy + api: + condition: service_healthy networks: - web - internal - labels: - "traefik.enable=true" - "traefik.http.routers.test-funkwhale-${COMPOSE_PROJECT_NAME:-funkwhale}-web.rule=Host(`${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}`)" - "traefik.http.routers.test-funkwhale-${COMPOSE_PROJECT_NAME:-funkwhale}-web.entrypoints=web" + - "traefik.http.routers.test-funkwhale-${COMPOSE_PROJECT_NAME:-funkwhale}-web.middlewares=redirect-scheme@file" - "traefik.http.routers.test-funkwhale-${COMPOSE_PROJECT_NAME:-funkwhale}-webs.rule=Host(`${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}`)" - "traefik.http.routers.test-funkwhale-${COMPOSE_PROJECT_NAME:-funkwhale}-webs.entrypoints=webs" diff --git a/compose/app.django.yml b/compose/app.django.yml index 2b2b80f0e..b609da6b0 100644 --- a/compose/app.django.yml +++ b/compose/app.django.yml @@ -1,20 +1,49 @@ x-django: &django image: funkwhale-api + init: true + networks: + - internal volumes: - ../api:/app - - ../.env:/app/.env - "${MUSIC_DIRECTORY_SERVE_PATH:-../.state/music}:/music:ro" - "../.state/plugins:/srv/funkwhale/plugins" - "../.state/staticfiles:/staticfiles" - "../.state/media:/protected/media" - "../.state/${COMPOSE_PROJECT_NAME:-funkwhale}/media:/data/media" - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - networks: - - internal + environment: + - DEBUG + - FORCE + - DEFAULT_FROM_EMAIL + + - DJANGO_SETTINGS_MODULE + - DJANGO_SECRET_KEY + + - EXTERNAL_REQUESTS_VERIFY_SSL + + - "FORCE_HTTPS_URLS=${FORCE_HTTPS_URLS:-False}" + - FUNKWHALE_PROTOCOL + - "FUNKWHALE_HOSTNAME=${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}" + + - DATABASE_URL + - CACHE_URL + - CELERY_BROKER_URL + - EMAIL_CONFIG + - TYPESENSE_API_KEY + + - "STATIC_URL=${FUNKWHALE_PROTOCOL}://${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}/static/" + - "MEDIA_URL=${FUNKWHALE_PROTOCOL}://${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}/media/" + + - STATIC_ROOT + - MEDIA_ROOT + + - FUNKWHALE_SPA_HTML_ROOT + - LDAP_ENABLED + - BROWSABLE_API_ENABLED + - "MUSIC_DIRECTORY_PATH=${MUSIC_DIRECTORY_PATH:-/music}" + + - C_FORCE_ROOT + - PYTHONDONTWRITEBYTECODE + - PYTHONTRACEMALLOC services: api: @@ -22,28 +51,28 @@ services: build: context: ../api dockerfile: Dockerfile.debian + command: develop healthcheck: test: [ "CMD-SHELL", - 'docker compose logs api | grep -q "Uvicorn running on" || exit 0', + "curl -o /dev/null -s -w '%{http_code}' http://localhost:5000/api/v1 | grep '301' || exit 1", ] - interval: 3s + interval: 10s timeout: 5s retries: 3 - command: > - sh -c " - funkwhale-manage collectstatic --no-input && - uvicorn --reload config.asgi:application --host 0.0.0.0 --port 5000 --reload-dir config/ --reload-dir funkwhale_api/ - " + start_period: 60s - celeryworker: + worker: <<: *django - command: > - sh -c ' - pip install watchdog[watchmedo] && - watchmedo auto-restart --patterns="*.py" --recursive -- celery -A funkwhale_api.taskapp worker -l debug -B --concurrency=${CELERYD_CONCURRENCY} - ' - depends_on: - api: - condition: service_healthy + command: develop-worker + healthcheck: + test: + [ + "CMD-SHELL", + "celery -A funkwhale_api.taskapp status | grep 'OK' || exit 1", + ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s diff --git a/compose/app.nginx.yml b/compose/app.nginx.yml index 86cca2bf0..1619fa6ee 100644 --- a/compose/app.nginx.yml +++ b/compose/app.nginx.yml @@ -1,9 +1,24 @@ services: - nginx: + web: image: nginx - depends_on: - - api - - front + networks: + - internal + environment: + - "MUSIC_DIRECTORY_PATH=${MUSIC_DIRECTORY_PATH:-/music}" + - "FUNKWHALE_HOSTNAME=${COMPOSE_PROJECT_NAME:-funkwhale}.${FUNKWHALE_DOMAIN}" + + - FUNKWHALE_PROTOCOL + + - FUNKWHALE_API_HOST + - FUNKWHALE_API_PORT + + - FUNKWHALE_FRONT_HOST + - FUNKWHALE_FRONT_PORT + + - NGINX_MAX_BODY_SIZE + + - STATIC_ROOT + - "MEDIA_ROOT=${MEDIA_ROOT:-/data/media}" volumes: - "${MUSIC_DIRECTORY_SERVE_PATH:-../.state/music}:${MUSIC_DIRECTORY_PATH:-/music}:ro" @@ -14,5 +29,8 @@ services: - ../.state/staticfiles:/usr/share/nginx/html/staticfiles:ro - ../.state/media:/protected/media:ro - ../.state/${COMPOSE_PROJECT_NAME:-funkwhale}/media:/data/media:ro - networks: - - internal + healthcheck: + test: 'curl -o /dev/null -s -w "%{http_code}" http://localhost:80/ | grep "200" || exit 1' + interval: 5s + timeout: 3s + retries: 3 diff --git a/compose/app.typesense.yml b/compose/app.typesense.yml index 110fa90f4..adfec6b75 100644 --- a/compose/app.typesense.yml +++ b/compose/app.typesense.yml @@ -2,9 +2,20 @@ services: typesense: environment: - TYPESENSE_API_KEY - image: typesense/typesense:27.1 + image: typesense/typesense:28.0 networks: - internal volumes: - ../.state/${COMPOSE_PROJECT_NAME:-funkwhale}/typesense/data:/data command: --data-dir /data --api-key=$${TYPESENSE_API_KEY} --enable-cors + healthcheck: + test: + [ + "CMD", + "bash", + "-c", + "exec 3<>/dev/tcp/localhost/8108 && printf 'GET /health HTTP/1.1\\r\\nConnection: close\\r\\n\\r\\n' >&3 && head -n1 <&3 | grep '200' && exec 3>&-", + ] + interval: 10s + timeout: 5s + retries: 3 diff --git a/compose/app.vue.yml b/compose/app.vue.yml new file mode 100644 index 000000000..758a0cb0c --- /dev/null +++ b/compose/app.vue.yml @@ -0,0 +1,17 @@ +services: + app: + image: funkwhale-app + build: + context: ../front + dockerfile: Dockerfile.dev + command: "yarn dev --host" + ports: + - "${VUE_PORT:-8080}:${VUE_PORT:-8080}" + networks: + - internal + volumes: + - "../front:/app" + - "/app/node_modules" + environment: + - HOST + - VUE_PORT diff --git a/compose/etc/nginx/conf.dev b/compose/etc/nginx/conf.dev index 40af155ae..e867e2277 100644 --- a/compose/etc/nginx/conf.dev +++ b/compose/etc/nginx/conf.dev @@ -4,7 +4,7 @@ upstream funkwhale-api { } upstream funkwhale-front { - server ${FUNKWHALE_FRONT_IP}:${FUNKWHALE_FRONT_PORT}; + server ${FUNKWHALE_FRONT_HOST}:${FUNKWHALE_FRONT_PORT}; } # Required for websocket support. diff --git a/compose/etc/traefik/dynamic/api-dashboard.yml b/compose/etc/traefik/dynamic/http.api-dashboard.yml similarity index 100% rename from compose/etc/traefik/dynamic/api-dashboard.yml rename to compose/etc/traefik/dynamic/http.api-dashboard.yml diff --git a/compose/etc/traefik/dynamic/mailpit.yml b/compose/etc/traefik/dynamic/http.mailpit.yml similarity index 100% rename from compose/etc/traefik/dynamic/mailpit.yml rename to compose/etc/traefik/dynamic/http.mailpit.yml diff --git a/compose/etc/traefik/dynamic/http.redirect-scheme.yml b/compose/etc/traefik/dynamic/http.redirect-scheme.yml new file mode 100644 index 000000000..f9292aae4 --- /dev/null +++ b/compose/etc/traefik/dynamic/http.redirect-scheme.yml @@ -0,0 +1,6 @@ +http: + middlewares: + redirect-scheme: + redirectScheme: + scheme: https + permanent: false diff --git a/compose/net.minio.yml b/compose/net.minio.yml index f8e180db9..ce724ab14 100644 --- a/compose/net.minio.yml +++ b/compose/net.minio.yml @@ -18,6 +18,6 @@ services: depends_on: minio: {} - celeryworker: + worker: depends_on: minio: {} diff --git a/compose/net.verify.yml b/compose/net.verify.yml index b6e5b1caa..8a6c8eb1a 100644 --- a/compose/net.verify.yml +++ b/compose/net.verify.yml @@ -1,7 +1,6 @@ x-verify: &verify init: true image: "busybox" - network_mode: bridge dns: 172.17.0.1 dns_search: funkwhale.test diff --git a/docs/developer/setup/docker.md b/docs/developer/setup/docker.md index 283259ecc..e7c904e85 100644 --- a/docs/developer/setup/docker.md +++ b/docs/developer/setup/docker.md @@ -33,10 +33,13 @@ Funkwhale can be run in Docker containers for local development. You can work on :::: 6. Activate the pre-commit hook: + ```sh pre-commit install ``` + 7. Finally, initialise the environment: + ```sh cp .env.example .env ``` @@ -50,16 +53,18 @@ Funkwhale provides a `compose.yml` file following the default file naming conven To set up your Docker environment: 1. Create a network for federation support via the web proxy: + ```sh docker network create web ``` + 2. Then build the application containers. Run this command any time there are upstream changes or dependency changes to ensure you're up-to-date. ```sh docker compose build ``` -## Set up auxiliary services +## Set up network services To support ActivityPub in the local development environment, we use a combination of auxiliary services that provide DNS-based discovery, local email delivery and web/TLS termination. This also has the benefit that we can talk to @@ -85,9 +90,11 @@ The services bind to the following ports on the default Docker bridge network: 1. Create a wildcard certificate for the Common Name (CN) `funkwhale.test` and the Subject Alternative Name (SAN) `*.funkwhale.test` which will be installed into your system and browser trust stores with: + ```sh mkcert -install -cert-file compose/var/test.crt -key-file compose/var/test.key "funkwhale.test" "*.funkwhale.test" ``` + It will be used by Træefik to secure connections, which is needed for ActivityPub to work locally. @@ -257,43 +264,113 @@ Review the configuration: docker compose config ``` -### Set up local data for development +## Set up example data for development -You can create local data to mimic a live environment. - -Add some fake data to populate the database. The following command creates 25 artists with random albums, tracks, and metadata. +You can create local data to simulate a live environment. We are providing a procedure to create fake data to populate the database. The following command creates 25 artists with random albums, tracks, and metadata. ```sh -command="from funkwhale_api.music import fake_data; fake_data.create_data()" -echo $command | docker compose run --rm -T api funkwhale-manage shell -i python +docker compose run --rm -T api \ + funkwhale-manage shell -i python \ + <<< \ + "from funkwhale_api.music import fake_data; fake_data.create_data()" ``` -This will launch a development funkwhale instance with a super user having `COMPOSE_PROJECT_NAME` as username and `funkwhale` as password. Libraries, listenings and music data will be associated with the superuser : +The generated tracks do not contain any audio and are here for testing purposes of metadata handling only. + +## Set up local documentation + +To build the documentation locally run: ```sh -export COMPOSE_PROJECT_NAME=node1 ; export VUE_PORT=8882 ; docker compose run --rm api funkwhale-manage migrate ; echo "from funkwhale_api.music import fake_data; fake_data.create_data(super_user_name=\"$COMPOSE_PROJECT_NAME\")" | docker compose run --rm -T api funkwhale-manage shell -i python - +docker compose -f compose.docs.yml up -d ``` -```{note} -Username `funkwhale` is not permitted. You need to export COMPOSE_PROJECT_NAME to make sure it's different from `funkwhale` +The documentation is then accessible at . The OpenAPI schema is available at . The UI component library will be served at . + +Fallback ports are available for the documentation at , for the OpenAPI schema at and for the UI component library at . + +Maintain their life cycle with similar commands to those used to +[set up network services (point 2.)](#set-up-network-services). + +## Lifecycle + +Your local Funkwhale development environment will undergo various lifecycles +during its existence. This is due to the immutable nature of containers and the +way how we selectively apply state to them. + +Make yourself familiar with the following lifecycle commands to get an +impression of the phase changes that happen in your local deployment. + +::::{tab-set} + +:::{tab-item} Application lifecycle + +Build the application images, which will be used to create containers later. +This is often also needed after switching a branch or pulling new commits. + +```sh +docker compose build ``` -### Lifecycle +> Selectively rebuild container images. +> +> ```sh +> docker compose build api +> ``` + +Start the whole composition detached (`-d`) in the background: + +```sh +docker compose up -d +``` + +List running containers: + +```sh +docker compose ps +``` + +> You can use the `-f` flag behind the `compose` command to target alternative +> compositions: +> +> ```sh +> docker compose -f compose.net.yml ps +> docker compose -f compose.docs.yml ps -a +> ``` Recycle individual containers: ```sh -docker compose rm -sf api celeryworker; docker compose up -d api celeryworker +docker compose up -d --force-recreate app api ``` -Once you're done with the containers, you can stop them all: +> Alternative, if the previous does not work as expected: +> +> ```sh +> docker compose rm -sf api worker; docker compose up -d api worker +> ``` + +Once you're done for the day, you can stop them all: ```sh docker compose stop ``` -If you want to destroy your containers, run the following: +List all containers, including expectedly or unexpectedly exited ones: + +```sh +docker compose ps -a +``` + +**The following commands are destructive.** + +Stop running containers and remove all of them, but keep the network: + +```sh +docker compose rm -sf +``` + +If you want to destroy your containers and the network, run: ```sh docker compose down @@ -305,22 +382,62 @@ Destroy all state of your containers: docker compose down --volumes ``` -Remove all state of all Funkwhale-related containers, incl. from additional -instances: +**Remove all state of all Funkwhale containers**, incl. from any additional +instance: ```sh rm -rf .state/ ``` -### Running multiple instances +::: + +:::{tab-item} Container engine lifecycle + +Your container engine, often Docker or Podman, over time will accumulate state. +This state comes in form of images, networks and containers. Due to the nature +of immutable infrastructure, assets are often replaced with newer generations. + +For images this means that old ones pile up, while new ones are being used with +current containers. + +Delete dangling images regularly. + +```sh +docker image prune +``` + +Delete all unused images, which are not used by any container, regularly. + +```sh +docker image prune -a +``` + +Delete all unused networks regularly. + +```sh +docker network prune +``` + +In case you run into trouble, as a last resort you can always reset the state of +your container engine completely. + +```sh +docker system prune -a +``` + +::: + +:::: + +## Running multiple instances Set up as many different projects as you need. Make sure the `COMPOSE_PROJECT_NAME` and `VUE_PORT` variables are unique per instance. ```sh -export COMPOSE_PROJECT_NAME=node2 +export COMPOSE_PROJECT_NAME=node1 # VUE_PORT this has to be unique for each instance -export VUE_PORT=1234 +export VUE_PORT=8081 docker compose run --rm api funkwhale-manage fw users create --superuser docker compose up -d ``` @@ -333,23 +450,29 @@ You can access your project at `https://{COMPOSE_PROJECT_NAME}.funkwhale.test`. You may as well address the different Compose projects by using ad hoc environment variables: -``` -COMPOSE_PROJECT_NAME=node1 VUE_PORT=1234 docker compose run --rm api funkwhale-manage fw users create --superuser -COMPOSE_PROJECT_NAME=node1 VUE_PORT=1234 docker compose up -d +```sh +COMPOSE_PROJECT_NAME=node1 VUE_PORT=8081 docker compose run --rm api funkwhale-manage fw users create --superuser +COMPOSE_PROJECT_NAME=node1 VUE_PORT=8081 docker compose up -d ``` The `node1` instance will be available at [https://node1.funkwhale.test](https://node1.funkwhale.test). -``` -COMPOSE_PROJECT_NAME=node2 VUE_PORT=1235 docker compose run --rm api funkwhale-manage fw users create --superuser -COMPOSE_PROJECT_NAME=node2 VUE_PORT=1235 docker compose up -d +```sh +COMPOSE_PROJECT_NAME=node2 VUE_PORT=8082 docker compose run --rm api funkwhale-manage fw users create --superuser +COMPOSE_PROJECT_NAME=node2 VUE_PORT=8082 docker compose up -d ``` The `node2` instance will be available at [https://node2.funkwhale.test](https://node2.funkwhale.test). -Proceed freely with different sets of values for `COMPOSE_PROJECT_NAME` and +Proceed freely with different sets of non-overlapping values for `COMPOSE_PROJECT_NAME` and `VUE_PORT`. +As a rule of thumb, remember to: + +- Prepend `COMPOSE_PROJECT_NAME=node1 VUE_PORT=8081` to the lifecycle commands `up` and `run` to execute containers of additional instances. +- Prepend `COMPOSE_PROJECT_NAME=node1` to any other `docker compose` command to work with the indicated instance. + +By example, this mechanic also applies to the [set up of example data for development](#set-up-example-data-for-development) above. ::: ::::{tab-set} @@ -408,19 +531,60 @@ to learn how else you may interact directly with containers, when needed. :::: -## Local documentation +## Updating local environments -To build the documentation locally run: +During development you will find yourself switching between branches and pulling new configuration from your remotes, at least from `develop` and your feature branches. + +If the `.env.example` file changed, you need to make sure all are present in your current environment `.env`. ```sh -docker compose -f compose.docs.yml up -d +diff .env .env.example ``` -The documentation is then accessible at [https://docs.funkwhale.test](https://docs.funkwhale.test). The OpenAPI schema is available at [https://openapi.funkwhale.test](https://openapi.funkwhale.test). +In most cases when (a) changes are present and (b) you did not customise or modify the setup, then you are able to simply copy the new version. -Fallback ports are available for the documentation at -[http://localhost:8001/](http://localhost:8001/) and for the OpenAPI schema at -[http://localhost:8002/](http://localhost:8002/). +```sh +cp .env.example .env +``` -Maintain their life cycle with similar commands to those used to -[set up auxiliary services (point 2.)](#set-up-auxiliary-services). +In presence of customisations, you need to adapt the values manually to the example. + +If any of the `Dockerfile` manifests changed, you need to rebuild the (affected) containers. + +```sh +docker compose build +``` + +Then recreate the application containers. + +```sh +docker compose up -d --force-recreate +``` + +For the additional instances, this reads: + +```sh +COMPOSE_PROJECT_NAME=node1 VUE_PORT=8081 docker compose up -d --force-recreate +``` + +The build images from the primary instance will be reused here, since they +carry the same name and are expected to be at the same revision. + +## Seeding additional instances + +We provide a convenience method to initialise the additional Funkwhale instances with fake seed data altogether with a super user having `COMPOSE_PROJECT_NAME` as username and `funkwhale` as password. Libraries, listenings and music data will be associated to that superuser. + +```sh +COMPOSE_PROJECT_NAME=node1 docker compose run --rm -T api \ + funkwhale-manage shell -i python \ + <<< \ + "from os import getenv; from funkwhale_api.music import fake_data; fake_data.create_data(super_user_name=getenv('FUNKWHALE_HOSTNAME').split('.')[0])" +``` + +```{note} +The username `funkwhale` is not permitted, since it violates the password constraint of not being equal to the password. Therefore you need to export the `COMPOSE_PROJECT_NAME` to make sure the method is only run in cases where it will be different from `funkwhale`. + +This step does not apply to the default instance when running `docker compose` without specifying a `COMPOSE_PROJECT_NAME`. + +In this case follow the manual steps from above. First create a super user as described in [set up application services](#set-up-application-services) and then continue with the [set up of example data for development](#set-up-example-data-for-development). +``` diff --git a/front/Dockerfile.dev b/front/Dockerfile.dev index f280b65ae..b11b022ee 100644 --- a/front/Dockerfile.dev +++ b/front/Dockerfile.dev @@ -11,4 +11,7 @@ RUN yarn install COPY . . -CMD ["yarn", "serve"] +CMD [ "yarn", "serve" ] + +HEALTHCHECK --start-period=30s --interval=10s --timeout=5s \ + CMD wget --no-verbose --tries=1 --spider http://localhost:${VUE_PORT}/ || exit 1 diff --git a/templates/nginx.conf.j2 b/templates/nginx.conf.j2 index 503d7abcb..8af7dbc4b 100644 --- a/templates/nginx.conf.j2 +++ b/templates/nginx.conf.j2 @@ -19,7 +19,7 @@ upstream funkwhale-api { {% if config.proxy_frontend %} upstream funkwhale-front { - server ${FUNKWHALE_FRONT_IP}:${FUNKWHALE_FRONT_PORT}; + server ${FUNKWHALE_FRONT_HOST}:${FUNKWHALE_FRONT_PORT}; } {% endif %}