From e40d5c88cde14b86a04a01f80a0d859125a7625d Mon Sep 17 00:00:00 2001 From: jon r Date: Thu, 5 Sep 2024 15:06:31 +0200 Subject: [PATCH] feat(compose): refactor manifests for separate use cases --- .env.dev | 2 +- .env.single-node | 2 + CHANGELOG.md | 2 +- compose.single-node.yml | 43 +++++++ compose.yml | 144 ++++++----------------- docker/api-docs.yml | 10 ++ docker/dnsmasq.yml | 10 ++ docker/minio.yml | 27 +++++ docker/traefik.yml | 10 -- docker/typesense.yml | 13 ++ docs/developer/setup/docker.md | 53 +++++++-- docs/locales/fr/LC_MESSAGES/changelog.po | 2 +- 12 files changed, 190 insertions(+), 128 deletions(-) create mode 100644 .env.single-node create mode 100644 compose.single-node.yml create mode 100644 docker/api-docs.yml create mode 100644 docker/dnsmasq.yml create mode 100644 docker/minio.yml create mode 100644 docker/typesense.yml diff --git a/.env.dev b/.env.dev index b04b394bb..008b917d4 100644 --- a/.env.dev +++ b/.env.dev @@ -1,4 +1,4 @@ -DJANGO_ALLOWED_HOSTS=localhost,nginx,0.0.0.0,127.0.0.1,.gitpod.io +DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx,0.0.0.0,127.0.0.1,.gitpod.io DJANGO_SETTINGS_MODULE=config.settings.local DJANGO_SECRET_KEY=dev C_FORCE_ROOT=true diff --git a/.env.single-node b/.env.single-node new file mode 100644 index 000000000..9378b6e2c --- /dev/null +++ b/.env.single-node @@ -0,0 +1,2 @@ +DJANGO_ALLOWED_HOSTS=localhost,nginx,0.0.0.0,127.0.0.1 +FUNKWHALE_PORT=8000 diff --git a/CHANGELOG.md b/CHANGELOG.md index b047e34d9..1c863b567 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -402,7 +402,7 @@ Update instructions: 2. Stop your containers using the **docker-compose** syntax. ```sh - sudo docker-compose down + docker compose down ``` 3. Bring the containers back up using the **docker compose** syntax. diff --git a/compose.single-node.yml b/compose.single-node.yml new file mode 100644 index 000000000..9e3530269 --- /dev/null +++ b/compose.single-node.yml @@ -0,0 +1,43 @@ +networks: + internal: + federation: + external: true + +x-django: &django + +services: + front: + ports: + - "${VUE_PORT:-8080}" + + api: + env_file: + - .env.single-node + - .env.dev + - .env + environment: + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME:-localhost}" + - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL:-http}" + - "FUNKWHALE_PORT=${FUNKWHALE_PORT:-8000}" + - "DATABASE_URL=postgresql://postgres@postgres/postgres" + - "CACHE_URL=redis://redis:6379/0" + - "STATIC_ROOT=/staticfiles" + extra_hosts: [] + + celeryworker: + env_file: + - .env.single-node + - .env.dev + - .env + environment: + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME:-localhost}" + - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL:-http}" + - "FUNKWHALE_PORT=${FUNKWHALE_PORT:-8000}" + - "DATABASE_URL=postgresql://postgres@postgres/postgres" + - "CACHE_URL=redis://redis:6379/0" + extra_hosts: [] + + nginx: + ports: + - "${NGINX_PORTS_MAPPING:-8000:80}" + labels: {} diff --git a/compose.yml b/compose.yml index f2ec3d247..df9c2e490 100644 --- a/compose.yml +++ b/compose.yml @@ -3,6 +3,40 @@ networks: federation: external: true +x-django: &django + env_file: + - .env.dev + - .env + build: + context: ./api + dockerfile: Dockerfile + args: + install_dev_deps: 1 + image: funkwhale-api + volumes: + - ./api:/app + - "${MUSIC_DIRECTORY_SERVE_PATH-./data/music}:/music:ro" + - "./data/plugins:/srv/funkwhale/plugins" + - "./data/staticfiles:/staticfiles" + - "./data/media:/data/media" + environment: + - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME:-localhost}" + - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test" + - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}" + - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL:-http}" + - "DATABASE_URL=postgresql://postgres@postgres/postgres" + - "CACHE_URL=redis://redis:6379/0" + - "STATIC_ROOT=/staticfiles" + depends_on: + - postgres + - redis + networks: + - internal + extra_hosts: + - "node1.funkwhale.test:host-gateway" + - "node2.funkwhale.test:host-gateway" + - "node3.funkwhale.test:host-gateway" + services: front: build: @@ -14,8 +48,6 @@ services: environment: - "HOST=0.0.0.0" - "VUE_PORT=${VUE_PORT:-8080}" - ports: - - "${VUE_PORT:-8080}" volumes: - "./front:/app" - "/app/node_modules" @@ -53,84 +85,22 @@ services: - internal api: - env_file: - - .env.dev - - .env - build: - context: ./api - dockerfile: Dockerfile - args: - install_dev_deps: 1 - image: funkwhale-api + <<: *django command: > bash -c "funkwhale-manage collectstatic --no-input && uvicorn --reload config.asgi:application --host 0.0.0.0 --port 5000 --reload-dir config/ --reload-dir=funkwhale_api/" - volumes: - - ./api:/app - - "${MUSIC_DIRECTORY_SERVE_PATH-./data/music}:/music:ro" - - "./data/plugins:/srv/funkwhale/plugins" - - "./data/staticfiles:/staticfiles" - - "./data/media:/data/media" - environment: - - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME:-localhost}" - - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test" - - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}" - - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL:-http}" - - "DATABASE_URL=postgresql://postgres@postgres/postgres" - - "CACHE_URL=redis://redis:6379/0" - - "STATIC_ROOT=/staticfiles" - # - FUNKWHALE_PORT=${FUNKWHALE_PORT} - - depends_on: - - postgres - # - minio - - redis - networks: - - internal cap_add: - SYS_PTRACE - extra_hosts: - - "node1.funkwhale.test:host-gateway" - - "node2.funkwhale.test:host-gateway" - - "node3.funkwhale.test:host-gateway" celeryworker: - env_file: - - .env.dev - - .env - image: funkwhale-api - depends_on: - - postgres - # - minio - - redis + <<: *django command: celery -A funkwhale_api.taskapp worker -l debug -B --concurrency=${CELERYD_CONCURRENCY-0} - environment: - - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME:-localhost}" - - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test" - - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}" - - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL:-http}" - - "DATABASE_URL=postgresql://postgres@postgres/postgres" - - "CACHE_URL=redis://redis:6379/0" - # - FUNKWHALE_PORT=${FUNKWHALE_PORT} - volumes: - - ./api:/app - - "${MUSIC_DIRECTORY_SERVE_PATH:-./data/music}:/music:ro" - - "./data/plugins:/srv/funkwhale/plugins" - - "./data/media:/data/media" - networks: - - internal - extra_hosts: - - "node1.funkwhale.test:host-gateway" - - "node2.funkwhale.test:host-gateway" - - "node3.funkwhale.test:host-gateway" nginx: env_file: - .env.dev - .env image: nginx - ports: - - "${NGINX_PORTS_MAPPING-8000:80}" environment: - "NGINX_MAX_BODY_SIZE=${NGINX_MAX_BODY_SIZE:-100M}" - "FUNKWHALE_API_HOST=${FUNKWHALE_API_HOST:-api}" @@ -155,48 +125,10 @@ services: - internal labels: - traefik.backend: "${COMPOSE_PROJECT_NAME:-node1}" - traefik.frontend.rule: "Host:${COMPOSE_PROJECT_NAME:-node1}.funkwhale.test,${NODE_IP:-127.0.0.1},${DJANGO_ALLOWED_HOSTS}" + traefik.backend: "${COMPOSE_PROJECT_NAME}" + traefik.frontend.rule: "Host:${COMPOSE_PROJECT_NAME}.funkwhale.test" traefik.enable: "true" traefik.federation.protocol: "http" traefik.federation.port: "80" traefik.frontend.passHostHeader: true traefik.docker.network: federation - - api-docs: - image: swaggerapi/swagger-ui:v3.37.2 - environment: - - "API_URL=/swagger.yml" - ports: - - "8002:8080" - volumes: - - "./docs/swagger.yml:/usr/share/nginx/html/swagger.yml" - - "./docs/api:/usr/share/nginx/html/api" - - typesense: - env_file: - - .env.dev - - .env - image: typesense/typesense:0.24.0 - networks: - - internal - volumes: - - ./typesense/data:/data - command: --data-dir /data --enable-cors - profiles: - - typesense - - # minio: - # image: minio/minio - # command: server /data - # volumes: - # - "./data/${COMPOSE_PROJECT_NAME-node1}/minio:/data" - # environment: - # - "MINIO_ACCESS_KEY=${AWS_ACCESS_KEY_ID-access_key}" - # - "MINIO_SECRET_KEY=${AWS_SECRET_ACCESS_KEY-secret_key}" - # - "MINIO_HTTP_TRACE: /dev/stdout" - # ports: - # - "9000:9000" - # networks: - # - federation - # - internal diff --git a/docker/api-docs.yml b/docker/api-docs.yml new file mode 100644 index 000000000..c0b557ef2 --- /dev/null +++ b/docker/api-docs.yml @@ -0,0 +1,10 @@ +services: + api-docs: + image: swaggerapi/swagger-ui:v3.37.2 + environment: + - "API_URL=/swagger.yml" + ports: + - "8002:8080" + volumes: + - "./docs/swagger.yml:/usr/share/nginx/html/swagger.yml" + - "./docs/api:/usr/share/nginx/html/api" diff --git a/docker/dnsmasq.yml b/docker/dnsmasq.yml new file mode 100644 index 000000000..480c8fb80 --- /dev/null +++ b/docker/dnsmasq.yml @@ -0,0 +1,10 @@ +services: + dnsmasq: + image: jamesmorrison/dnsmasq + ports: + - "172.17.0.1:53:53/udp" + - "172.17.0.1:53:53/tcp" + volumes: + - ./dnsmasq.conf:/etc/dnsmasq.conf + - ./funkwhale.test.conf:/etc/dnsmasq.d/funkwhale.test.conf + network_mode: bridge diff --git a/docker/minio.yml b/docker/minio.yml new file mode 100644 index 000000000..e1ac2b4db --- /dev/null +++ b/docker/minio.yml @@ -0,0 +1,27 @@ +services: + minio: + image: minio/minio + command: server /data + volumes: + - "./data/${COMPOSE_PROJECT_NAME-node1}/minio:/data" + environment: + - "MINIO_ACCESS_KEY=${AWS_ACCESS_KEY_ID-access_key}" + - "MINIO_SECRET_KEY=${AWS_SECRET_ACCESS_KEY-secret_key}" + - "MINIO_HTTP_TRACE: /dev/stdout" + ports: + - "9000:9000" + networks: + - federation + - internal + + api: + depends_on: + - postgres + - minio + - redis + + celeryworker: + depends_on: + - postgres + - minio + - redis diff --git a/docker/traefik.yml b/docker/traefik.yml index dcd7aa623..5c35d71de 100644 --- a/docker/traefik.yml +++ b/docker/traefik.yml @@ -13,16 +13,6 @@ services: networks: - federation - dnsmasq: - image: jamesmorrison/dnsmasq - ports: - - "172.17.0.1:53:53/udp" - - "172.17.0.1:53:53/tcp" - volumes: - - ./dnsmasq.conf:/etc/dnsmasq.conf - - ./funkwhale.test.conf:/etc/dnsmasq.d/funkwhale.test.conf - network_mode: bridge - networks: federation: external: true diff --git a/docker/typesense.yml b/docker/typesense.yml new file mode 100644 index 000000000..afbf4645c --- /dev/null +++ b/docker/typesense.yml @@ -0,0 +1,13 @@ +services: + typesense: + env_file: + - .env.dev + - .env + image: typesense/typesense:0.24.0 + networks: + - internal + volumes: + - ./typesense/data:/data + command: --data-dir /data --enable-cors + profiles: + - typesense diff --git a/docs/developer/setup/docker.md b/docs/developer/setup/docker.md index e267c43a3..ffca96e73 100644 --- a/docs/developer/setup/docker.md +++ b/docs/developer/setup/docker.md @@ -94,7 +94,7 @@ You need to create some local data to mimic a production environment. Once you have set up your containers, launch all services to start working on them: ```sh -docker compose up front api nginx celeryworker +docker compose up -d ``` This gives you access to the following: @@ -103,6 +103,12 @@ This gives you access to the following: - The Funkwhale API on `http://localhost:8000/api/v1` - The Django admin interface on `http://localhost:8000/api/admin` +Please note that at this stage, login will only work if you choose to run this together with the `single-node` overlay: + +```sh +docker compose -f compose.yml -f compose.single-node.yml up -d +``` + Once you're done with the containers, you can stop them all: ```sh @@ -111,6 +117,12 @@ docker compose stop If you want to destroy your containers, run the following: +```sh +docker compose down +``` + +To also destroy the state of your containers, run: + ```sh docker compose down -v ``` @@ -119,8 +131,8 @@ docker compose down -v Working on federation features requires some additional setup. You need to do the following: -1. Update your DNS resolver to resolve all your .dev hostnames locally -2. Set up a reverse proxy (such as traefik) to catch .dev requests with a TLS certificate +1. Update your DNS resolver to resolve all your `.test` hostnames locally +2. Set up a reverse proxy (such as traefik) to catch `.test` requests with a TLS certificate 3. Set up two or more local instances To resolve hostnames locally, run the following: @@ -145,20 +157,40 @@ sudo systemctl restart NetworkManager ::: +:::{tab-item} dnsmasq in Docker + +```sh +docker compose -f docker/dnsmasq.yml up -d +sudo resolvectl domain docker0 '~funkwhale.test.' +sudo resolvectl dns docker0 172.17.0.1 +``` + +::: + :::: -To add a wildcard certificate, copy the test certificate from the `docker/ssl` folder. This certificate is a wildcard for `*.funkwhale.test` +To add a wildcard certificate, copy the test certificate from the `docker/ssl` folder to your system store. This certificate is a wildcard for `*.funkwhale.test`. + +For Debian-based systems, like Ubuntu, run: ```sh sudo cp docker/ssl/test.crt /usr/local/share/ca-certificates/ sudo update-ca-certificates ``` +For RHEL-based systems, like Fedora, run: + +```sh +sudo cp docker/ssl/test.crt /etc/pki/ca-trust/source/anchors/ +sudo update-ca-trust +``` + To run a reverse proxy for your app: 1. Add the following configuration to your `.env` file: - ```text + ```sh + cat >> .env <<< " # Remove any port binding so you can specify this per-instance VUE_PORT_BINDING= # Disable certificate validation @@ -166,7 +198,7 @@ To run a reverse proxy for your app: # Ensure all links use https FUNKWHALE_PROTOCOL=https # Disable host ports binding for the nginx container so that traefik handles everything - NGINX_PORTS_MAPPING=80 + NGINX_PORTS_MAPPING=80" ``` 2. Launch traefik using the bundled configuration: @@ -175,14 +207,17 @@ To run a reverse proxy for your app: docker compose -f docker/traefik.yml up -d ``` + Your previous instance is now reachable at https://funkwhale.funkwhale.test + 3. Set up as many different projects as you need. Make sure the `COMPOSE_PROJECT_NAME` and `VUE_PORT` variables are unique per instance ```sh export COMPOSE_PROJECT_NAME=node2 - export VUE_PORT=1234 # this has to be unique for each instance + # VUE_PORT this has to be unique for each instance + export VUE_PORT=1234 docker compose run --rm api funkwhale-manage migrate docker compose run --rm api funkwhale-manage fw users create --superuser - docker compose up nginx api front nginx api celeryworker + docker compose up -d ``` -You can access your project at `https://{COMPOSE_PROJECT_NAME}.funkwhale.test`. + You can access your project at `https://{COMPOSE_PROJECT_NAME}.funkwhale.test`. diff --git a/docs/locales/fr/LC_MESSAGES/changelog.po b/docs/locales/fr/LC_MESSAGES/changelog.po index cd3343089..341fb352e 100644 --- a/docs/locales/fr/LC_MESSAGES/changelog.po +++ b/docs/locales/fr/LC_MESSAGES/changelog.po @@ -9178,7 +9178,7 @@ msgstr "" #~ "using-the-repository>`_" #~ msgstr "" -#~ msgid "```sh sudo docker-compose down ```" +#~ msgid "```sh docker compose down ```" #~ msgstr "" #~ msgid "```sh docker compose up -d ```"