We can now work on federation locally thank to traefik

.env.dev

@@ -1,9 +1,11 @@
 API_AUTHENTICATION_REQUIRED=True
 RAVEN_ENABLED=false
 RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5
-DJANGO_ALLOWED_HOSTS=localhost,nginx
+DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx
 DJANGO_SETTINGS_MODULE=config.settings.local
 DJANGO_SECRET_KEY=dev
 C_FORCE_ROOT=true
-FUNKWHALE_URL=http://localhost
+FUNKWHALE_HOSTNAME=localhost
+FUNKWHALE_PROTOCOL=http
 PYTHONDONTWRITEBYTECODE=true
+WEBPACK_DEVSERVER_PORT=8080

README.rst

@@ -206,3 +206,70 @@ Typical workflow for a merge request
 6. Push your branch
 7. Create your merge request
 8. Take a step back and enjoy, we're really grateful you did all of this and took the time to contribute!
+
+
+Working with federation locally
+-------------------------------
+
+To achieve that, you'll need:
+
+1. to update your dns resolver to resolve all your .dev hostnames locally
+2. a reverse proxy (such as traefik) to catch those .dev requests and
+   and with https certificate
+3. two instances (or more) running locally, following the regular dev setup
+
+Resolve .dev names locally
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you use dnsmasq, this is as simple as doing::
+
+    echo "address=/test/172.17.0.1" | sudo tee /etc/dnsmasq.d/test.conf
+    sudo systemctl restart dnsmasq
+
+If you use NetworkManager with dnsmasq integration, use this instead::
+
+    echo "address=/test/172.17.0.1" | sudo tee /etc/NetworkManager/dnsmasq.d/test.conf
+    sudo systemctl restart NetworkManager
+
+Add wildcard certificate to the trusted certificates
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Simply copy bundled certificates::
+
+    sudo cp docker/ssl/test.crt /usr/local/share/ca-certificates/
+    sudo update-ca-certificates
+
+This certificate is a wildcard for ``*.funkwhale.test``
+
+Run a reverse proxy for your instances
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+Crete docker network
+^^^^^^^^^^^^^^^^^^^^
+
+Create the federation network::
+
+    docker network create federation
+
+Launch everything
+^^^^^^^^^^^^^^^^^
+
+Launch the traefik proxy::
+
+    docker-compose -f docker/traefik.yml up -d
+
+Then, in separate terminals, you can setup as many different instances as you
+need::
+
+    export COMPOSE_PROJECT_NAME=node2
+    docker-compose -f dev.yml run --rm api python manage.py migrate
+    docker-compose -f dev.yml run --rm api python manage.py createsuperuser
+    docker-compose -f dev.yml up nginx api front
+
+Note that by default, if you don't export the COMPOSE_PROJECT_NAME,
+we will default to node1 as the name of your instance.
+
+Assuming your project name is ``node1``, your server will be reachable
+at ``https://node1.funkwhale.test/``. Not that you'll have to trust
+the SSL Certificate as it's self signed.

api/config/settings/common.py

@@ -25,8 +25,26 @@ try:
 except FileNotFoundError:
     pass
 
-FUNKWHALE_URL = env('FUNKWHALE_URL')
+FUNKWHALE_HOSTNAME = None
-FUNKWHALE_HOSTNAME = urlsplit(FUNKWHALE_URL).netloc
+FUNKWHALE_HOSTNAME_SUFFIX = env('FUNKWHALE_HOSTNAME_SUFFIX', default=None)
+FUNKWHALE_HOSTNAME_PREFIX = env('FUNKWHALE_HOSTNAME_PREFIX', default=None)
+if FUNKWHALE_HOSTNAME_PREFIX and FUNKWHALE_HOSTNAME_SUFFIX:
+    # We're in traefik case, in development
+    FUNKWHALE_HOSTNAME = '{}.{}'.format(
+        FUNKWHALE_HOSTNAME_PREFIX, FUNKWHALE_HOSTNAME_SUFFIX)
+    FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https')
+else:
+    try:
+        FUNKWHALE_HOSTNAME = env('FUNKWHALE_HOSTNAME')
+        FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https')
+    except Exception:
+        FUNKWHALE_URL = env('FUNKWHALE_URL')
+        _parsed = urlsplit(FUNKWHALE_URL)
+        FUNKWHALE_HOSTNAME = _parsed.netloc
+        FUNKWHALE_PROTOCOL = _parsed.scheme
+
+FUNKWHALE_URL = '{}://{}'.format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME)
+
 
 FEDERATION_ENABLED = env.bool('FEDERATION_ENABLED', default=True)
 FEDERATION_HOSTNAME = env('FEDERATION_HOSTNAME', default=FUNKWHALE_HOSTNAME)
@@ -406,3 +424,8 @@ ACCOUNT_USERNAME_BLACKLIST = [
     'staff',
     'service',
 ] + env.list('ACCOUNT_USERNAME_BLACKLIST', default=[])
+
+EXTERNAL_REQUESTS_VERIFY_SSL = env.bool(
+    'EXTERNAL_REQUESTS_VERIFY_SSL',
+    default=True
+)

dev.yml

@@ -10,22 +10,39 @@ services:
       - "HOST=0.0.0.0"
       - "WEBPACK_DEVSERVER_PORT=${WEBPACK_DEVSERVER_PORT-8080}"
     ports:
-      - "${WEBPACK_DEVSERVER_PORT-8080}:${WEBPACK_DEVSERVER_PORT-8080}"
+      - "${WEBPACK_DEVSERVER_PORT_BINDING-8080:}${WEBPACK_DEVSERVER_PORT-8080}"
     volumes:
       - './front:/app'
       - './po:/po'
+    networks:
+      - federation
+      - internal
+    labels:
+      traefik.backend: "${COMPOSE_PROJECT_NAME-node1}"
+      traefik.frontend.rule: "Host: ${COMPOSE_PROJECT_NAME-node1}.funkwhale.test"
+      traefik.enable: 'true'
+      traefik.federation.protocol: 'http'
+      traefik.federation.port: "${WEBPACK_DEVSERVER_PORT-8080}"
 
   postgres:
     env_file:
       - .env.dev
       - .env
     image: postgres
+    volumes:
+      - "./data/${COMPOSE_PROJECT_NAME-node1}/postgres:/var/lib/postgresql/data"
+    networks:
+      - internal
 
   redis:
     env_file:
       - .env.dev
       - .env
     image: redis:3.0
+    volumes:
+      - "./data/${COMPOSE_PROJECT_NAME-node1}/redis:/data"
+    networks:
+      - internal
 
   celeryworker:
     env_file:
@@ -39,11 +56,17 @@ services:
      - redis
     command: celery -A funkwhale_api.taskapp worker -l debug
     environment:
+      - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}"
+      - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test"
+      - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}"
+      - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL-http}"
       - "DATABASE_URL=postgresql://postgres@postgres/postgres"
       - "CACHE_URL=redis://redis:6379/0"
     volumes:
       - ./api:/app
       - ./data/music:/music
+    networks:
+      - internal
   api:
     env_file:
       - .env.dev
@@ -56,12 +79,17 @@ services:
       - ./api:/app
       - ./data/music:/music
     environment:
+      - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}"
+      - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test"
+      - "FUNKWHALE_HOSTNAME_PREFIX=${COMPOSE_PROJECT_NAME}"
+      - "FUNKWHALE_PROTOCOL=${FUNKWHALE_PROTOCOL-http}"
       - "DATABASE_URL=postgresql://postgres@postgres/postgres"
       - "CACHE_URL=redis://redis:6379/0"
     links:
       - postgres
       - redis
-
+    networks:
+      - internal
   nginx:
     command: /entrypoint.sh
     env_file:
@@ -70,6 +98,8 @@ services:
     image: nginx
     environment:
       - "WEBPACK_DEVSERVER_PORT=${WEBPACK_DEVSERVER_PORT-8080}"
+      - "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME- }"
+      - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}"
     links:
       - api
       - front
@@ -79,8 +109,9 @@ services:
       - ./deploy/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf.template:ro
       - ./api/funkwhale_api/media:/protected/media
     ports:
-      - "0.0.0.0:6001:6001"
+      - "6001"
-
+    networks:
+      - internal
   docs:
     build: docs
     command: python serve.py
@@ -89,3 +120,9 @@ services:
     ports:
       - '35730:35730'
       - '8001:8001'
+
+networks:
+  internal:
+  federation:
+    external:
+      name: federation

docker/nginx/entrypoint.sh

@@ -1,10 +1,17 @@
 #!/bin/bash -eux
-FIRST_HOST=$(echo ${DJANGO_ALLOWED_HOSTS} | cut -d, -f1)
+
+FORWARDED_PORT="$WEBPACK_DEVSERVER_PORT"
+COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME// /}"
+if [ -n "$COMPOSE_PROJECT_NAME" ]; then
+  echo
+    FUNKWHALE_HOSTNAME="$COMPOSE_PROJECT_NAME.funkwhale.test"
+    FORWARDED_PORT="443"
+fi
 echo "Copying template file..."
 cp /etc/nginx/funkwhale_proxy.conf{.template,}
-sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host ${FIRST_HOST}:${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf
+sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host ${FUNKWHALE_HOSTNAME}:${FORWARDED_PORT}/" /etc/nginx/funkwhale_proxy.conf
-sed -i "s/proxy_set_header Host \$host/proxy_set_header Host ${FIRST_HOST}/" /etc/nginx/funkwhale_proxy.conf
+sed -i "s/proxy_set_header Host \$host/proxy_set_header Host ${FUNKWHALE_HOSTNAME}/" /etc/nginx/funkwhale_proxy.conf
-sed -i "s/proxy_set_header X-Forwarded-Port \$server_port/proxy_set_header X-Forwarded-Port ${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf
+sed -i "s/proxy_set_header X-Forwarded-Port \$server_port/proxy_set_header X-Forwarded-Port ${FORWARDED_PORT}/" /etc/nginx/funkwhale_proxy.conf
 
 cat /etc/nginx/funkwhale_proxy.conf
 nginx -g "daemon off;"

docker/ssl/test.crt

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDljCCAn6gAwIBAgIJAOA/w9NwL3aMMA0GCSqGSIb3DQEBCwUAMGAxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQxGTAXBgNVBAMMECouZnVua3doYWxlLnRlc3QwHhcNMTgw
+NDA4MTMwNDAzWhcNMjgwNDA1MTMwNDAzWjBgMQswCQYDVQQGEwJBVTETMBEGA1UE
+CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk
+MRkwFwYDVQQDDBAqLmZ1bmt3aGFsZS50ZXN0MIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAyGqRLEMFs1mpRwauTicIRj2zwBUe6JMNRbIvOUkaj2KY6avA
+7tiNti/ygBoTyJl2JK3mmLqxElqedpMhjVvYde/PyjXoZ+0Vq4FWv89LV6ZM/Scf
+TCIYwWF1ppi6GYFmU3WCIMISkKiPBtMArB0oZxiUWLmkyd8jih2wnQOpkQ20FfG0
+CtlrKlQKyAe7X3zPuqGfaMUN7J4w9g3/SC66YulbAtI1/Z4tuG8J4m2RC6jH1hVy
+364l3ifEC+m9Kax/ystfu/mkLdyQgRfOZTNf2JhS3BL8zpoWMXFK+4+7TYisrV1h
+0pzIAsoQeBB+cFOOFEwRAv0FxSWnZ+/shjnwbwIDAQABo1MwUTAdBgNVHQ4EFgQU
+sULmofttRyWUMM93IsD8jBvyCd4wHwYDVR0jBBgwFoAUsULmofttRyWUMM93IsD8
+jBvyCd4wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAUg/fiXut
+hW6fDx9f0JdB4uLiLnv8tDP35ackLLapFJhXtflIXcqCzxStQ46nMs1wjaZPb+ws
+pLULzvTKTxJbu+JYc2nvis4m2oSFczJ3S9tgug4Ppv8yS7N1pp7kfjOvBjgh6sYW
+p+Ctb5r8qvgvT9yDTeCnsqktb/OkRHlHwhRYfnuxh+96s4mzifqFUP4uCCcFYPTc
+RE0Ag3oI5sHOdDk/cdYE5PGQPjSP6gzn0lsrz1Q3x1C8+txSHzsJnvS3Ost+dwcy
+JSjDBXauy9cZv93Voevcl16Ioo7trtkp4dwAoep52vOT/KMkJ4zm19msV3BP4wMa
+BUqrV2F7twD5zw==
+-----END CERTIFICATE-----

docker/ssl/test.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDIapEsQwWzWalH
+Bq5OJwhGPbPAFR7okw1Fsi85SRqPYpjpq8Du2I22L/KAGhPImXYkreaYurESWp52
+kyGNW9h178/KNehn7RWrgVa/z0tXpkz9Jx9MIhjBYXWmmLoZgWZTdYIgwhKQqI8G
+0wCsHShnGJRYuaTJ3yOKHbCdA6mRDbQV8bQK2WsqVArIB7tffM+6oZ9oxQ3snjD2
+Df9ILrpi6VsC0jX9ni24bwnibZELqMfWFXLfriXeJ8QL6b0prH/Ky1+7+aQt3JCB
+F85lM1/YmFLcEvzOmhYxcUr7j7tNiKytXWHSnMgCyhB4EH5wU44UTBEC/QXFJadn
+7+yGOfBvAgMBAAECggEBAMVB3lEqRloYTbxSnwzc7g/0ew77usg+tDl8/23qvfGS
+od6b5fEvw4sl9hCPmhk+skG3x9dbKR1fg8hBWCzB0XOC7YmhNXXUrBd53eA8L3O9
+gtlHwE424Ra0zg+DEug3rHdImSOU4KDwxpV46Jh+ul1+m8QYNFFdBqXSQxrHmAXj
+MQ6++rjoJ+bhucmjBouzMYXHTGhdae3kjDFrFJ4cUsH6F03NcDwS+AmZxa/DWQ/H
+SoBQBeLoE6I1aKhLgY91yO1e7CtSzS2GFCODReN4b3cylaR7jE7Mg87TZcga6Wfa
+Xcd120VVlVq6HmZc/Xob7aUim3AuY2er8bcvmg1XOsECgYEA5EMM5UlpLdNWv1hp
+5IMvkeCbXtLJ3IOHO0xLkFdx0CxaR9TyAAqIrSh1t9rFhYqLUNiOdMc2TqrvdgEU
+B/QZrAevWRc5sjPvFXmYeWSCi/tjRgQh4jClWDX/TlfAlP55z2BFyMPMX6//WbBQ
+5aL9xymTymzFFcaE8EytT5Jz8rUCgYEA4MVF3IkaQepl6H1gf2T6ev+MtGk9AGg9
+DSJpio7hfMcY5X3NrTJJFF9DJFXqfo3ILOMyUpIUHqkCGKXil0n9ypLp4vq7l+6c
+m1gtKFXh7uKAV4XtSnR0nuK/N10JJp2HbbFYGlziRaa1iEPAFvLDQHu4jyf5sXyV
+HvreuQgGWRMCgYEAlUaQKWaP5UsfoPUGE04DjwfvM9zv7EkL6CimBhhZswU+aVmG
+haZd6bfa/EiTAhkvsMheqVoaVuoMvgRIgEcPfuRrtPyuW68A/O9PWpvzj+3v5zsO
+maisiPqPI0HaDNY6/PZ9zKTXhABKIvJehT7JbjTvlOL7JJl2GNxcPvyM3T0CgYEA
+tnVtUKi69+ce8qtUOhXufwoTXiBPtJTpelAE/MUfpfq46xJEc+PuDuuFxWk5AaJ2
+bHnBz+VlD76CRR/j4IvfySGZWvfOcHbyCeh6P9P3o8OaC3JcPaRrRs8qCfcsBny6
+AwGDU2MzCvdZRVQ6CmbmuOG13//DYaCQLKXZRrqM7KECgYEAxDsqtyHA/a38UhS8
+iQ8HqrZp8CuzJoJw/QILvzjojD1cvmwF73RrPEpRfEaLWVQGQ5F1IlHk/009C5zy
+eUT4ZaPxLem6khBf7pn3xXaVBGZsYoltek5sUBsu/jA+4Sw6bcUmhBRBCs98JGpR
+DVJtvOTk9aGW8M8UbgqwW+e/6ng=
+-----END PRIVATE KEY-----

docker/traefik.toml

@@ -0,0 +1,26 @@
+defaultEntryPoints = ["http", "https"]
+
+################################################################
+# Web configuration backend
+################################################################
+[web]
+address = ":8040"
+################################################################
+# Docker configuration backend
+################################################################
+[docker]
+domain = "funkwhale.test"
+watch = true
+exposedbydefault = false
+
+[entryPoints]
+  [entryPoints.http]
+  address = ":80"
+    [entryPoints.http.redirect]
+    entryPoint = "https"
+  [entryPoints.https]
+  address = ":443"
+    [entryPoints.https.tls]
+      [[entryPoints.https.tls.certificates]]
+      certFile = "/ssl/traefik.crt"
+      keyFile = "/ssl/traefik.key"

docker/traefik.yml

@@ -0,0 +1,22 @@
+version: '2.1'
+
+services:
+  traefik:
+    image: traefik:alpine
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./traefik.toml:/traefik.toml
+      - ./ssl/test.key:/ssl/traefik.key
+      - ./ssl/test.crt:/ssl/traefik.crt
+    ports:
+      - '80:80'
+      - '443:443'
+      - '8040:8040'
+    networks:
+      federation:
+
+
+networks:
+  federation:
+    external:
+      name: federation