Merge branch '565-external-storages' into 'develop'

Fix #565: store media files in S3 bucket

Closes #565

See merge request funkwhale/funkwhale!735

api/config/settings/common.py

@@ -306,6 +306,19 @@ STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR("staticfiles")))
 STATIC_URL = env("STATIC_URL", default="/staticfiles/")
 DEFAULT_FILE_STORAGE = "funkwhale_api.common.storage.ASCIIFileSystemStorage"
 
+AWS_DEFAULT_ACL = None
+AWS_QUERYSTRING_AUTH = False
+
+AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID", default=None)
+
+if AWS_ACCESS_KEY_ID:
+    AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
+    AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
+    AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL", default=None)
+    AWS_LOCATION = env("AWS_LOCATION", default="")
+    DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+
 # See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
 STATICFILES_DIRS = (str(APPS_DIR.path("static")),)
 

api/funkwhale_api/federation/serializers.py

@@ -838,7 +838,7 @@ class AlbumSerializer(MusicEntitySerializer):
             d["cover"] = {
                 "type": "Link",
                 "href": utils.full_url(instance.cover.url),
-                "mediaType": mimetypes.guess_type(instance.cover.path)[0]
+                "mediaType": mimetypes.guess_type(instance.cover_path)[0]
                 or "image/jpeg",
             }
         if self.context.get("include_ap_context", self.parent is None):

api/funkwhale_api/music/models.py

@@ -346,6 +346,16 @@ class Album(APIModelMixin):
     def __str__(self):
         return self.title
 
+    @property
+    def cover_path(self):
+        if not self.cover:
+            return None
+        try:
+            return self.cover.path
+        except NotImplementedError:
+            # external storage
+            return self.cover.name
+
     @property
     def tags(self):
         t = []

api/funkwhale_api/music/views.py

@@ -240,6 +240,8 @@ def get_file_path(audio_file):
                     "MUSIC_DIRECTORY_PATH to serve in-place imported files"
                 )
             path = "/music" + audio_file.replace(prefix, "", 1)
+        if path.startswith("http://") or path.startswith("https://"):
+            return (settings.PROTECT_FILES_PATH + "/media/" + path).encode("utf-8")
         return (settings.PROTECT_FILES_PATH + path).encode("utf-8")
     if t == "apache2":
         try:

api/requirements/base.txt

@@ -69,3 +69,5 @@ aiohttp==3.5.4
 autobahn>=19.3.2
 
 django-oauth-toolkit==1.2
+django-storages==1.7.1
+boto3

changes/changelog.d/565.feature

@@ -0,0 +1 @@
+Support S3-compatible storages for media files (#565)

changes/notes.rst

@@ -79,6 +79,16 @@ or invalid, and additional debug information to share in your support requests.
 
 This information is available in all pages that list uploads, when clicking on the button next to the upload status.
 
+Support for S3-compatible storages to store media files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Storing all media files on the Funkwhale server itself may not be possible or desirable
+in all scenarios. You can now configure Funkwhale to store those files in a S3
+bucket instead.
+
+Check-out `https://docs.funkwhale.audio/admin/external-storages.html`_ if you want to use
+this feature.
+
 Prune library command
 ^^^^^^^^^^^^^^^^^^^^^
 

deploy/docker.nginx.template

@@ -57,13 +57,20 @@ server {
         alias ${MEDIA_ROOT}/;
     }
 
+    # this is an internal location that is used to serve
+    # audio files once correct permission / authentication
+    # has been checked on API side
     location /_protected/media {
-        # this is an internal location that is used to serve
-        # audio files once correct permission / authentication
-        # has been checked on API side
         internal;
         alias   ${MEDIA_ROOT};
+
     }
+    # Comment the previous location and uncomment this one if you're storing
+    # media files in a S3 bucket
+    # location ~ /_protected/media/(.+) {
+    #     internal;
+    #     proxy_pass $1;
+    # }
 
     location /_protected/music {
         # this is an internal location that is used to serve

deploy/env.prod.sample

@@ -136,3 +136,19 @@ FUNKWHALE_FRONTEND_PATH=/srv/funkwhale/front/dist
 
 # Nginx related configuration
 NGINX_MAX_BODY_SIZE=100M
+
+## External storages configuration
+# Funkwhale can store uploaded files on Amazon S3 and S3-compatible storages (such as Minio)
+# Uncomment and fill the variables below
+
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_STORAGE_BUCKET_NAME=
+# An optional bucket subdirectory were you want to store the files. This is especially useful
+# if you plan to use share the bucket with other services
+# AWS_LOCATION=
+
+# If you use a S3-compatible storage such as minio, set the following variable
+# the full URL to the storage server. Example:
+#   AWS_S3_ENDPOINT_URL=https://minio.mydomain.com
+# AWS_S3_ENDPOINT_URL=

deploy/nginx.template

@@ -109,9 +109,23 @@ server {
         # audio files once correct permission / authentication
         # has been checked on API side
         internal;
-        alias   ${MEDIA_ROOT};
     }
 
+    # this is an internal location that is used to serve
+    # audio files once correct permission / authentication
+    # has been checked on API side
+    location /_protected/media {
+        internal;
+        alias   ${MEDIA_ROOT};
+
+    }
+    # Comment the previous location and uncomment this one if you're storing
+    # media files in a S3 bucket
+    # location ~ /_protected/media/(.+) {
+    #     internal;
+    #     proxy_pass $1;
+    # }
+
     location /_protected/music {
         # this is an internal location that is used to serve
         # audio files once correct permission / authentication

dev.yml

@@ -63,6 +63,7 @@ services:
 
     depends_on:
       - postgres
+      # - minio
       - redis
     networks:
       - internal
@@ -76,6 +77,7 @@ services:
     build: *backend
     depends_on:
       - postgres
+      # - minio
       - redis
     command: celery -A funkwhale_api.taskapp worker -l debug -B
     environment:
@@ -146,6 +148,23 @@ services:
     volumes:
       - "./docs/swagger.yml:/usr/share/nginx/html/swagger.yml"
 
+  # minio:
+  #   image: minio/minio
+  #   command: server /data
+  #   volumes:
+  #     - "./data/${COMPOSE_PROJECT_NAME-node1}/minio:/data"
+  #   environment:
+  #     - "MINIO_ACCESS_KEY=${AWS_ACCESS_KEY_ID-access_key}"
+  #     - "MINIO_SECRET_KEY=${AWS_SECRET_ACCESS_KEY-secret_key}"
+  #     - "MINIO_HTTP_TRACE: /dev/stdout"
+  #   ports:
+  #     - "9000:9000"
+  #   networks:
+  #     - federation
+  #     - internal
+
+
+
 networks:
   ? internal
   federation:

docker/nginx/conf.dev

@@ -93,13 +93,22 @@ http {
             alias /protected/media/;
         }
 
+        # this is an internal location that is used to serve
+        # audio files once correct permission / authentication
+        # has been checked on API side
         location /_protected/media {
-            # this is an internal location that is used to serve
-            # audio files once correct permission / authentication
-            # has been checked on API side
             internal;
             alias   /protected/media;
+
         }
+        # Comment the previous location and uncomment this one if you're storing
+        # media files in a S3 bucket
+        # location ~ /_protected/media/(.+) {
+        #     internal;
+        #     resolver 127.0.0.11;
+        #     proxy_pass $1;
+        #     add_header X-Remote-URL "$1";
+        # }
 
         location /_protected/music {
             # this is an internal location that is used to serve

docs/admin/external-storages.rst

@@ -0,0 +1,92 @@
+Using external storages to store Funkwhale content
+==================================================
+
+By default, Funkwhale will store user-uploaded and related media such as audio files,
+transcoded files, avatars and album covers on a server directory.
+
+However, for bigger instances or more complex deployment scenarios, you may want
+to use distributed or external storages.
+
+S3 and S3-compatible servers
+----------------------------
+
+.. note::
+
+    This feature was released in Funkwhale 0.19 and is still considered experimental.
+    Please let us know if you see anything unusual while using it.
+
+Funkwhale supports storing media files Amazon S3 and compatible implementations such as Minio or Wasabi.
+
+In this scenario, the content itself is stored in the S3 bucket. Non-sensitive media such as
+album covers or user avatars are served directly from the bucket. However, audio files
+are still served by the reverse proxy, to enforce proper authentication.
+
+To enable S3 on Funkwhale, add the following environment variables::
+
+    AWS_ACCESS_KEY_ID=
+    AWS_SECRET_ACCESS_KEY=
+    AWS_STORAGE_BUCKET_NAME=
+    # An optional bucket subdirectory were you want to store the files. This is especially useful
+    # if you plan to use share the bucket with other services
+    # AWS_LOCATION=
+
+    # If you use a S3-compatible storage such as minio, set the following variable
+    # the full URL to the storage server. Example:
+    #   AWS_S3_ENDPOINT_URL=https://minio.mydomain.com
+    # AWS_S3_ENDPOINT_URL=
+
+Then, edit your nginx configuration. On docker setups, the file is located at ``/srv/funkwhale/nginx/funkwhale.template``,
+and at ``/etc/nginx/sites-available/funkwhale.template`` on non-docker setups.
+
+Replace the ``location /_protected/media`` block with the following::
+
+    location ~ /_protected/media/(.+) {
+        internal;
+        proxy_pass $1;
+    }
+
+Then restart Funkwhale and nginx.
+
+From now on, media files will be stored on the S3 bucket you configured. If you already
+had media files before configuring the S3 bucket, you also have to move those on the bucket
+by hand (which is outside the scope of this guide).
+
+.. note::
+
+    At the moment, we do not support S3 when using Apache as a reverse proxy.
+
+
+Securing your S3 bucket
+***********************
+
+It's important to ensure your the root of your bucket doesn't list its content,
+which is the default on many S3 servers. Otherwise, anyone could find out the true
+URLs of your audio files and bypass authentication.
+
+To avoid that, you can set the following policy on your bucket::
+
+    {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+            "Action": [
+                "s3:GetObject"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::<yourbucketname>/*"
+            ],
+            "Sid": "Public"
+            }
+        ]
+    }
+
+If you are using ``awscli``, you can store this policy in a ``/tmp/policy`` file, and
+apply it using the following command::
+
+    aws s3api put-bucket-policy --bucket <yourbucketname> --policy file:///tmp/policy

docs/admin/index.rst

@@ -14,6 +14,7 @@ Setup Guides
    ../installation/index
    configuration
    importing-music
+   external-storages
 
 Administration
 --------------