From 22edaabd9854af2e86d254ef37e916751e93a06c Mon Sep 17 00:00:00 2001 From: Jee Date: Tue, 24 Apr 2018 19:47:06 +0200 Subject: [PATCH 01/47] remove a quote --- docs/installation/external_dependencies.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/external_dependencies.rst b/docs/installation/external_dependencies.rst index 6641bef00..059226979 100644 --- a/docs/installation/external_dependencies.rst +++ b/docs/installation/external_dependencies.rst @@ -49,7 +49,7 @@ for funkwhale to work properly: .. code-block:: shell - sudo -u postgres psql -c 'CREATE EXTENSION "unaccent";'' + sudo -u postgres psql -c 'CREATE EXTENSION "unaccent";' Cache setup (Redis) From a49d3b425156009baa643d000394ef2d678bf219 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Wed, 25 Apr 2018 18:50:06 +0200 Subject: [PATCH 02/47] Fixed #182: X-sendfile not working with in-place imports --- api/funkwhale_api/music/models.py | 20 -------- api/funkwhale_api/music/views.py | 18 ++++++-- api/tests/music/test_views.py | 76 ++++++++++++++++++------------- changes/changelog.d/182.bugfix | 0 4 files changed, 59 insertions(+), 55 deletions(-) create mode 100644 changes/changelog.d/182.bugfix diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index 98fc1965b..18f181e88 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -463,26 +463,6 @@ class TrackFile(models.Model): self.mimetype = utils.guess_mimetype(self.audio_file) return super().save(**kwargs) - @property - def serve_from_source_path(self): - if not self.source or not self.source.startswith('file://'): - raise ValueError('Cannot serve this file from source') - serve_path = settings.MUSIC_DIRECTORY_SERVE_PATH - prefix = settings.MUSIC_DIRECTORY_PATH - if not serve_path or not prefix: - raise ValueError( - 'You need to specify MUSIC_DIRECTORY_SERVE_PATH and ' - 'MUSIC_DIRECTORY_PATH to serve in-place imported files' - ) - file_path = self.source.replace('file://', '', 1) - parts = os.path.split(file_path.replace(prefix, '', 1)) - if parts[0] == '/': - parts = parts[1:] - return os.path.join( - serve_path, - *parts - ) - IMPORT_STATUS_CHOICES = ( ('pending', 'Pending'), diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index af063da46..cab8eb738 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -206,6 +206,8 @@ class TrackViewSet( def get_file_path(audio_file): + serve_path = settings.MUSIC_DIRECTORY_SERVE_PATH + prefix = settings.MUSIC_DIRECTORY_PATH t = settings.REVERSE_PROXY_TYPE if t == 'nginx': # we have to use the internal locations @@ -213,14 +215,24 @@ def get_file_path(audio_file): path = audio_file.url except AttributeError: # a path was given - path = '/music' + audio_file + if not serve_path or not prefix: + raise ValueError( + 'You need to specify MUSIC_DIRECTORY_SERVE_PATH and ' + 'MUSIC_DIRECTORY_PATH to serve in-place imported files' + ) + path = '/music' + audio_file.replace(prefix, '', 1) return settings.PROTECT_FILES_PATH + path if t == 'apache2': try: path = audio_file.path except AttributeError: # a path was given - path = audio_file + if not serve_path or not prefix: + raise ValueError( + 'You need to specify MUSIC_DIRECTORY_SERVE_PATH and ' + 'MUSIC_DIRECTORY_PATH to serve in-place imported files' + ) + path = audio_file.replace(prefix, serve_path, 1) return path @@ -267,7 +279,7 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet): elif audio_file: file_path = get_file_path(audio_file) elif f.source and f.source.startswith('file://'): - file_path = get_file_path(f.serve_from_source_path) + file_path = get_file_path(f.source.replace('file://', '', 1)) response = Response() filename = f.filename mapping = { diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py index 5d7589af0..7d4117f80 100644 --- a/api/tests/music/test_views.py +++ b/api/tests/music/test_views.py @@ -76,29 +76,60 @@ def test_can_serve_track_file_as_remote_library_deny_not_following( assert response.status_code == 403 -def test_serve_file_apache(factories, api_client, settings): +@pytest.mark.parametrize('proxy,serve_path,expected', [ + ('apache2', '/host/music', '/host/music/hello/world.mp3'), + ('apache2', '/app/music', '/app/music/hello/world.mp3'), + ('nginx', '/host/music', '/_protected/music/hello/world.mp3'), + ('nginx', '/app/music', '/_protected/music/hello/world.mp3'), +]) +def test_serve_file_in_place( + proxy, serve_path, expected, factories, api_client, settings): + headers = { + 'apache2': 'X-Sendfile', + 'nginx': 'X-Accel-Redirect', + } settings.PROTECT_AUDIO_FILES = False - settings.REVERSE_PROXY_TYPE = 'apache2' - tf = factories['music.TrackFile']() + settings.PROTECT_FILE_PATH = '/_protected/music' + settings.REVERSE_PROXY_TYPE = proxy + settings.MUSIC_DIRECTORY_PATH = '/app/music' + settings.MUSIC_DIRECTORY_SERVE_PATH = serve_path + tf = factories['music.TrackFile']( + in_place=True, + source='file:///app/music/hello/world.mp3' + ) response = api_client.get(tf.path) assert response.status_code == 200 - assert response['X-Sendfile'] == tf.audio_file.path + assert response[headers[proxy]] == expected -def test_serve_file_apache_in_place(factories, api_client, settings): +@pytest.mark.parametrize('proxy,serve_path,expected', [ + ('apache2', '/host/music', '/host/media/tracks/hello/world.mp3'), + # apache with container not supported yet + # ('apache2', '/app/music', '/app/music/tracks/hello/world.mp3'), + ('nginx', '/host/music', '/_protected/media/tracks/hello/world.mp3'), + ('nginx', '/app/music', '/_protected/media/tracks/hello/world.mp3'), +]) +def test_serve_file_media( + proxy, serve_path, expected, factories, api_client, settings): + headers = { + 'apache2': 'X-Sendfile', + 'nginx': 'X-Accel-Redirect', + } settings.PROTECT_AUDIO_FILES = False - settings.REVERSE_PROXY_TYPE = 'apache2' - settings.MUSIC_DIRECTORY_PATH = '/music' - settings.MUSIC_DIRECTORY_SERVE_PATH = '/host/music' - track_file = factories['music.TrackFile']( - in_place=True, - source='file:///music/test.ogg') + settings.MEDIA_ROOT = '/host/media' + settings.PROTECT_FILE_PATH = '/_protected/music' + settings.REVERSE_PROXY_TYPE = proxy + settings.MUSIC_DIRECTORY_PATH = '/app/music' + settings.MUSIC_DIRECTORY_SERVE_PATH = serve_path - response = api_client.get(track_file.path) + tf = factories['music.TrackFile']() + tf.__class__.objects.filter(pk=tf.pk).update( + audio_file='tracks/hello/world.mp3') + response = api_client.get(tf.path) assert response.status_code == 200 - assert response['X-Sendfile'] == '/host/music/test.ogg' + assert response[headers[proxy]] == expected def test_can_proxy_remote_track( @@ -118,25 +149,6 @@ def test_can_proxy_remote_track( assert library_track.audio_file.read() == b'test' -def test_can_serve_in_place_imported_file( - factories, settings, api_client, r_mock): - settings.PROTECT_AUDIO_FILES = False - settings.MUSIC_DIRECTORY_SERVE_PATH = '/host/music' - settings.MUSIC_DIRECTORY_PATH = '/music' - settings.MUSIC_DIRECTORY_PATH = '/music' - track_file = factories['music.TrackFile']( - in_place=True, - source='file:///music/test.ogg') - - response = api_client.get(track_file.path) - - assert response.status_code == 200 - assert response['X-Accel-Redirect'] == '{}{}'.format( - settings.PROTECT_FILES_PATH, - '/music/host/music/test.ogg' - ) - - def test_can_create_import_from_federation_tracks( factories, superuser_api_client, mocker): lts = factories['federation.LibraryTrack'].create_batch(size=5) diff --git a/changes/changelog.d/182.bugfix b/changes/changelog.d/182.bugfix new file mode 100644 index 000000000..e69de29bb From 3e233cbb9859defac6ff2d34c28f9c974fda020b Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 26 Apr 2018 12:02:05 +0200 Subject: [PATCH 03/47] Fix #180: Added a documentation area for third-party projects --- changes/changelog.d/180.doc | 1 + docs/index.rst | 1 + docs/third-party.rst | 17 +++++++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 changes/changelog.d/180.doc create mode 100644 docs/third-party.rst diff --git a/changes/changelog.d/180.doc b/changes/changelog.d/180.doc new file mode 100644 index 000000000..ee79f3e3f --- /dev/null +++ b/changes/changelog.d/180.doc @@ -0,0 +1 @@ +Added a documentation area for third-party projects (#180) diff --git a/docs/index.rst b/docs/index.rst index 82dcf8c88..a48b8353c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ Funkwhale is a self-hosted, modern free and open-source music server, heavily in importing-music federation upgrading + third-party changelog Indices and tables diff --git a/docs/third-party.rst b/docs/third-party.rst new file mode 100644 index 000000000..0335f8c71 --- /dev/null +++ b/docs/third-party.rst @@ -0,0 +1,17 @@ +Third party projects +==================== + +This page lists all known projects that are maintained by third-parties +and integrate or relates to Funkwhale. + +.. note:: + + If you want your project to be added or removed from this page, + please open an issue on our issue tracker. + + +API Clients +----------- + +- `libfunkwhale `_: a Funkwhale API written in Vala +- `Funkwhale-javalib `_: a Funkwhale API client written in Java From 7c13875b64e22ab08c4fa8bc9d5e4db088a89d41 Mon Sep 17 00:00:00 2001 From: Hazmo Date: Thu, 26 Apr 2018 12:32:21 +0200 Subject: [PATCH 04/47] First version of Apache2 conf (transcoding, auth and ws missing) --- deploy/apache.conf | 126 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 deploy/apache.conf diff --git a/deploy/apache.conf b/deploy/apache.conf new file mode 100644 index 000000000..a9f427fac --- /dev/null +++ b/deploy/apache.conf @@ -0,0 +1,126 @@ +# Following variables should be modified according to your setup +Define funkwhale-api http://192.168.1.199:5000 +Define funkwhale-api-ws ws://192.168.1.199:5000 +Define funkwhale-sn funkwhale.duckdns.org +Define MUSIC_DIRECTORY_PATH /music/directory/path + + +# HTTP request redirected to HTTPS + + ServerName ${funkwhale-sn} + + # Default is to force https + RewriteEngine on + RewriteCond %{SERVER_NAME} =${funkwhale-sn} + RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [END,QSA,R=permanent] + + + Options None + Require all granted + + + + + + + + ServerName ${funkwhale-sn} + + # Path to ErrorLog and access log + ErrorLog ${APACHE_LOG_DIR}/funkwhale/error.log + CustomLog ${APACHE_LOG_DIR}/funkwhale/access.log combined + + # TLS + # Feel free to use your own configuration for SSL here or simply remove the + # lines and move the configuration to the previous server block if you + # don't want to run funkwhale behind https (this is not recommanded) + # have a look here for let's encrypt configuration: + # https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx + SSLEngine on + SSLProxyEngine On + SSLCertificateFile /etc/letsencrypt/live/${funkwhale-sn}/fullchain.pem + SSLCertificateKeyFile /etc/letsencrypt/live/${funkwhale-sn}/privkey.pem + Include /etc/letsencrypt/options-ssl-apache.conf + + + DocumentRoot /srv/funkwhale/front/dist + + FallbackResource /index.html + + # Configure Proxy settings + # ProxyPreserveHost pass the original Host header to the backend server + ProxyVia On + ProxyPreserveHost On + + RemoteIPHeader X-Forwarded-For + + + # Turning ProxyRequests on and allowing proxying from all may allow + # spammers to use your proxy to send email. + ProxyRequests Off + + + AddDefaultCharset off + Order Allow,Deny + Allow from all + # Here you can set a password using htpasswd to protect your proxy server + #Authtype Basic + #Authname "Password Required" + #AuthUserFile /etc/apache2/.htpasswd + #Require valid-user + + + # Activating WebSockets (not working) + ProxyPass "/api/v1/instance/activity" "ws://192.168.1.199:5000/api/v1/instance/activity" + + + # similar to nginx 'client_max_body_size 30M;' + LimitRequestBody 31457280 + + ProxyPass ${funkwhale-api}/api + ProxyPassReverse ${funkwhale-api}/api + + + ProxyPass ${funkwhale-api}/federation + ProxyPassReverse ${funkwhale-api}/federation + + + + ProxyPass ${funkwhale-api}/.well-known/webfinger + ProxyPassReverse ${funkwhale-api}/.well-known/webfinger + + + Alias /media /srv/funkwhale/data/media + + # Following alias is bypassing the auth check done in nginx + Alias /_protected/media /srv/funkwhale/data/media + + Alias /staticfiles /srv/funkwhale/data/static + + # Setting appropriate access levels to serve frontend + + Options FollowSymLinks + AllowOverride None + Require all granted + + + + Options FollowSymLinks + AllowOverride None + Require all granted + + + # XSendFile is serving audio files + # WARNING : permissions on paths specified below overrides previous definition, + # everything under those paths is potentially exposed. + # Following directive may be needed to ensure xsendfile is loaded + #LoadModule xsendfile_module modules/mod_xsendfile.so + + XSendFile On + XSendFilePath /srv/funkwhale/data/media + XSendFilePath ${MUSIC_DIRECTORY_PATH} + SetEnv MOD_X_SENDFILE_ENABLED 1 + + + + From 2dc70dcd25bdcfd4730168ba95cbb38b2b721517 Mon Sep 17 00:00:00 2001 From: Hazmo Date: Thu, 26 Apr 2018 13:14:44 +0200 Subject: [PATCH 05/47] Minor fixes on debian doc --- docs/installation/debian.rst | 2 +- docs/installation/external_dependencies.rst | 2 +- docs/installation/index.rst | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/installation/debian.rst b/docs/installation/debian.rst index c4e54218d..eb0c3f0ea 100644 --- a/docs/installation/debian.rst +++ b/docs/installation/debian.rst @@ -31,7 +31,7 @@ Layout All funkwhale-related files will be located under ``/srv/funkwhale`` apart from database files and a few configuration files. We will also have a -dedicated ``funwhale`` user to launch the processes we need and own those files. +dedicated ``funkwhale`` user to launch the processes we need and own those files. You are free to use different values here, just remember to adapt those in the next steps. diff --git a/docs/installation/external_dependencies.rst b/docs/installation/external_dependencies.rst index 059226979..52f9f92c3 100644 --- a/docs/installation/external_dependencies.rst +++ b/docs/installation/external_dependencies.rst @@ -18,7 +18,7 @@ On debian-like systems, you would install the database server like this: .. code-block:: shell - sudo apt-get install postgresql + sudo apt-get install postgresql postgresql-contrib The remaining steps are heavily inspired from `this Digital Ocean guide `_. diff --git a/docs/installation/index.rst b/docs/installation/index.rst index 776c22424..c2a70421b 100644 --- a/docs/installation/index.rst +++ b/docs/installation/index.rst @@ -103,7 +103,8 @@ Then, download our sample virtualhost file and proxy conf: .. parsed-literal:: curl -L -o /etc/nginx/funkwhale_proxy.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale_proxy.conf" - curl -L -o /etc/nginx/sites-enabled/funkwhale.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/nginx.conf" + curl -L -o /etc/nginx/sites-available/funkwhale.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/nginx.conf" + ln -s /etc/nginx/sites-available/funkwhale.conf /etc/nginx/sites-enabled/ Ensure static assets and proxy pass match your configuration, and check the configuration is valid with ``nginx -t``. If everything is fine, you can restart your nginx server with ``service nginx restart``. From 472e9f7605d9aef9275998a049bf8e70bdc9bb4a Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 26 Apr 2018 14:26:01 +0200 Subject: [PATCH 06/47] Added q filter on artists --- api/funkwhale_api/music/filters.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/funkwhale_api/music/filters.py b/api/funkwhale_api/music/filters.py index 752422e75..6da9cca63 100644 --- a/api/funkwhale_api/music/filters.py +++ b/api/funkwhale_api/music/filters.py @@ -20,6 +20,9 @@ class ListenableMixin(filters.FilterSet): class ArtistFilter(ListenableMixin): + q = fields.SearchFilter(search_fields=[ + 'name', + ]) class Meta: model = models.Artist From 2477aa31f9a7a89313a5bf8b649d40b67240e21e Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 26 Apr 2018 14:26:11 +0200 Subject: [PATCH 07/47] Initial swagger setup --- api/docs/swagger.yml | 825 +++++++++++++++++++++++++++++++++++++++++++ dev.yml | 9 + 2 files changed, 834 insertions(+) create mode 100644 api/docs/swagger.yml diff --git a/api/docs/swagger.yml b/api/docs/swagger.yml new file mode 100644 index 000000000..160ef56b8 --- /dev/null +++ b/api/docs/swagger.yml @@ -0,0 +1,825 @@ +swagger: "2.0" +info: + description: "Documentation for [Funkwhale](https://funkwhale.audio) API. The API is **not** stable yet." + version: "1.0.0" + title: "Funkwhale API" +host: "demo.funkwhale.audio" +basePath: "/api/v1" +tags: +- name: "artists" + description: "Artists data" +- name: "store" + description: "Access to Petstore orders" +- name: "user" + description: "Operations about user" + +schemes: +- "http" +paths: + /artists/: + get: + tags: + - "artists" + parameters: + - name: "q" + in: "query" + description: "Search query used to filter artists" + required: false + type: "string" + example: "carpenter" + - name: "listenable" + in: "query" + description: "Filter/exclude artists with listenable tracks" + required: false + type: "boolean" + responses: + 200: + schema: + type: "object" + properties: + count: + $ref: "#/properties/resultsCount" + results: + type: "array" + items: + $ref: "#/definitions/ArtistNested" + /pet: + post: + tags: + - "pet" + summary: "Add a new pet to the store" + description: "" + operationId: "addPet" + consumes: + - "application/json" + - "application/xml" + produces: + - "application/xml" + - "application/json" + parameters: + - in: "body" + name: "body" + description: "Pet object that needs to be added to the store" + required: true + schema: + $ref: "#/definitions/Pet" + responses: + 405: + description: "Invalid input" + security: + - petstore_auth: + - "write:pets" + - "read:pets" + put: + tags: + - "pet" + summary: "Update an existing pet" + description: "" + operationId: "updatePet" + consumes: + - "application/json" + - "application/xml" + produces: + - "application/xml" + - "application/json" + parameters: + - in: "body" + name: "body" + description: "Pet object that needs to be added to the store" + required: true + schema: + $ref: "#/definitions/Pet" + responses: + 400: + description: "Invalid ID supplied" + 404: + description: "Pet not found" + 405: + description: "Validation exception" + security: + - petstore_auth: + - "write:pets" + - "read:pets" + /pet/findByStatus: + get: + tags: + - "pet" + summary: "Finds Pets by status" + description: "Multiple status values can be provided with comma separated strings" + operationId: "findPetsByStatus" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "status" + in: "query" + description: "Status values that need to be considered for filter" + required: true + type: "array" + items: + type: "string" + enum: + - "available" + - "pending" + - "sold" + default: "available" + collectionFormat: "multi" + responses: + 200: + description: "successful operation" + schema: + type: "array" + items: + $ref: "#/definitions/Pet" + 400: + description: "Invalid status value" + security: + - petstore_auth: + - "write:pets" + - "read:pets" + /pet/findByTags: + get: + tags: + - "pet" + summary: "Finds Pets by tags" + description: "Muliple tags can be provided with comma separated strings. Use tag1, tag2, tag3 for testing." + operationId: "findPetsByTags" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "tags" + in: "query" + description: "Tags to filter by" + required: true + type: "array" + items: + type: "string" + collectionFormat: "multi" + responses: + 200: + description: "successful operation" + schema: + type: "array" + items: + $ref: "#/definitions/Pet" + 400: + description: "Invalid tag value" + security: + - petstore_auth: + - "write:pets" + - "read:pets" + deprecated: true + /pet/{petId}: + get: + tags: + - "pet" + summary: "Find pet by ID" + description: "Returns a single pet" + operationId: "getPetById" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "petId" + in: "path" + description: "ID of pet to return" + required: true + type: "integer" + format: "int64" + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/Pet" + 400: + description: "Invalid ID supplied" + 404: + description: "Pet not found" + security: + - api_key: [] + post: + tags: + - "pet" + summary: "Updates a pet in the store with form data" + description: "" + operationId: "updatePetWithForm" + consumes: + - "application/x-www-form-urlencoded" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "petId" + in: "path" + description: "ID of pet that needs to be updated" + required: true + type: "integer" + format: "int64" + - name: "name" + in: "formData" + description: "Updated name of the pet" + required: false + type: "string" + - name: "status" + in: "formData" + description: "Updated status of the pet" + required: false + type: "string" + responses: + 405: + description: "Invalid input" + security: + - petstore_auth: + - "write:pets" + - "read:pets" + delete: + tags: + - "pet" + summary: "Deletes a pet" + description: "" + operationId: "deletePet" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "api_key" + in: "header" + required: false + type: "string" + - name: "petId" + in: "path" + description: "Pet id to delete" + required: true + type: "integer" + format: "int64" + responses: + 400: + description: "Invalid ID supplied" + 404: + description: "Pet not found" + security: + - petstore_auth: + - "write:pets" + - "read:pets" + /pet/{petId}/uploadImage: + post: + tags: + - "pet" + summary: "uploads an image" + description: "" + operationId: "uploadFile" + consumes: + - "multipart/form-data" + produces: + - "application/json" + parameters: + - name: "petId" + in: "path" + description: "ID of pet to update" + required: true + type: "integer" + format: "int64" + - name: "additionalMetadata" + in: "formData" + description: "Additional data to pass to server" + required: false + type: "string" + - name: "file" + in: "formData" + description: "file to upload" + required: false + type: "file" + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/ApiResponse" + security: + - petstore_auth: + - "write:pets" + - "read:pets" + /store/inventory: + get: + tags: + - "store" + summary: "Returns pet inventories by status" + description: "Returns a map of status codes to quantities" + operationId: "getInventory" + produces: + - "application/json" + parameters: [] + responses: + 200: + description: "successful operation" + schema: + type: "object" + additionalProperties: + type: "integer" + format: "int32" + security: + - api_key: [] + /store/order: + post: + tags: + - "store" + summary: "Place an order for a pet" + description: "" + operationId: "placeOrder" + produces: + - "application/xml" + - "application/json" + parameters: + - in: "body" + name: "body" + description: "order placed for purchasing the pet" + required: true + schema: + $ref: "#/definitions/Order" + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/Order" + 400: + description: "Invalid Order" + /store/order/{orderId}: + get: + tags: + - "store" + summary: "Find purchase order by ID" + description: "For valid response try integer IDs with value >= 1 and <= 10. Other values will generated exceptions" + operationId: "getOrderById" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "orderId" + in: "path" + description: "ID of pet that needs to be fetched" + required: true + type: "integer" + maximum: 10.0 + minimum: 1.0 + format: "int64" + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/Order" + 400: + description: "Invalid ID supplied" + 404: + description: "Order not found" + delete: + tags: + - "store" + summary: "Delete purchase order by ID" + description: "For valid response try integer IDs with positive integer value. Negative or non-integer values will generate API errors" + operationId: "deleteOrder" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "orderId" + in: "path" + description: "ID of the order that needs to be deleted" + required: true + type: "integer" + minimum: 1.0 + format: "int64" + responses: + 400: + description: "Invalid ID supplied" + 404: + description: "Order not found" + /user: + post: + tags: + - "user" + summary: "Create user" + description: "This can only be done by the logged in user." + operationId: "createUser" + produces: + - "application/xml" + - "application/json" + parameters: + - in: "body" + name: "body" + description: "Created user object" + required: true + schema: + $ref: "#/definitions/User" + responses: + default: + description: "successful operation" + /user/createWithArray: + post: + tags: + - "user" + summary: "Creates list of users with given input array" + description: "" + operationId: "createUsersWithArrayInput" + produces: + - "application/xml" + - "application/json" + parameters: + - in: "body" + name: "body" + description: "List of user object" + required: true + schema: + type: "array" + items: + $ref: "#/definitions/User" + responses: + default: + description: "successful operation" + /user/createWithList: + post: + tags: + - "user" + summary: "Creates list of users with given input array" + description: "" + operationId: "createUsersWithListInput" + produces: + - "application/xml" + - "application/json" + parameters: + - in: "body" + name: "body" + description: "List of user object" + required: true + schema: + type: "array" + items: + $ref: "#/definitions/User" + responses: + default: + description: "successful operation" + /user/login: + get: + tags: + - "user" + summary: "Logs user into the system" + description: "" + operationId: "loginUser" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "username" + in: "query" + description: "The user name for login" + required: true + type: "string" + - name: "password" + in: "query" + description: "The password for login in clear text" + required: true + type: "string" + responses: + 200: + description: "successful operation" + schema: + type: "string" + headers: + X-Rate-Limit: + type: "integer" + format: "int32" + description: "calls per hour allowed by the user" + X-Expires-After: + type: "string" + format: "date-time" + description: "date in UTC when token expires" + 400: + description: "Invalid username/password supplied" + /user/logout: + get: + tags: + - "user" + summary: "Logs out current logged in user session" + description: "" + operationId: "logoutUser" + produces: + - "application/xml" + - "application/json" + parameters: [] + responses: + default: + description: "successful operation" + /user/{username}: + get: + tags: + - "user" + summary: "Get user by user name" + description: "" + operationId: "getUserByName" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "username" + in: "path" + description: "The name that needs to be fetched. Use user1 for testing. " + required: true + type: "string" + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/User" + 400: + description: "Invalid username supplied" + 404: + description: "User not found" + put: + tags: + - "user" + summary: "Updated user" + description: "This can only be done by the logged in user." + operationId: "updateUser" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "username" + in: "path" + description: "name that need to be updated" + required: true + type: "string" + - in: "body" + name: "body" + description: "Updated user object" + required: true + schema: + $ref: "#/definitions/User" + responses: + 400: + description: "Invalid user supplied" + 404: + description: "User not found" + delete: + tags: + - "user" + summary: "Delete user" + description: "This can only be done by the logged in user." + operationId: "deleteUser" + produces: + - "application/xml" + - "application/json" + parameters: + - name: "username" + in: "path" + description: "The name that needs to be deleted" + required: true + type: "string" + responses: + 400: + description: "Invalid username supplied" + 404: + description: "User not found" +securityDefinitions: + components: + securitySchemes: + bearerAuth: # arbitrary name for the security scheme + type: http + scheme: bearer + bearerFormat: JWT # optional, arbitrary value for documentation purposes + petstore_auth: + type: "oauth2" + authorizationUrl: "http://petstore.swagger.io/oauth/dialog" + flow: "implicit" + scopes: + write:pets: "modify pets in your account" + read:pets: "read your pets" + api_key: + type: "apiKey" + name: "api_key" + in: "header" +properties: + resultsCount: + type: "integer" + format: "int64" + description: "The total number of resources matching the request" + mbid: + type: "string" + formats: "uuid" + description: "A musicbrainz ID" +definitions: + Artist: + type: "object" + properties: + mbid: + required: false + $ref: "#/properties/mbid" + id: + type: "integer" + format: "int64" + example: 42 + name: + type: "string" + example: "System of a Down" + creation_date: + type: "string" + format: "date-time" + ArtistNested: + type: "object" + allOf: + - $ref: "#/definitions/Artist" + - type: "object" + properties: + albums: + type: "array" + items: + $ref: "#/definitions/AlbumNested" + + Album: + type: "object" + properties: + mbid: + required: false + $ref: "#/properties/mbid" + id: + type: "integer" + format: "int64" + example: 16 + artist: + type: "integer" + format: "int64" + example: 42 + title: + type: "string" + example: "Toxicity" + creation_date: + type: "string" + format: "date-time" + release_date: + type: "string" + required: false + format: "date" + example: "2001-01-01" + + AlbumNested: + type: "object" + allOf: + - $ref: "#/definitions/Album" + - type: "object" + properties: + tracks: + type: "array" + items: + $ref: "#/definitions/Track" + + Track: + type: "object" + properties: + mbid: + required: false + $ref: "#/properties/mbid" + id: + type: "integer" + format: "int64" + example: 66 + artist: + type: "integer" + format: "int64" + example: 42 + album: + type: "integer" + format: "int64" + example: 16 + title: + type: "string" + example: "Chop Suey!" + position: + required: false + description: "Position of the track in the album" + type: "number" + minimum: 1 + example: 1 + creation_date: + type: "string" + format: "date-time" + + Order: + type: "object" + properties: + id: + type: "integer" + format: "int64" + petId: + type: "integer" + format: "int64" + quantity: + type: "integer" + format: "int32" + shipDate: + type: "string" + format: "date-time" + status: + type: "string" + description: "Order Status" + enum: + - "placed" + - "approved" + - "delivered" + complete: + type: "boolean" + default: false + xml: + name: "Order" + Category: + type: "object" + properties: + id: + type: "integer" + format: "int64" + name: + type: "string" + xml: + name: "Category" + User: + type: "object" + properties: + id: + type: "integer" + format: "int64" + username: + type: "string" + firstName: + type: "string" + lastName: + type: "string" + email: + type: "string" + password: + type: "string" + phone: + type: "string" + userStatus: + type: "integer" + format: "int32" + description: "User Status" + xml: + name: "User" + Tag: + type: "object" + properties: + id: + type: "integer" + format: "int64" + name: + type: "string" + xml: + name: "Tag" + Pet: + type: "object" + required: + - "name" + - "photoUrls" + properties: + id: + type: "integer" + format: "int64" + category: + $ref: "#/definitions/Category" + name: + type: "string" + example: "doggie" + photoUrls: + type: "array" + xml: + name: "photoUrl" + wrapped: true + items: + type: "string" + tags: + type: "array" + xml: + name: "tag" + wrapped: true + items: + $ref: "#/definitions/Tag" + status: + type: "string" + description: "pet status in the store" + enum: + - "available" + - "pending" + - "sold" + xml: + name: "Pet" + ApiResponse: + type: "object" + properties: + code: + type: "integer" + format: "int32" + type: + type: "string" + message: + type: "string" +externalDocs: + description: "Find out more about Funkwhale" + url: "https://docs.funkwhale.audio" diff --git a/dev.yml b/dev.yml index 264fc9534..534d8f5b5 100644 --- a/dev.yml +++ b/dev.yml @@ -123,6 +123,15 @@ services: - '35730:35730' - '8001:8001' + api-docs: + image: swaggerapi/swagger-ui + environment: + - "API_URL=/swagger.yml" + ports: + - '8002:8080' + volumes: + - "./api/docs/swagger.yml:/usr/share/nginx/html/swagger.yml" + networks: internal: federation: From d2c2fb837e9d269305b7a8f064c7180c9ded3ff3 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 26 Apr 2018 15:17:51 +0200 Subject: [PATCH 08/47] Now support Bearer auth in complement of JWT --- api/config/settings/common.py | 1 + api/funkwhale_api/common/auth.py | 3 -- api/funkwhale_api/common/authentication.py | 37 ++++++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index de1d653cb..f1a383c58 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -377,6 +377,7 @@ REST_FRAMEWORK = { ), 'DEFAULT_AUTHENTICATION_CLASSES': ( 'funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS', + 'funkwhale_api.common.authentication.BearerTokenHeaderAuth', 'rest_framework_jwt.authentication.JSONWebTokenAuthentication', 'rest_framework.authentication.SessionAuthentication', 'rest_framework.authentication.BasicAuthentication', diff --git a/api/funkwhale_api/common/auth.py b/api/funkwhale_api/common/auth.py index 75839b936..faf13571d 100644 --- a/api/funkwhale_api/common/auth.py +++ b/api/funkwhale_api/common/auth.py @@ -29,9 +29,6 @@ class TokenHeaderAuth(BaseJSONWebTokenAuthentication): class TokenAuthMiddleware: - """ - Custom middleware (insecure) that takes user IDs from the query string. - """ def __init__(self, inner): # Store the ASGI application we were passed diff --git a/api/funkwhale_api/common/authentication.py b/api/funkwhale_api/common/authentication.py index b75f3b516..c7566eac8 100644 --- a/api/funkwhale_api/common/authentication.py +++ b/api/funkwhale_api/common/authentication.py @@ -1,3 +1,6 @@ +from django.utils.encoding import smart_text +from django.utils.translation import ugettext as _ + from rest_framework import exceptions from rest_framework_jwt import authentication from rest_framework_jwt.settings import api_settings @@ -18,3 +21,37 @@ class JSONWebTokenAuthenticationQS( def authenticate_header(self, request): return '{0} realm="{1}"'.format( api_settings.JWT_AUTH_HEADER_PREFIX, self.www_authenticate_realm) + + +class BearerTokenHeaderAuth( + authentication.BaseJSONWebTokenAuthentication): + """ + For backward compatibility purpose, we used Authorization: JWT + but Authorization: Bearer is probably better. + """ + www_authenticate_realm = 'api' + + def get_jwt_value(self, request): + auth = authentication.get_authorization_header(request).split() + auth_header_prefix = 'bearer' + + if not auth: + if api_settings.JWT_AUTH_COOKIE: + return request.COOKIES.get(api_settings.JWT_AUTH_COOKIE) + return None + + if smart_text(auth[0].lower()) != auth_header_prefix: + return None + + if len(auth) == 1: + msg = _('Invalid Authorization header. No credentials provided.') + raise exceptions.AuthenticationFailed(msg) + elif len(auth) > 2: + msg = _('Invalid Authorization header. Credentials string ' + 'should not contain spaces.') + raise exceptions.AuthenticationFailed(msg) + + return auth[1] + + def authenticate_header(self, request): + return '{0} realm="{1}"'.format('Bearer', self.www_authenticate_realm) From c4777532ebe88bf393578f2d5aeb246afced6ef9 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 26 Apr 2018 18:12:08 +0200 Subject: [PATCH 09/47] Bundle swagger docs with sphinx docs --- .gitignore | 1 + .gitlab-ci.yml | 6 +- api/docs/swagger.yml | 825 ------------------------------------------ docs/build_docs.sh | 5 + docs/build_swagger.sh | 9 + docs/swagger.yml | 186 ++++++++++ 6 files changed, 205 insertions(+), 827 deletions(-) delete mode 100644 api/docs/swagger.yml create mode 100755 docs/build_docs.sh create mode 100755 docs/build_swagger.sh create mode 100644 docs/swagger.yml diff --git a/.gitignore b/.gitignore index 548cfd7b3..25b088739 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,4 @@ data/ .env po/*.po +docs/swagger diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a0f4b9d8..5f65e60da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,12 +92,14 @@ build_front: pages: stage: test - image: python:3.6-alpine + image: python:3.6 + variables: + BUILD_PATH: "../public" before_script: - cd docs script: - pip install sphinx - - python -m sphinx . ../public + - ./build_docs.sh artifacts: paths: - public diff --git a/api/docs/swagger.yml b/api/docs/swagger.yml deleted file mode 100644 index 160ef56b8..000000000 --- a/api/docs/swagger.yml +++ /dev/null @@ -1,825 +0,0 @@ -swagger: "2.0" -info: - description: "Documentation for [Funkwhale](https://funkwhale.audio) API. The API is **not** stable yet." - version: "1.0.0" - title: "Funkwhale API" -host: "demo.funkwhale.audio" -basePath: "/api/v1" -tags: -- name: "artists" - description: "Artists data" -- name: "store" - description: "Access to Petstore orders" -- name: "user" - description: "Operations about user" - -schemes: -- "http" -paths: - /artists/: - get: - tags: - - "artists" - parameters: - - name: "q" - in: "query" - description: "Search query used to filter artists" - required: false - type: "string" - example: "carpenter" - - name: "listenable" - in: "query" - description: "Filter/exclude artists with listenable tracks" - required: false - type: "boolean" - responses: - 200: - schema: - type: "object" - properties: - count: - $ref: "#/properties/resultsCount" - results: - type: "array" - items: - $ref: "#/definitions/ArtistNested" - /pet: - post: - tags: - - "pet" - summary: "Add a new pet to the store" - description: "" - operationId: "addPet" - consumes: - - "application/json" - - "application/xml" - produces: - - "application/xml" - - "application/json" - parameters: - - in: "body" - name: "body" - description: "Pet object that needs to be added to the store" - required: true - schema: - $ref: "#/definitions/Pet" - responses: - 405: - description: "Invalid input" - security: - - petstore_auth: - - "write:pets" - - "read:pets" - put: - tags: - - "pet" - summary: "Update an existing pet" - description: "" - operationId: "updatePet" - consumes: - - "application/json" - - "application/xml" - produces: - - "application/xml" - - "application/json" - parameters: - - in: "body" - name: "body" - description: "Pet object that needs to be added to the store" - required: true - schema: - $ref: "#/definitions/Pet" - responses: - 400: - description: "Invalid ID supplied" - 404: - description: "Pet not found" - 405: - description: "Validation exception" - security: - - petstore_auth: - - "write:pets" - - "read:pets" - /pet/findByStatus: - get: - tags: - - "pet" - summary: "Finds Pets by status" - description: "Multiple status values can be provided with comma separated strings" - operationId: "findPetsByStatus" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "status" - in: "query" - description: "Status values that need to be considered for filter" - required: true - type: "array" - items: - type: "string" - enum: - - "available" - - "pending" - - "sold" - default: "available" - collectionFormat: "multi" - responses: - 200: - description: "successful operation" - schema: - type: "array" - items: - $ref: "#/definitions/Pet" - 400: - description: "Invalid status value" - security: - - petstore_auth: - - "write:pets" - - "read:pets" - /pet/findByTags: - get: - tags: - - "pet" - summary: "Finds Pets by tags" - description: "Muliple tags can be provided with comma separated strings. Use tag1, tag2, tag3 for testing." - operationId: "findPetsByTags" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "tags" - in: "query" - description: "Tags to filter by" - required: true - type: "array" - items: - type: "string" - collectionFormat: "multi" - responses: - 200: - description: "successful operation" - schema: - type: "array" - items: - $ref: "#/definitions/Pet" - 400: - description: "Invalid tag value" - security: - - petstore_auth: - - "write:pets" - - "read:pets" - deprecated: true - /pet/{petId}: - get: - tags: - - "pet" - summary: "Find pet by ID" - description: "Returns a single pet" - operationId: "getPetById" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "petId" - in: "path" - description: "ID of pet to return" - required: true - type: "integer" - format: "int64" - responses: - 200: - description: "successful operation" - schema: - $ref: "#/definitions/Pet" - 400: - description: "Invalid ID supplied" - 404: - description: "Pet not found" - security: - - api_key: [] - post: - tags: - - "pet" - summary: "Updates a pet in the store with form data" - description: "" - operationId: "updatePetWithForm" - consumes: - - "application/x-www-form-urlencoded" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "petId" - in: "path" - description: "ID of pet that needs to be updated" - required: true - type: "integer" - format: "int64" - - name: "name" - in: "formData" - description: "Updated name of the pet" - required: false - type: "string" - - name: "status" - in: "formData" - description: "Updated status of the pet" - required: false - type: "string" - responses: - 405: - description: "Invalid input" - security: - - petstore_auth: - - "write:pets" - - "read:pets" - delete: - tags: - - "pet" - summary: "Deletes a pet" - description: "" - operationId: "deletePet" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "api_key" - in: "header" - required: false - type: "string" - - name: "petId" - in: "path" - description: "Pet id to delete" - required: true - type: "integer" - format: "int64" - responses: - 400: - description: "Invalid ID supplied" - 404: - description: "Pet not found" - security: - - petstore_auth: - - "write:pets" - - "read:pets" - /pet/{petId}/uploadImage: - post: - tags: - - "pet" - summary: "uploads an image" - description: "" - operationId: "uploadFile" - consumes: - - "multipart/form-data" - produces: - - "application/json" - parameters: - - name: "petId" - in: "path" - description: "ID of pet to update" - required: true - type: "integer" - format: "int64" - - name: "additionalMetadata" - in: "formData" - description: "Additional data to pass to server" - required: false - type: "string" - - name: "file" - in: "formData" - description: "file to upload" - required: false - type: "file" - responses: - 200: - description: "successful operation" - schema: - $ref: "#/definitions/ApiResponse" - security: - - petstore_auth: - - "write:pets" - - "read:pets" - /store/inventory: - get: - tags: - - "store" - summary: "Returns pet inventories by status" - description: "Returns a map of status codes to quantities" - operationId: "getInventory" - produces: - - "application/json" - parameters: [] - responses: - 200: - description: "successful operation" - schema: - type: "object" - additionalProperties: - type: "integer" - format: "int32" - security: - - api_key: [] - /store/order: - post: - tags: - - "store" - summary: "Place an order for a pet" - description: "" - operationId: "placeOrder" - produces: - - "application/xml" - - "application/json" - parameters: - - in: "body" - name: "body" - description: "order placed for purchasing the pet" - required: true - schema: - $ref: "#/definitions/Order" - responses: - 200: - description: "successful operation" - schema: - $ref: "#/definitions/Order" - 400: - description: "Invalid Order" - /store/order/{orderId}: - get: - tags: - - "store" - summary: "Find purchase order by ID" - description: "For valid response try integer IDs with value >= 1 and <= 10. Other values will generated exceptions" - operationId: "getOrderById" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "orderId" - in: "path" - description: "ID of pet that needs to be fetched" - required: true - type: "integer" - maximum: 10.0 - minimum: 1.0 - format: "int64" - responses: - 200: - description: "successful operation" - schema: - $ref: "#/definitions/Order" - 400: - description: "Invalid ID supplied" - 404: - description: "Order not found" - delete: - tags: - - "store" - summary: "Delete purchase order by ID" - description: "For valid response try integer IDs with positive integer value. Negative or non-integer values will generate API errors" - operationId: "deleteOrder" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "orderId" - in: "path" - description: "ID of the order that needs to be deleted" - required: true - type: "integer" - minimum: 1.0 - format: "int64" - responses: - 400: - description: "Invalid ID supplied" - 404: - description: "Order not found" - /user: - post: - tags: - - "user" - summary: "Create user" - description: "This can only be done by the logged in user." - operationId: "createUser" - produces: - - "application/xml" - - "application/json" - parameters: - - in: "body" - name: "body" - description: "Created user object" - required: true - schema: - $ref: "#/definitions/User" - responses: - default: - description: "successful operation" - /user/createWithArray: - post: - tags: - - "user" - summary: "Creates list of users with given input array" - description: "" - operationId: "createUsersWithArrayInput" - produces: - - "application/xml" - - "application/json" - parameters: - - in: "body" - name: "body" - description: "List of user object" - required: true - schema: - type: "array" - items: - $ref: "#/definitions/User" - responses: - default: - description: "successful operation" - /user/createWithList: - post: - tags: - - "user" - summary: "Creates list of users with given input array" - description: "" - operationId: "createUsersWithListInput" - produces: - - "application/xml" - - "application/json" - parameters: - - in: "body" - name: "body" - description: "List of user object" - required: true - schema: - type: "array" - items: - $ref: "#/definitions/User" - responses: - default: - description: "successful operation" - /user/login: - get: - tags: - - "user" - summary: "Logs user into the system" - description: "" - operationId: "loginUser" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "username" - in: "query" - description: "The user name for login" - required: true - type: "string" - - name: "password" - in: "query" - description: "The password for login in clear text" - required: true - type: "string" - responses: - 200: - description: "successful operation" - schema: - type: "string" - headers: - X-Rate-Limit: - type: "integer" - format: "int32" - description: "calls per hour allowed by the user" - X-Expires-After: - type: "string" - format: "date-time" - description: "date in UTC when token expires" - 400: - description: "Invalid username/password supplied" - /user/logout: - get: - tags: - - "user" - summary: "Logs out current logged in user session" - description: "" - operationId: "logoutUser" - produces: - - "application/xml" - - "application/json" - parameters: [] - responses: - default: - description: "successful operation" - /user/{username}: - get: - tags: - - "user" - summary: "Get user by user name" - description: "" - operationId: "getUserByName" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "username" - in: "path" - description: "The name that needs to be fetched. Use user1 for testing. " - required: true - type: "string" - responses: - 200: - description: "successful operation" - schema: - $ref: "#/definitions/User" - 400: - description: "Invalid username supplied" - 404: - description: "User not found" - put: - tags: - - "user" - summary: "Updated user" - description: "This can only be done by the logged in user." - operationId: "updateUser" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "username" - in: "path" - description: "name that need to be updated" - required: true - type: "string" - - in: "body" - name: "body" - description: "Updated user object" - required: true - schema: - $ref: "#/definitions/User" - responses: - 400: - description: "Invalid user supplied" - 404: - description: "User not found" - delete: - tags: - - "user" - summary: "Delete user" - description: "This can only be done by the logged in user." - operationId: "deleteUser" - produces: - - "application/xml" - - "application/json" - parameters: - - name: "username" - in: "path" - description: "The name that needs to be deleted" - required: true - type: "string" - responses: - 400: - description: "Invalid username supplied" - 404: - description: "User not found" -securityDefinitions: - components: - securitySchemes: - bearerAuth: # arbitrary name for the security scheme - type: http - scheme: bearer - bearerFormat: JWT # optional, arbitrary value for documentation purposes - petstore_auth: - type: "oauth2" - authorizationUrl: "http://petstore.swagger.io/oauth/dialog" - flow: "implicit" - scopes: - write:pets: "modify pets in your account" - read:pets: "read your pets" - api_key: - type: "apiKey" - name: "api_key" - in: "header" -properties: - resultsCount: - type: "integer" - format: "int64" - description: "The total number of resources matching the request" - mbid: - type: "string" - formats: "uuid" - description: "A musicbrainz ID" -definitions: - Artist: - type: "object" - properties: - mbid: - required: false - $ref: "#/properties/mbid" - id: - type: "integer" - format: "int64" - example: 42 - name: - type: "string" - example: "System of a Down" - creation_date: - type: "string" - format: "date-time" - ArtistNested: - type: "object" - allOf: - - $ref: "#/definitions/Artist" - - type: "object" - properties: - albums: - type: "array" - items: - $ref: "#/definitions/AlbumNested" - - Album: - type: "object" - properties: - mbid: - required: false - $ref: "#/properties/mbid" - id: - type: "integer" - format: "int64" - example: 16 - artist: - type: "integer" - format: "int64" - example: 42 - title: - type: "string" - example: "Toxicity" - creation_date: - type: "string" - format: "date-time" - release_date: - type: "string" - required: false - format: "date" - example: "2001-01-01" - - AlbumNested: - type: "object" - allOf: - - $ref: "#/definitions/Album" - - type: "object" - properties: - tracks: - type: "array" - items: - $ref: "#/definitions/Track" - - Track: - type: "object" - properties: - mbid: - required: false - $ref: "#/properties/mbid" - id: - type: "integer" - format: "int64" - example: 66 - artist: - type: "integer" - format: "int64" - example: 42 - album: - type: "integer" - format: "int64" - example: 16 - title: - type: "string" - example: "Chop Suey!" - position: - required: false - description: "Position of the track in the album" - type: "number" - minimum: 1 - example: 1 - creation_date: - type: "string" - format: "date-time" - - Order: - type: "object" - properties: - id: - type: "integer" - format: "int64" - petId: - type: "integer" - format: "int64" - quantity: - type: "integer" - format: "int32" - shipDate: - type: "string" - format: "date-time" - status: - type: "string" - description: "Order Status" - enum: - - "placed" - - "approved" - - "delivered" - complete: - type: "boolean" - default: false - xml: - name: "Order" - Category: - type: "object" - properties: - id: - type: "integer" - format: "int64" - name: - type: "string" - xml: - name: "Category" - User: - type: "object" - properties: - id: - type: "integer" - format: "int64" - username: - type: "string" - firstName: - type: "string" - lastName: - type: "string" - email: - type: "string" - password: - type: "string" - phone: - type: "string" - userStatus: - type: "integer" - format: "int32" - description: "User Status" - xml: - name: "User" - Tag: - type: "object" - properties: - id: - type: "integer" - format: "int64" - name: - type: "string" - xml: - name: "Tag" - Pet: - type: "object" - required: - - "name" - - "photoUrls" - properties: - id: - type: "integer" - format: "int64" - category: - $ref: "#/definitions/Category" - name: - type: "string" - example: "doggie" - photoUrls: - type: "array" - xml: - name: "photoUrl" - wrapped: true - items: - type: "string" - tags: - type: "array" - xml: - name: "tag" - wrapped: true - items: - $ref: "#/definitions/Tag" - status: - type: "string" - description: "pet status in the store" - enum: - - "available" - - "pending" - - "sold" - xml: - name: "Pet" - ApiResponse: - type: "object" - properties: - code: - type: "integer" - format: "int32" - type: - type: "string" - message: - type: "string" -externalDocs: - description: "Find out more about Funkwhale" - url: "https://docs.funkwhale.audio" diff --git a/docs/build_docs.sh b/docs/build_docs.sh new file mode 100755 index 000000000..fbf2036af --- /dev/null +++ b/docs/build_docs.sh @@ -0,0 +1,5 @@ +#!/bin/bash -eux +# Building sphinx and swagger docs + +python -m sphinx . $BUILD_PATH +TARGET_PATH="$BUILD_PATH/swagger" ./build_swagger.sh diff --git a/docs/build_swagger.sh b/docs/build_swagger.sh new file mode 100755 index 000000000..13ae21b06 --- /dev/null +++ b/docs/build_swagger.sh @@ -0,0 +1,9 @@ +#!/bin/bash -eux + +SWAGGER_VERSION="3.13.6" +TARGET_PATH=${TARGET_PATH-"swagger"} +rm -rf $TARGET_PATH /tmp/swagger-ui +git clone --branch="v$SWAGGER_VERSION" --depth=1 "https://github.com/swagger-api/swagger-ui.git" /tmp/swagger-ui +mv /tmp/swagger-ui/dist $TARGET_PATH +cp swagger.yml $TARGET_PATH +sed -i "s,http://petstore.swagger.io/v2/swagger.json,swagger.yml,g" $TARGET_PATH/index.html diff --git a/docs/swagger.yml b/docs/swagger.yml new file mode 100644 index 000000000..7735a8f20 --- /dev/null +++ b/docs/swagger.yml @@ -0,0 +1,186 @@ +openapi: "3.0" +info: + description: "Documentation for [Funkwhale](https://funkwhale.audio) API. The API is **not** stable yet." + version: "1.0.0" + title: "Funkwhale API" + +servers: + - url: https://demo.funkwhale.audio/api/v1 + description: Demo server + - url: https://node1.funkwhale.test/api/v1 + description: Node 1 (local) + +components: + securitySchemes: + jwt: + type: http + scheme: bearer + bearerFormat: JWT + description: "You can get a token by using the /token endpoint" + +security: + - jwt: [] + +paths: + /token/: + post: + tags: + - "auth" + description: + Obtain a JWT token you can use for authenticating your next requests. + security: [] + responses: + '200': + description: Successfull auth + '400': + description: Invalid credentials + requestBody: + required: true + content: + application/json: + schema: + type: "object" + properties: + username: + type: "string" + example: "demo" + password: + type: "string" + example: "demo" + + /artists/: + get: + tags: + - "artists" + parameters: + - name: "q" + in: "query" + description: "Search query used to filter artists" + schema: + required: false + type: "string" + example: "carpenter" + - name: "listenable" + in: "query" + description: "Filter/exclude artists with listenable tracks" + schema: + required: false + type: "boolean" + responses: + 200: + content: + application/json: + schema: + type: "object" + properties: + count: + $ref: "#/properties/resultsCount" + results: + type: "array" + items: + $ref: "#/definitions/ArtistNested" + +properties: + resultsCount: + type: "integer" + format: "int64" + description: "The total number of resources matching the request" + mbid: + type: "string" + formats: "uuid" + description: "A musicbrainz ID" +definitions: + Artist: + type: "object" + properties: + mbid: + required: false + $ref: "#/properties/mbid" + id: + type: "integer" + format: "int64" + example: 42 + name: + type: "string" + example: "System of a Down" + creation_date: + type: "string" + format: "date-time" + ArtistNested: + type: "object" + allOf: + - $ref: "#/definitions/Artist" + - type: "object" + properties: + albums: + type: "array" + items: + $ref: "#/definitions/AlbumNested" + + Album: + type: "object" + properties: + mbid: + required: false + $ref: "#/properties/mbid" + id: + type: "integer" + format: "int64" + example: 16 + artist: + type: "integer" + format: "int64" + example: 42 + title: + type: "string" + example: "Toxicity" + creation_date: + type: "string" + format: "date-time" + release_date: + type: "string" + required: false + format: "date" + example: "2001-01-01" + + AlbumNested: + type: "object" + allOf: + - $ref: "#/definitions/Album" + - type: "object" + properties: + tracks: + type: "array" + items: + $ref: "#/definitions/Track" + + Track: + type: "object" + properties: + mbid: + required: false + $ref: "#/properties/mbid" + id: + type: "integer" + format: "int64" + example: 66 + artist: + type: "integer" + format: "int64" + example: 42 + album: + type: "integer" + format: "int64" + example: 16 + title: + type: "string" + example: "Chop Suey!" + position: + required: false + description: "Position of the track in the album" + type: "number" + minimum: 1 + example: 1 + creation_date: + type: "string" + format: "date-time" From 56d9c5873c04b29b612a482e273fab637b2c42b0 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 26 Apr 2018 18:23:50 +0200 Subject: [PATCH 10/47] Fix #178: Foundations for API documentation with Swagger --- changes/changelog.d/178.doc | 1 + docs/api.rst | 6 ++++++ docs/index.rst | 1 + 3 files changed, 8 insertions(+) create mode 100644 changes/changelog.d/178.doc create mode 100644 docs/api.rst diff --git a/changes/changelog.d/178.doc b/changes/changelog.d/178.doc new file mode 100644 index 000000000..419e6984b --- /dev/null +++ b/changes/changelog.d/178.doc @@ -0,0 +1 @@ +Foundations for API documentation with Swagger (#178) diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 000000000..650b3885e --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,6 @@ +Funkwhale API +============= + +Funkwhale API is still a work in progress and should not be considered as +stable. We offer an `interactive documentation using swagger `_ +were you can browse available endpoints and try the API. diff --git a/docs/index.rst b/docs/index.rst index 82dcf8c88..cea50ea22 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ Funkwhale is a self-hosted, modern free and open-source music server, heavily in configuration importing-music federation + api upgrading changelog From 51b23b5efd8fea1f09f583bc48c48edde4c854e4 Mon Sep 17 00:00:00 2001 From: Hazmo Date: Thu, 26 Apr 2018 12:32:21 +0200 Subject: [PATCH 11/47] First version of Apache2 conf (transcoding, auth and ws missing) --- deploy/apache.conf | 126 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 deploy/apache.conf diff --git a/deploy/apache.conf b/deploy/apache.conf new file mode 100644 index 000000000..a9f427fac --- /dev/null +++ b/deploy/apache.conf @@ -0,0 +1,126 @@ +# Following variables should be modified according to your setup +Define funkwhale-api http://192.168.1.199:5000 +Define funkwhale-api-ws ws://192.168.1.199:5000 +Define funkwhale-sn funkwhale.duckdns.org +Define MUSIC_DIRECTORY_PATH /music/directory/path + + +# HTTP request redirected to HTTPS + + ServerName ${funkwhale-sn} + + # Default is to force https + RewriteEngine on + RewriteCond %{SERVER_NAME} =${funkwhale-sn} + RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [END,QSA,R=permanent] + + + Options None + Require all granted + + + + + + + + ServerName ${funkwhale-sn} + + # Path to ErrorLog and access log + ErrorLog ${APACHE_LOG_DIR}/funkwhale/error.log + CustomLog ${APACHE_LOG_DIR}/funkwhale/access.log combined + + # TLS + # Feel free to use your own configuration for SSL here or simply remove the + # lines and move the configuration to the previous server block if you + # don't want to run funkwhale behind https (this is not recommanded) + # have a look here for let's encrypt configuration: + # https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx + SSLEngine on + SSLProxyEngine On + SSLCertificateFile /etc/letsencrypt/live/${funkwhale-sn}/fullchain.pem + SSLCertificateKeyFile /etc/letsencrypt/live/${funkwhale-sn}/privkey.pem + Include /etc/letsencrypt/options-ssl-apache.conf + + + DocumentRoot /srv/funkwhale/front/dist + + FallbackResource /index.html + + # Configure Proxy settings + # ProxyPreserveHost pass the original Host header to the backend server + ProxyVia On + ProxyPreserveHost On + + RemoteIPHeader X-Forwarded-For + + + # Turning ProxyRequests on and allowing proxying from all may allow + # spammers to use your proxy to send email. + ProxyRequests Off + + + AddDefaultCharset off + Order Allow,Deny + Allow from all + # Here you can set a password using htpasswd to protect your proxy server + #Authtype Basic + #Authname "Password Required" + #AuthUserFile /etc/apache2/.htpasswd + #Require valid-user + + + # Activating WebSockets (not working) + ProxyPass "/api/v1/instance/activity" "ws://192.168.1.199:5000/api/v1/instance/activity" + + + # similar to nginx 'client_max_body_size 30M;' + LimitRequestBody 31457280 + + ProxyPass ${funkwhale-api}/api + ProxyPassReverse ${funkwhale-api}/api + + + ProxyPass ${funkwhale-api}/federation + ProxyPassReverse ${funkwhale-api}/federation + + + + ProxyPass ${funkwhale-api}/.well-known/webfinger + ProxyPassReverse ${funkwhale-api}/.well-known/webfinger + + + Alias /media /srv/funkwhale/data/media + + # Following alias is bypassing the auth check done in nginx + Alias /_protected/media /srv/funkwhale/data/media + + Alias /staticfiles /srv/funkwhale/data/static + + # Setting appropriate access levels to serve frontend + + Options FollowSymLinks + AllowOverride None + Require all granted + + + + Options FollowSymLinks + AllowOverride None + Require all granted + + + # XSendFile is serving audio files + # WARNING : permissions on paths specified below overrides previous definition, + # everything under those paths is potentially exposed. + # Following directive may be needed to ensure xsendfile is loaded + #LoadModule xsendfile_module modules/mod_xsendfile.so + + XSendFile On + XSendFilePath /srv/funkwhale/data/media + XSendFilePath ${MUSIC_DIRECTORY_PATH} + SetEnv MOD_X_SENDFILE_ENABLED 1 + + + + From ed63824c8f2313c9843172b43a1e0a5f66c0123b Mon Sep 17 00:00:00 2001 From: Hazmo Date: Thu, 26 Apr 2018 13:14:44 +0200 Subject: [PATCH 12/47] Minor fixes on debian doc --- docs/installation/debian.rst | 2 +- docs/installation/external_dependencies.rst | 2 +- docs/installation/index.rst | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/installation/debian.rst b/docs/installation/debian.rst index c4e54218d..eb0c3f0ea 100644 --- a/docs/installation/debian.rst +++ b/docs/installation/debian.rst @@ -31,7 +31,7 @@ Layout All funkwhale-related files will be located under ``/srv/funkwhale`` apart from database files and a few configuration files. We will also have a -dedicated ``funwhale`` user to launch the processes we need and own those files. +dedicated ``funkwhale`` user to launch the processes we need and own those files. You are free to use different values here, just remember to adapt those in the next steps. diff --git a/docs/installation/external_dependencies.rst b/docs/installation/external_dependencies.rst index 059226979..52f9f92c3 100644 --- a/docs/installation/external_dependencies.rst +++ b/docs/installation/external_dependencies.rst @@ -18,7 +18,7 @@ On debian-like systems, you would install the database server like this: .. code-block:: shell - sudo apt-get install postgresql + sudo apt-get install postgresql postgresql-contrib The remaining steps are heavily inspired from `this Digital Ocean guide `_. diff --git a/docs/installation/index.rst b/docs/installation/index.rst index 776c22424..c2a70421b 100644 --- a/docs/installation/index.rst +++ b/docs/installation/index.rst @@ -103,7 +103,8 @@ Then, download our sample virtualhost file and proxy conf: .. parsed-literal:: curl -L -o /etc/nginx/funkwhale_proxy.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale_proxy.conf" - curl -L -o /etc/nginx/sites-enabled/funkwhale.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/nginx.conf" + curl -L -o /etc/nginx/sites-available/funkwhale.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/nginx.conf" + ln -s /etc/nginx/sites-available/funkwhale.conf /etc/nginx/sites-enabled/ Ensure static assets and proxy pass match your configuration, and check the configuration is valid with ``nginx -t``. If everything is fine, you can restart your nginx server with ``service nginx restart``. From a79f42b0ca1d1f83bfcc97f0d23ad5fd77033625 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Fri, 27 Apr 2018 18:28:44 +0200 Subject: [PATCH 13/47] Fix #185: Document that the database should use an utf-8 encoding --- changes/changelog.d/185.doc | 1 + docs/installation/external_dependencies.rst | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 changes/changelog.d/185.doc diff --git a/changes/changelog.d/185.doc b/changes/changelog.d/185.doc new file mode 100644 index 000000000..72144e343 --- /dev/null +++ b/changes/changelog.d/185.doc @@ -0,0 +1 @@ +Document that the database should use an utf-8 encoding (#185) diff --git a/docs/installation/external_dependencies.rst b/docs/installation/external_dependencies.rst index 059226979..7de8abca0 100644 --- a/docs/installation/external_dependencies.rst +++ b/docs/installation/external_dependencies.rst @@ -32,13 +32,22 @@ Create the project database and user: .. code-block:: shell - CREATE DATABASE funkwhale; + CREATE DATABASE "scratch" + WITH ENCODING 'utf8' + LC_COLLATE = 'en_US.utf8' + LC_CTYPE = 'en_US.utf8'; CREATE USER funkwhale; GRANT ALL PRIVILEGES ON DATABASE funkwhale TO funkwhale; Assuming you already have :ref:`created your funkwhale user `, you should now be able to open a postgresql shell: +.. warning:: + + It's importing that you use utf-8 encoding for your database, + otherwise you'll end up with errors and crashes later on when dealing + with music metedata that contains non-ascii chars. + .. code-block:: shell sudo -u funkwhale -H psql From 3d6f0b8b2c13d780e83c9f6072900ac695f13e47 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Fri, 27 Apr 2018 21:10:02 +0200 Subject: [PATCH 14/47] Fix #183: ensure in place imported files get a proper mimetype --- api/funkwhale_api/federation/serializers.py | 5 +++++ api/funkwhale_api/music/tasks.py | 7 +++++++ api/funkwhale_api/music/utils.py | 21 +++++++++++++++++---- api/tests/music/test_import.py | 1 + changes/changelog.d/183.bugfix | 1 + 5 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 changes/changelog.d/183.bugfix diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 00bb7d45b..38efdd3bf 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -1,3 +1,4 @@ +import logging import urllib.parse from django.urls import reverse @@ -21,6 +22,8 @@ AP_CONTEXT = [ {}, ] +logger = logging.getLogger(__name__) + class ActorSerializer(serializers.Serializer): id = serializers.URLField() @@ -620,6 +623,8 @@ class CollectionPageSerializer(serializers.Serializer): for i in raw_items: if i.is_valid(): valid_items.append(i) + else: + logger.debug('Invalid item %s: %s', i.data, i.errors) return valid_items diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py index f2244d785..aaaa2cdca 100644 --- a/api/funkwhale_api/music/tasks.py +++ b/api/funkwhale_api/music/tasks.py @@ -1,3 +1,5 @@ +import os + from django.core.files.base import ContentFile from dynamic_preferences.registries import global_preferences_registry @@ -13,6 +15,7 @@ from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path from django.conf import settings from . import models from . import lyrics as lyrics_utils +from . import utils as music_utils @celery.app.task(name='acoustid.set_on_track_file') @@ -129,6 +132,10 @@ def _do_import(import_job, replace=False, use_acoustid=True): elif not import_job.audio_file and not import_job.source.startswith('file://'): # not an implace import, and we have a source, so let's download it track_file.download_file() + elif not import_job.audio_file and import_job.source.startswith('file://'): + # in place import, we set mimetype from extension + path, ext = os.path.splitext(import_job.source) + track_file.mimetype = music_utils.get_type_from_ext(ext) track_file.save() import_job.status = 'finished' import_job.track_file = track_file diff --git a/api/funkwhale_api/music/utils.py b/api/funkwhale_api/music/utils.py index 7a851f7cc..49a639303 100644 --- a/api/funkwhale_api/music/utils.py +++ b/api/funkwhale_api/music/utils.py @@ -63,8 +63,21 @@ def compute_status(jobs): return 'finished' +AUDIO_EXTENSIONS_AND_MIMETYPE = [ + ('ogg', 'audio/ogg'), + ('mp3', 'audio/mpeg'), +] + +EXTENSION_TO_MIMETYPE = {ext: mt for ext, mt in AUDIO_EXTENSIONS_AND_MIMETYPE} +MIMETYPE_TO_EXTENSION = {mt: ext for ext, mt in AUDIO_EXTENSIONS_AND_MIMETYPE} + + def get_ext_from_type(mimetype): - mapping = { - 'audio/ogg': 'ogg', - 'audio/mpeg': 'mp3', - } + return MIMETYPE_TO_EXTENSION.get(mimetype) + + +def get_type_from_ext(extension): + if extension.startswith('.'): + # we remove leading dot + extension = extension[1:] + return EXTENSION_TO_MIMETYPE.get(extension) diff --git a/api/tests/music/test_import.py b/api/tests/music/test_import.py index 65e0242fb..fa1c98eb4 100644 --- a/api/tests/music/test_import.py +++ b/api/tests/music/test_import.py @@ -243,3 +243,4 @@ def test__do_import_in_place_mbid(factories, tmpfile): assert bool(tf.audio_file) is False assert tf.source == 'file:///test.ogg' + assert tf.mimetype == 'audio/ogg' diff --git a/changes/changelog.d/183.bugfix b/changes/changelog.d/183.bugfix new file mode 100644 index 000000000..03a28e9c3 --- /dev/null +++ b/changes/changelog.d/183.bugfix @@ -0,0 +1 @@ +Ensure in place imported files get a proper mimetype (#183) From 7d3da3d7573e87eee020595fccc85dc4de9ebef4 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Fri, 27 Apr 2018 21:11:20 +0200 Subject: [PATCH 15/47] Added a fix_track_files command to run checks and fixes against library (#183) --- .../music/management/__init__.py | 0 .../music/management/commands/__init__.py | 0 .../management/commands/fix_track_files.py | 45 +++++++++++++++++++ changes/changelog.d/183.enhancement | 1 + 4 files changed, 46 insertions(+) create mode 100644 api/funkwhale_api/music/management/__init__.py create mode 100644 api/funkwhale_api/music/management/commands/__init__.py create mode 100644 api/funkwhale_api/music/management/commands/fix_track_files.py create mode 100644 changes/changelog.d/183.enhancement diff --git a/api/funkwhale_api/music/management/__init__.py b/api/funkwhale_api/music/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/funkwhale_api/music/management/commands/__init__.py b/api/funkwhale_api/music/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/funkwhale_api/music/management/commands/fix_track_files.py b/api/funkwhale_api/music/management/commands/fix_track_files.py new file mode 100644 index 000000000..f68bcf135 --- /dev/null +++ b/api/funkwhale_api/music/management/commands/fix_track_files.py @@ -0,0 +1,45 @@ +import cacheops +import os + +from django.db import transaction +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from funkwhale_api.music import models, utils + + +class Command(BaseCommand): + help = 'Run common checks and fix against imported tracks' + + def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + dest='dry_run', + default=False, + help='Do not execute anything' + ) + + def handle(self, *args, **options): + if options['dry_run']: + self.stdout.write('Dry-run on, will not commit anything') + self.fix_mimetypes(**options) + cacheops.invalidate_model(models.TrackFile) + + @transaction.atomic + def fix_mimetypes(self, dry_run, **kwargs): + self.stdout.write('Fixing missing mimetypes...') + matching = models.TrackFile.objects.filter( + source__startswith='file://', mimetype=None) + self.stdout.write( + '[mimetypes] {} entries found with no mimetype'.format( + matching.count())) + for extension, mimetype in utils.EXTENSION_TO_MIMETYPE.items(): + qs = matching.filter(source__endswith='.{}'.format(extension)) + self.stdout.write( + '[mimetypes] setting {} {} files to {}'.format( + qs.count(), extension, mimetype + )) + if not dry_run: + self.stdout.write('[mimetypes] commiting...') + qs.update(mimetype=mimetype) diff --git a/changes/changelog.d/183.enhancement b/changes/changelog.d/183.enhancement new file mode 100644 index 000000000..2549db810 --- /dev/null +++ b/changes/changelog.d/183.enhancement @@ -0,0 +1 @@ +Added a fix_track_files command to run checks and fixes against library (#183) From 71bd0961af9d4b34a393f43247fbd5897fd12d35 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Fri, 27 Apr 2018 21:50:25 +0200 Subject: [PATCH 16/47] Fixed #184: small UI glitches/bugs in federation tabs --- changes/changelog.d/184.bugfix | 1 + front/src/App.vue | 6 ++++++ front/src/components/federation/LibraryCard.vue | 8 ++++++-- front/src/components/federation/LibraryForm.vue | 2 +- front/src/components/federation/LibraryTrackTable.vue | 2 +- front/src/views/federation/LibraryDetail.vue | 7 ++++++- front/src/views/federation/LibraryList.vue | 3 ++- 7 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 changes/changelog.d/184.bugfix diff --git a/changes/changelog.d/184.bugfix b/changes/changelog.d/184.bugfix new file mode 100644 index 000000000..354b691db --- /dev/null +++ b/changes/changelog.d/184.bugfix @@ -0,0 +1 @@ +Fixed small UI glitches/bugs in federation tabs (#184) diff --git a/front/src/App.vue b/front/src/App.vue index e8cac7476..a21337428 100644 --- a/front/src/App.vue +++ b/front/src/App.vue @@ -97,6 +97,12 @@ html, body { } } +.ellipsis { + text-overflow: ellipsis; + white-space: nowrap; + overflow: hidden; +} + .ui.small.text.container { max-width: 500px !important; } diff --git a/front/src/components/federation/LibraryCard.vue b/front/src/components/federation/LibraryCard.vue index 757561fb3..a16c80f7e 100644 --- a/front/src/components/federation/LibraryCard.vue +++ b/front/src/components/federation/LibraryCard.vue @@ -1,8 +1,12 @@ + + + diff --git a/front/src/views/auth/PasswordResetConfirm.vue b/front/src/views/auth/PasswordResetConfirm.vue new file mode 100644 index 000000000..d29192498 --- /dev/null +++ b/front/src/views/auth/PasswordResetConfirm.vue @@ -0,0 +1,85 @@ + + + + + + From 44ebb9287475279833a33a27b7eea8aae2d389c1 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 6 May 2018 12:50:53 +0200 Subject: [PATCH 40/47] See #187: Front logic for password reset and email confirmation --- .../email/email_confirmation_message.txt | 8 +++ api/funkwhale_api/users/adapters.py | 7 +- front/src/router/index.js | 9 +++ front/src/views/auth/EmailConfirm.vue | 71 +++++++++++++++++++ front/src/views/auth/PasswordReset.vue | 2 +- front/src/views/auth/PasswordResetConfirm.vue | 2 +- 6 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 api/funkwhale_api/templates/account/email/email_confirmation_message.txt create mode 100644 front/src/views/auth/EmailConfirm.vue diff --git a/api/funkwhale_api/templates/account/email/email_confirmation_message.txt b/api/funkwhale_api/templates/account/email/email_confirmation_message.txt new file mode 100644 index 000000000..8aec540fe --- /dev/null +++ b/api/funkwhale_api/templates/account/email/email_confirmation_message.txt @@ -0,0 +1,8 @@ +{% load account %}{% user_display user as user_display %}{% load i18n %}{% autoescape off %}{% blocktrans with site_name=current_site.name site_domain=current_site.domain %}Hello from {{ site_name }}! + +You're receiving this e-mail because user {{ user_display }} at {{ site_domain }} has given yours as an e-mail address to connect their account. + +To confirm this is correct, go to {{ funkwhale_url }}/auth/email/confirm?key={{ key }} +{% endblocktrans %}{% endautoescape %} +{% blocktrans with site_name=current_site.name site_domain=current_site.domain %}Thank you from {{ site_name }}! +{{ site_domain }}{% endblocktrans %} diff --git a/api/funkwhale_api/users/adapters.py b/api/funkwhale_api/users/adapters.py index 96d1b8b1d..7bd341d14 100644 --- a/api/funkwhale_api/users/adapters.py +++ b/api/funkwhale_api/users/adapters.py @@ -1,5 +1,6 @@ -from allauth.account.adapter import DefaultAccountAdapter +from django.conf import settings +from allauth.account.adapter import DefaultAccountAdapter from dynamic_preferences.registries import global_preferences_registry @@ -8,3 +9,7 @@ class FunkwhaleAccountAdapter(DefaultAccountAdapter): def is_open_for_signup(self, request): manager = global_preferences_registry.manager() return manager['users__registration_enabled'] + + def send_mail(self, template_prefix, email, context): + context['funkwhale_url'] = settings.FUNKWHALE_URL + return super().send_mail(template_prefix, email, context) diff --git a/front/src/router/index.js b/front/src/router/index.js index 3bad260bc..b1e208023 100644 --- a/front/src/router/index.js +++ b/front/src/router/index.js @@ -11,6 +11,7 @@ import Settings from '@/components/auth/Settings' import Logout from '@/components/auth/Logout' import PasswordReset from '@/views/auth/PasswordReset' import PasswordResetConfirm from '@/views/auth/PasswordResetConfirm' +import EmailConfirm from '@/views/auth/EmailConfirm' import Library from '@/components/library/Library' import LibraryHome from '@/components/library/Home' import LibraryArtist from '@/components/library/Artist' @@ -69,6 +70,14 @@ export default new Router({ defaultEmail: route.query.email }) }, + { + path: '/auth/email/confirm', + name: 'auth.email-confirm', + component: EmailConfirm, + props: (route) => ({ + defaultKey: route.query.key + }) + }, { path: '/auth/password/reset/confirm', name: 'auth.password-reset-confirm', diff --git a/front/src/views/auth/EmailConfirm.vue b/front/src/views/auth/EmailConfirm.vue new file mode 100644 index 000000000..7ffa3c8d1 --- /dev/null +++ b/front/src/views/auth/EmailConfirm.vue @@ -0,0 +1,71 @@ + + + + + + diff --git a/front/src/views/auth/PasswordReset.vue b/front/src/views/auth/PasswordReset.vue index 6e80661b6..f6b445e00 100644 --- a/front/src/views/auth/PasswordReset.vue +++ b/front/src/views/auth/PasswordReset.vue @@ -5,7 +5,7 @@

{{ $t('Reset your password') }}

-
{{ $('Error while asking for a password reset') }}
+
{{ $t('Error while asking for a password reset') }}
  • {{ error }}
diff --git a/front/src/views/auth/PasswordResetConfirm.vue b/front/src/views/auth/PasswordResetConfirm.vue index d29192498..102ed6126 100644 --- a/front/src/views/auth/PasswordResetConfirm.vue +++ b/front/src/views/auth/PasswordResetConfirm.vue @@ -5,7 +5,7 @@

{{ $t('Change your password') }}

-
{{ $('Error while changing your password') }}
+
{{ $t('Error while changing your password') }}
  • {{ error }}
From 4a7105ae7ef27c144d3813d968893760b5e60b9f Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 6 May 2018 13:48:23 +0200 Subject: [PATCH 41/47] Fix #187: documentation and changelog for email configuration --- api/config/settings/common.py | 6 +++++- api/config/settings/local.py | 3 --- api/setup.cfg | 2 +- changes/changelog.d/187.feature | 24 ++++++++++++++++++++++++ deploy/env.prod.sample | 11 +++++++++++ docs/configuration.rst | 18 ++++++++++++++++++ 6 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 changes/changelog.d/187.feature diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 9c5487d64..1372f59e3 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -172,7 +172,10 @@ FIXTURE_DIRS = ( # EMAIL CONFIGURATION # ------------------------------------------------------------------------------ -EMAIL_BACKEND = env('DJANGO_EMAIL_BACKEND', default='django.core.mail.backends.smtp.EmailBackend') +EMAIL_CONFIG = env.email_url( + 'EMAIL_CONFIG', default='consolemail://') + +vars().update(EMAIL_CONFIG) # DATABASE CONFIGURATION # ------------------------------------------------------------------------------ @@ -367,6 +370,7 @@ CORS_ORIGIN_ALLOW_ALL = True # 'funkwhale.localhost', # ) CORS_ALLOW_CREDENTIALS = True + REST_FRAMEWORK = { 'DEFAULT_PERMISSION_CLASSES': ( 'rest_framework.permissions.IsAuthenticated', diff --git a/api/config/settings/local.py b/api/config/settings/local.py index dcbea66d2..592600629 100644 --- a/api/config/settings/local.py +++ b/api/config/settings/local.py @@ -25,9 +25,6 @@ SECRET_KEY = env("DJANGO_SECRET_KEY", default='mc$&b=5j#6^bv7tld1gyjp2&+^-qrdy=0 # ------------------------------------------------------------------------------ EMAIL_HOST = 'localhost' EMAIL_PORT = 1025 -EMAIL_BACKEND = env('DJANGO_EMAIL_BACKEND', - default='django.core.mail.backends.console.EmailBackend') - # django-debug-toolbar # ------------------------------------------------------------------------------ diff --git a/api/setup.cfg b/api/setup.cfg index a2b8b92c6..b1267c904 100644 --- a/api/setup.cfg +++ b/api/setup.cfg @@ -11,7 +11,7 @@ python_files = tests.py test_*.py *_tests.py testpaths = tests env = SECRET_KEY=test - DJANGO_EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend + EMAIL_CONFIG=consolemail:// CELERY_BROKER_URL=memory:// CELERY_TASK_ALWAYS_EAGER=True CACHEOPS_ENABLED=False diff --git a/changes/changelog.d/187.feature b/changes/changelog.d/187.feature new file mode 100644 index 000000000..501331a19 --- /dev/null +++ b/changes/changelog.d/187.feature @@ -0,0 +1,24 @@ +Users can now request password reset by email, assuming +a SMTP server was correctly configured (#187) + +Update +^^^^^^ + +Starting from this release, Funkwhale will send two types +of emails: + +- Email confirmation emails, to ensure a user's email is valid +- Password reset emails, enabling user to reset their password without an admin's intervention + +Email sending is disabled by default, as it requires additional configuration. +In this mode, emails are simply outputed on stdout. + +If you want to actually send those emails to your users, you should edit your +.env file and tweak the EMAIL_CONFIG variable. See :ref:`setting-EMAIL_CONFIG` +for more details. + +.. note:: + + As a result of these changes, the DJANGO_EMAIL_BACKEND variable, + which was not documented, has no effect anymore. You can safely remove it from + your .env file if it is set. diff --git a/deploy/env.prod.sample b/deploy/env.prod.sample index f1718ff7e..dfd17ff4d 100644 --- a/deploy/env.prod.sample +++ b/deploy/env.prod.sample @@ -6,6 +6,7 @@ # - DJANGO_SECRET_KEY # - DJANGO_ALLOWED_HOSTS # - FUNKWHALE_URL +# - EMAIL_CONFIG (if you plan to send emails) # On non-docker setup **only**, you'll also have to tweak/uncomment those variables: # - DATABASE_URL # - CACHE_URL @@ -41,6 +42,16 @@ FUNKWHALE_API_PORT=5000 # your instance FUNKWHALE_URL=https://yourdomain.funwhale +# Configure email sending using this variale +# By default, funkwhale will output emails sent to stdout +# here are a few examples for this setting +# EMAIL_CONFIG=consolemail:// # output emails to console (the default) +# EMAIL_CONFIG=dummymail:// # disable email sending completely +# On a production instance, you'll usually want to use an external SMTP server: +# EMAIL_CONFIG=smtp://user@:password@youremail.host:25' +# EMAIL_CONFIG=smtp+ssl://user@:password@youremail.host:465' +# EMAIL_CONFIG=smtp+tls://user@:password@youremail.host:587' + # Depending on the reverse proxy used in front of your funkwhale instance, # the API will use different kind of headers to serve audio files # Allowed values: nginx, apache2 diff --git a/docs/configuration.rst b/docs/configuration.rst index 1c89feeb8..f498b9c87 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -39,6 +39,24 @@ settings in this interface. Configuration reference ----------------------- +.. _setting-EMAIL_CONFIG: + +``EMAIL_CONFIG`` +^^^^^^^^^^^^^^^^ + +Determine how emails are sent. + +Default: ``consolemail://`` + +Possible values: + +- ``consolemail://``: Output sent emails to stdout +- ``dummymail://``: Completely discard sent emails +- ``smtp://user:password@youremail.host:25``: Send emails via SMTP via youremail.host on port 25, without encryption, authenticating as user "user" with password "password" +- ``smtp+ssl://user:password@youremail.host:465``: Send emails via SMTP via youremail.host on port 465, using SSL encryption, authenticating as user "user" with password "password" +- ``smtp+tls://user:password@youremail.host:587``: Send emails via SMTP via youremail.host on port 587, using TLS encryption, authenticating as user "user" with password "password" + + .. _setting-MUSIC_DIRECTORY_PATH: ``MUSIC_DIRECTORY_PATH`` From f3431598564e84817f7bbdddcae6053fa5df1ebd Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 6 May 2018 15:36:18 +0200 Subject: [PATCH 42/47] Added an accessed_date field on TrackFile for easier cache deletion (#189) --- .../migrations/0026_trackfile_accessed_date.py | 18 ++++++++++++++++++ api/funkwhale_api/music/models.py | 1 + api/funkwhale_api/music/views.py | 5 +++++ api/tests/music/test_views.py | 14 ++++++++++++++ 4 files changed, 38 insertions(+) create mode 100644 api/funkwhale_api/music/migrations/0026_trackfile_accessed_date.py diff --git a/api/funkwhale_api/music/migrations/0026_trackfile_accessed_date.py b/api/funkwhale_api/music/migrations/0026_trackfile_accessed_date.py new file mode 100644 index 000000000..1d5327d93 --- /dev/null +++ b/api/funkwhale_api/music/migrations/0026_trackfile_accessed_date.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.3 on 2018-05-06 12:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('music', '0025_auto_20180419_2023'), + ] + + operations = [ + migrations.AddField( + model_name='trackfile', + name='accessed_date', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index 18f181e88..655d38755 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -415,6 +415,7 @@ class TrackFile(models.Model): source = models.URLField(null=True, blank=True, max_length=500) creation_date = models.DateTimeField(default=timezone.now) modification_date = models.DateTimeField(auto_now=True) + accessed_date = models.DateTimeField(null=True, blank=True) duration = models.IntegerField(null=True, blank=True) acoustid_track_id = models.UUIDField(null=True, blank=True) mimetype = models.CharField(null=True, blank=True, max_length=200) diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index f53de1b0a..76fc8bc3e 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -14,6 +14,7 @@ from django.db.models.functions import Length from django.db.models import Count from django.http import StreamingHttpResponse from django.urls import reverse +from django.utils import timezone from django.utils.decorators import method_decorator from rest_framework import viewsets, views, mixins @@ -264,6 +265,10 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet): except models.TrackFile.DoesNotExist: return Response(status=404) + # we update the accessed_date + f.accessed_date = timezone.now() + f.save(update_fields=['accessed_date']) + mt = f.mimetype audio_file = f.audio_file try: diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py index 2cdee4e8c..b22ab7fd5 100644 --- a/api/tests/music/test_views.py +++ b/api/tests/music/test_views.py @@ -2,6 +2,7 @@ import io import pytest from django.urls import reverse +from django.utils import timezone from funkwhale_api.music import views from funkwhale_api.federation import actors @@ -149,6 +150,19 @@ def test_can_proxy_remote_track( assert library_track.audio_file.read() == b'test' +def test_serve_updates_access_date(factories, settings, api_client): + settings.PROTECT_AUDIO_FILES = False + track_file = factories['music.TrackFile']() + now = timezone.now() + assert track_file.accessed_date is None + + response = api_client.get(track_file.path) + track_file.refresh_from_db() + + assert response.status_code == 200 + assert track_file.accessed_date > now + + def test_can_create_import_from_federation_tracks( factories, superuser_api_client, mocker): lts = factories['federation.LibraryTrack'].create_batch(size=5) From bc2c9950e375400ac5acee3180b9d0cfab8b90dc Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 6 May 2018 15:36:49 +0200 Subject: [PATCH 43/47] Fix #189: federation cache should now delete properly, including orphaned files --- api/funkwhale_api/federation/tasks.py | 44 +++++++++++++++++----- api/tests/federation/test_tasks.py | 53 +++++++++++++++++++++------ changes/changelog.d/189.bugfix | 1 + 3 files changed, 77 insertions(+), 21 deletions(-) create mode 100644 changes/changelog.d/189.bugfix diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index adc354c4f..8f931b0ed 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -1,8 +1,10 @@ import datetime import json import logging +import os from django.conf import settings +from django.db.models import Q from django.utils import timezone from requests.exceptions import RequestException @@ -96,16 +98,38 @@ def clean_music_cache(): delay = preferences['federation__music_cache_duration'] if delay < 1: return # cache clearing disabled + limit = timezone.now() - datetime.timedelta(minutes=delay) candidates = models.LibraryTrack.objects.filter( - audio_file__isnull=False - ).values_list('local_track_file__track', flat=True) - listenings = Listening.objects.filter( - creation_date__gte=timezone.now() - datetime.timedelta(minutes=delay), - track__pk__in=candidates).values_list('track', flat=True) - too_old = set(candidates) - set(listenings) - - to_remove = models.LibraryTrack.objects.filter( - local_track_file__track__pk__in=too_old).only('audio_file') - for lt in to_remove: + Q(audio_file__isnull=False) & ( + Q(local_track_file__accessed_date__lt=limit) | + Q(local_track_file__accessed_date=None) + ) + ).exclude(audio_file='').only('audio_file', 'id') + for lt in candidates: lt.audio_file.delete() + + # we also delete orphaned files, if any + storage = models.LibraryTrack._meta.get_field('audio_file').storage + files = get_files(storage, 'federation_cache') + existing = models.LibraryTrack.objects.filter(audio_file__in=files) + missing = set(files) - set(existing.values_list('audio_file', flat=True)) + for m in missing: + storage.delete(m) + + +def get_files(storage, *parts): + """ + This is a recursive function that return all files available + in a given directory using django's storage. + """ + if not parts: + raise ValueError('Missing path') + + dirs, files = storage.listdir(os.path.join(*parts)) + for dir in dirs: + files += get_files(storage, *(list(parts) + [dir])) + return [ + os.path.join(parts[-1], path) + for path in files + ] diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py index 506fbc1fe..3517e8feb 100644 --- a/api/tests/federation/test_tasks.py +++ b/api/tests/federation/test_tasks.py @@ -1,4 +1,7 @@ import datetime +import os +import pathlib +import pytest from django.core.paginator import Paginator from django.utils import timezone @@ -117,17 +120,16 @@ def test_clean_federation_music_cache_if_no_listen(preferences, factories): lt1 = factories['federation.LibraryTrack'](with_audio_file=True) lt2 = factories['federation.LibraryTrack'](with_audio_file=True) lt3 = factories['federation.LibraryTrack'](with_audio_file=True) - tf1 = factories['music.TrackFile'](library_track=lt1) - tf2 = factories['music.TrackFile'](library_track=lt2) - tf3 = factories['music.TrackFile'](library_track=lt3) - - # we listen to the first one, and the second one (but weeks ago) - listening1 = factories['history.Listening']( - track=tf1.track, - creation_date=timezone.now()) - listening2 = factories['history.Listening']( - track=tf2.track, - creation_date=timezone.now() - datetime.timedelta(minutes=61)) + tf1 = factories['music.TrackFile']( + accessed_date=timezone.now(), library_track=lt1) + tf2 = factories['music.TrackFile']( + accessed_date=timezone.now()-datetime.timedelta(minutes=61), + library_track=lt2) + tf3 = factories['music.TrackFile']( + accessed_date=None, library_track=lt3) + path1 = lt1.audio_file.path + path2 = lt2.audio_file.path + path3 = lt3.audio_file.path tasks.clean_music_cache() @@ -138,3 +140,32 @@ def test_clean_federation_music_cache_if_no_listen(preferences, factories): assert bool(lt1.audio_file) is True assert bool(lt2.audio_file) is False assert bool(lt3.audio_file) is False + assert os.path.exists(path1) is True + assert os.path.exists(path2) is False + assert os.path.exists(path3) is False + + +def test_clean_federation_music_cache_orphaned( + settings, preferences, factories): + preferences['federation__music_cache_duration'] = 60 + path = os.path.join(settings.MEDIA_ROOT, 'federation_cache') + keep_path = os.path.join(os.path.join(path, '1a', 'b2'), 'keep.ogg') + remove_path = os.path.join(os.path.join(path, 'c3', 'd4'), 'remove.ogg') + os.makedirs(os.path.dirname(keep_path), exist_ok=True) + os.makedirs(os.path.dirname(remove_path), exist_ok=True) + pathlib.Path(keep_path).touch() + pathlib.Path(remove_path).touch() + lt = factories['federation.LibraryTrack']( + with_audio_file=True, + audio_file__path=keep_path) + tf = factories['music.TrackFile']( + library_track=lt, + accessed_date=timezone.now()) + + tasks.clean_music_cache() + + lt.refresh_from_db() + + assert bool(lt.audio_file) is True + assert os.path.exists(lt.audio_file.path) is True + assert os.path.exists(remove_path) is False diff --git a/changes/changelog.d/189.bugfix b/changes/changelog.d/189.bugfix new file mode 100644 index 000000000..076058e63 --- /dev/null +++ b/changes/changelog.d/189.bugfix @@ -0,0 +1 @@ +Federation cache suppression is now simpler and also deletes orphaned files (#189) From 9130e14fa0ee9e22cbedd8c9329c96e79faa8e80 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 6 May 2018 15:43:26 +0200 Subject: [PATCH 44/47] fixed missing changelog and typos --- changes/changelog.d/176.enhancement | 2 +- changes/changelog.d/182.bugfix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changes/changelog.d/176.enhancement b/changes/changelog.d/176.enhancement index 0e431f28c..874aed727 100644 --- a/changes/changelog.d/176.enhancement +++ b/changes/changelog.d/176.enhancement @@ -1 +1 @@ -Can now relaunch erored jobs and batches (#176) +Can now relaunch errored jobs and batches (#176) diff --git a/changes/changelog.d/182.bugfix b/changes/changelog.d/182.bugfix index e69de29bb..6a880c4b0 100644 --- a/changes/changelog.d/182.bugfix +++ b/changes/changelog.d/182.bugfix @@ -0,0 +1 @@ +X-sendfile not working with in place import (#182) From 480b6d7fd638dfb6c6e23d199262447b8cfdf1a6 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 6 May 2018 15:47:34 +0200 Subject: [PATCH 45/47] Include link to upgrade instructions in changelog --- changes/template.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changes/template.rst b/changes/template.rst index f4d94dee8..24f0e87eb 100644 --- a/changes/template.rst +++ b/changes/template.rst @@ -1,3 +1,6 @@ + +Upgrade instructions are available at https://docs.funkwhale.audio/upgrading.html + {% for section, _ in sections.items() %} {% if sections[section] %} {% for category, val in definitions.items() if category in sections[section]%} From 82f5dc20f3e0bea60d765f173ef7c28c3965d09a Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 6 May 2018 16:23:23 +0200 Subject: [PATCH 46/47] Documentation for missing DEFAULT_FROM_EMAIL setting --- api/config/settings/common.py | 12 ++++++++++++ api/config/settings/production.py | 10 ---------- deploy/env.prod.sample | 5 ++++- docs/configuration.rst | 14 ++++++++++++++ 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 1372f59e3..f88aa5dd5 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -172,6 +172,18 @@ FIXTURE_DIRS = ( # EMAIL CONFIGURATION # ------------------------------------------------------------------------------ + +# EMAIL +# ------------------------------------------------------------------------------ +DEFAULT_FROM_EMAIL = env( + 'DEFAULT_FROM_EMAIL', + default='Funkwhale '.format(FUNKWHALE_HOSTNAME)) + +EMAIL_SUBJECT_PREFIX = env( + "EMAIL_SUBJECT_PREFIX", default='[Funkwhale] ') +SERVER_EMAIL = env('SERVER_EMAIL', default=DEFAULT_FROM_EMAIL) + + EMAIL_CONFIG = env.email_url( 'EMAIL_CONFIG', default='consolemail://') diff --git a/api/config/settings/production.py b/api/config/settings/production.py index f238c2d20..2866e9103 100644 --- a/api/config/settings/production.py +++ b/api/config/settings/production.py @@ -68,16 +68,6 @@ DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' # ------------------------ STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage' - -# EMAIL -# ------------------------------------------------------------------------------ -DEFAULT_FROM_EMAIL = env('DJANGO_DEFAULT_FROM_EMAIL', - default='funkwhale_api ') - -EMAIL_SUBJECT_PREFIX = env("DJANGO_EMAIL_SUBJECT_PREFIX", default='[funkwhale_api] ') -SERVER_EMAIL = env('DJANGO_SERVER_EMAIL', default=DEFAULT_FROM_EMAIL) - - # TEMPLATE CONFIGURATION # ------------------------------------------------------------------------------ # See: diff --git a/deploy/env.prod.sample b/deploy/env.prod.sample index dfd17ff4d..4b27595af 100644 --- a/deploy/env.prod.sample +++ b/deploy/env.prod.sample @@ -6,7 +6,7 @@ # - DJANGO_SECRET_KEY # - DJANGO_ALLOWED_HOSTS # - FUNKWHALE_URL -# - EMAIL_CONFIG (if you plan to send emails) +# - EMAIL_CONFIG and DEFAULT_FROM_EMAIL if you plan to send emails) # On non-docker setup **only**, you'll also have to tweak/uncomment those variables: # - DATABASE_URL # - CACHE_URL @@ -52,6 +52,9 @@ FUNKWHALE_URL=https://yourdomain.funwhale # EMAIL_CONFIG=smtp+ssl://user@:password@youremail.host:465' # EMAIL_CONFIG=smtp+tls://user@:password@youremail.host:587' +# The email address to use to send systme emails. By default, we will +# DEFAULT_FROM_EMAIL=noreply@yourdomain + # Depending on the reverse proxy used in front of your funkwhale instance, # the API will use different kind of headers to serve audio files # Allowed values: nginx, apache2 diff --git a/docs/configuration.rst b/docs/configuration.rst index f498b9c87..bbc658e08 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -56,6 +56,20 @@ Possible values: - ``smtp+ssl://user:password@youremail.host:465``: Send emails via SMTP via youremail.host on port 465, using SSL encryption, authenticating as user "user" with password "password" - ``smtp+tls://user:password@youremail.host:587``: Send emails via SMTP via youremail.host on port 587, using TLS encryption, authenticating as user "user" with password "password" +.. _setting-DEFAULT_FROM_EMAIL: + +``DEFAULT_FROM_EMAIL`` +^^^^^^^^^^^^^^^^^^^^^^ + +The email address to use to send email. + +Default: ``Funkwhale `` + +.. note:: + + Both the forms ``Funkwhale `` and + ``noreply@yourdomain`` work. + .. _setting-MUSIC_DIRECTORY_PATH: From 7908ae3942c94d33febcd69733593797a1fc5f5b Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 6 May 2018 16:24:12 +0200 Subject: [PATCH 47/47] Version bump and changelog --- CHANGELOG | 105 ++++++++++++++++++++ api/funkwhale_api/__init__.py | 2 +- changes/changelog.d/109.enhancement | 1 - changes/changelog.d/176.enhancement | 1 - changes/changelog.d/178.doc | 1 - changes/changelog.d/180.doc | 1 - changes/changelog.d/182.bugfix | 1 - changes/changelog.d/183.bugfix | 1 - changes/changelog.d/183.enhancement | 1 - changes/changelog.d/184.bugfix | 1 - changes/changelog.d/185.doc | 1 - changes/changelog.d/186.enhancement | 24 ----- changes/changelog.d/187.feature | 24 ----- changes/changelog.d/189.bugfix | 1 - changes/changelog.d/actor-fetch.enhancement | 1 - changes/changelog.d/apache.enhancement | 1 - changes/changelog.d/optimization.doc | 11 -- changes/changelog.d/sidebar.enhancement | 1 - 18 files changed, 106 insertions(+), 73 deletions(-) delete mode 100644 changes/changelog.d/109.enhancement delete mode 100644 changes/changelog.d/176.enhancement delete mode 100644 changes/changelog.d/178.doc delete mode 100644 changes/changelog.d/180.doc delete mode 100644 changes/changelog.d/182.bugfix delete mode 100644 changes/changelog.d/183.bugfix delete mode 100644 changes/changelog.d/183.enhancement delete mode 100644 changes/changelog.d/184.bugfix delete mode 100644 changes/changelog.d/185.doc delete mode 100644 changes/changelog.d/186.enhancement delete mode 100644 changes/changelog.d/187.feature delete mode 100644 changes/changelog.d/189.bugfix delete mode 100644 changes/changelog.d/actor-fetch.enhancement delete mode 100644 changes/changelog.d/apache.enhancement delete mode 100644 changes/changelog.d/optimization.doc delete mode 100644 changes/changelog.d/sidebar.enhancement diff --git a/CHANGELOG b/CHANGELOG index a9d92f50c..82c867bf8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -10,6 +10,111 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog. .. towncrier +0.11 (unreleased) +----------------- + +Upgrade instructions are available at https://docs.funkwhale.audio/upgrading.html + +Special thanks for this release go to @renon:matrix.org (@Hazmo on Gitlab) +for bringing Apache2 support to Funkwhale and contributing on other issues. +Thank you! + +Features: + +- Funkwhale now works behind an Apache2 reverse proxy (!165) + check out the brand new documentation at https://docs.funkwhale.audio/installation/index.html#apache2 + if you want to try it! +- Users can now request password reset by email, assuming a SMTP server was + correctly configured (#187) + +Enhancements: + +- Added a fix_track_files command to run checks and fixes against library + (#183) +- Avoid fetching Actor object on every request authentication +- Can now relaunch errored jobs and batches (#176) +- List pending requests by default, added a status filter for requests (#109) +- More structured menus in sidebar, added labels with notifications +- Sample virtual-host file for Apache2 reverse-proxy (!165) +- Store high-level settings (such as federation or auth-related ones) in + database (#186) + + +Bugfixes: + +- Ensure in place imported files get a proper mimetype (#183) +- Federation cache suppression is now simpler and also deletes orphaned files + (#189) +- Fixed small UI glitches/bugs in federation tabs (#184) +- X-sendfile not working with in place import (#182) + + +Documentation: + +- Added a documentation area for third-party projects (#180) +- Added documentation for optimizing Funkwhale and reduce its memory footprint. +- Document that the database should use an utf-8 encoding (#185) +- Foundations for API documentation with Swagger (#178) + + +Database storage for high-level settings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Due to the work done in #186, the following environment variables have been +deprecated: + +- FEDERATION_ENABLED +- FEDERATION_COLLECTION_PAGE_SIZE +- FEDERATION_MUSIC_NEEDS_APPROVAL +- FEDERATION_ACTOR_FETCH_DELAY +- PLAYLISTS_MAX_TRACKS +- API_AUTHENTICATION_REQUIRED + +Configuration for this settings has been moved to database, as it will provide +a better user-experience, by allowing you to edit these values on-the-fly, +without restarting Funkwhale processes. + +You can leave those environment variables in your .env file for now, as the +values will be used to populate the database entries. We'll make a proper +announcement when the variables won't be used anymore. + +Please browse https://docs.funkwhale.audio/configuration.html#instance-settings +for more information about instance configuration using the web interface. + + +System emails +^^^^^^^^^^^^^ + +Starting from this release, Funkwhale will send two types +of emails: + +- Email confirmation emails, to ensure a user's email is valid +- Password reset emails, enabling user to reset their password without an admin's intervention + +Email sending is disabled by default, as it requires additional configuration. +In this mode, emails are simply outputed on stdout. + +If you want to actually send those emails to your users, you should edit your +.env file and tweak the EMAIL_CONFIG variable. See :ref:`setting-EMAIL_CONFIG` +for more details. + +.. note:: + + As a result of these changes, the DJANGO_EMAIL_BACKEND variable, + which was not documented, has no effect anymore. You can safely remove it from + your .env file if it is set. + + +Proxy headers for non-docker deployments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For non-docker deployments, add ``--proxy-headers`` at the end of the ``daphne`` +command in :file:`/etc/systemd/system/funkwhale-server.service`. + +This will ensure the application receive the correct IP address from the client +and not the proxy's one. + + 0.10 (2018-04-23) ----------------- diff --git a/api/funkwhale_api/__init__.py b/api/funkwhale_api/__init__.py index 596926919..4f62dd9b5 100644 --- a/api/funkwhale_api/__init__.py +++ b/api/funkwhale_api/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__version__ = '0.10' +__version__ = '0.11' __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')]) diff --git a/changes/changelog.d/109.enhancement b/changes/changelog.d/109.enhancement deleted file mode 100644 index 60e740d73..000000000 --- a/changes/changelog.d/109.enhancement +++ /dev/null @@ -1 +0,0 @@ -List pending requests by default, added a status filter for requests (#109) diff --git a/changes/changelog.d/176.enhancement b/changes/changelog.d/176.enhancement deleted file mode 100644 index 874aed727..000000000 --- a/changes/changelog.d/176.enhancement +++ /dev/null @@ -1 +0,0 @@ -Can now relaunch errored jobs and batches (#176) diff --git a/changes/changelog.d/178.doc b/changes/changelog.d/178.doc deleted file mode 100644 index 419e6984b..000000000 --- a/changes/changelog.d/178.doc +++ /dev/null @@ -1 +0,0 @@ -Foundations for API documentation with Swagger (#178) diff --git a/changes/changelog.d/180.doc b/changes/changelog.d/180.doc deleted file mode 100644 index ee79f3e3f..000000000 --- a/changes/changelog.d/180.doc +++ /dev/null @@ -1 +0,0 @@ -Added a documentation area for third-party projects (#180) diff --git a/changes/changelog.d/182.bugfix b/changes/changelog.d/182.bugfix deleted file mode 100644 index 6a880c4b0..000000000 --- a/changes/changelog.d/182.bugfix +++ /dev/null @@ -1 +0,0 @@ -X-sendfile not working with in place import (#182) diff --git a/changes/changelog.d/183.bugfix b/changes/changelog.d/183.bugfix deleted file mode 100644 index 03a28e9c3..000000000 --- a/changes/changelog.d/183.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ensure in place imported files get a proper mimetype (#183) diff --git a/changes/changelog.d/183.enhancement b/changes/changelog.d/183.enhancement deleted file mode 100644 index 2549db810..000000000 --- a/changes/changelog.d/183.enhancement +++ /dev/null @@ -1 +0,0 @@ -Added a fix_track_files command to run checks and fixes against library (#183) diff --git a/changes/changelog.d/184.bugfix b/changes/changelog.d/184.bugfix deleted file mode 100644 index 354b691db..000000000 --- a/changes/changelog.d/184.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed small UI glitches/bugs in federation tabs (#184) diff --git a/changes/changelog.d/185.doc b/changes/changelog.d/185.doc deleted file mode 100644 index 72144e343..000000000 --- a/changes/changelog.d/185.doc +++ /dev/null @@ -1 +0,0 @@ -Document that the database should use an utf-8 encoding (#185) diff --git a/changes/changelog.d/186.enhancement b/changes/changelog.d/186.enhancement deleted file mode 100644 index 36777c786..000000000 --- a/changes/changelog.d/186.enhancement +++ /dev/null @@ -1,24 +0,0 @@ -Store high-level settings (such as federation or auth-related ones) in database (#186) - -Changelog -^^^^^^^^^ -Due to the work done in #186, the following environment variables have been -deprecated: - -- FEDERATION_ENABLED -- FEDERATION_COLLECTION_PAGE_SIZE -- FEDERATION_MUSIC_NEEDS_APPROVAL -- FEDERATION_ACTOR_FETCH_DELAY -- PLAYLISTS_MAX_TRACKS -- API_AUTHENTICATION_REQUIRED - -Configuration for this settings has been moved to database, as it will provide -a better user-experience, by allowing you to edit these values on-the-fly, -without restarting Funkwhale processes. - -You can leave those environment variables in your .env file for now, as the -values will be used to populate the database entries. We'll make a proper -announcement when the variables won't be used anymore. - -Please browse https://docs.funkwhale.audio/configuration.html#instance-settings -for more information about instance configuration using the web interface. diff --git a/changes/changelog.d/187.feature b/changes/changelog.d/187.feature deleted file mode 100644 index 501331a19..000000000 --- a/changes/changelog.d/187.feature +++ /dev/null @@ -1,24 +0,0 @@ -Users can now request password reset by email, assuming -a SMTP server was correctly configured (#187) - -Update -^^^^^^ - -Starting from this release, Funkwhale will send two types -of emails: - -- Email confirmation emails, to ensure a user's email is valid -- Password reset emails, enabling user to reset their password without an admin's intervention - -Email sending is disabled by default, as it requires additional configuration. -In this mode, emails are simply outputed on stdout. - -If you want to actually send those emails to your users, you should edit your -.env file and tweak the EMAIL_CONFIG variable. See :ref:`setting-EMAIL_CONFIG` -for more details. - -.. note:: - - As a result of these changes, the DJANGO_EMAIL_BACKEND variable, - which was not documented, has no effect anymore. You can safely remove it from - your .env file if it is set. diff --git a/changes/changelog.d/189.bugfix b/changes/changelog.d/189.bugfix deleted file mode 100644 index 076058e63..000000000 --- a/changes/changelog.d/189.bugfix +++ /dev/null @@ -1 +0,0 @@ -Federation cache suppression is now simpler and also deletes orphaned files (#189) diff --git a/changes/changelog.d/actor-fetch.enhancement b/changes/changelog.d/actor-fetch.enhancement deleted file mode 100644 index 17f3a88df..000000000 --- a/changes/changelog.d/actor-fetch.enhancement +++ /dev/null @@ -1 +0,0 @@ -Avoid fetching Actor object on every request authentication diff --git a/changes/changelog.d/apache.enhancement b/changes/changelog.d/apache.enhancement deleted file mode 100644 index 5aa433805..000000000 --- a/changes/changelog.d/apache.enhancement +++ /dev/null @@ -1 +0,0 @@ -Sample virtual-host file for Apache2 reverse-proxy (!165) diff --git a/changes/changelog.d/optimization.doc b/changes/changelog.d/optimization.doc deleted file mode 100644 index 929e14821..000000000 --- a/changes/changelog.d/optimization.doc +++ /dev/null @@ -1,11 +0,0 @@ -Added documentation for optimizing Funkwhale and reduce its memory -footprint. - -Changelog -^^^^^^^^^ - -For non-docker deployments, add ``--proxy-headers`` at the end of the ``daphne`` -command in :file:`/etc/systemd/system/funkwhale-server.service`. - -This will ensure the application receive the correct IP address from the client -and not the proxy's one. diff --git a/changes/changelog.d/sidebar.enhancement b/changes/changelog.d/sidebar.enhancement deleted file mode 100644 index 1bc1a482f..000000000 --- a/changes/changelog.d/sidebar.enhancement +++ /dev/null @@ -1 +0,0 @@ -More structured menus in sidebar, added labels with notifications