From c529d4d260a9ad23239b58e8915a19decd6704bf Mon Sep 17 00:00:00 2001 From: Agate Date: Tue, 21 Apr 2020 14:47:16 +0200 Subject: [PATCH] Fix #1077: improved performance and error handling in fetch_attachment --- api/funkwhale_api/common/tasks.py | 2 +- api/funkwhale_api/common/views.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/api/funkwhale_api/common/tasks.py b/api/funkwhale_api/common/tasks.py index 74ce3b0e1..6f4d9439c 100644 --- a/api/funkwhale_api/common/tasks.py +++ b/api/funkwhale_api/common/tasks.py @@ -77,7 +77,7 @@ def fetch_remote_attachment(attachment, filename=None, save=True): attachment.last_fetch_date = timezone.now() with tempfile.TemporaryFile() as tf: with s.get(attachment.url, timeout=5, stream=True) as r: - for chunk in r.iter_content(): + for chunk in r.iter_content(chunk_size=1024 * 100): tf.write(chunk) tf.seek(0) if not filename: diff --git a/api/funkwhale_api/common/views.py b/api/funkwhale_api/common/views.py index 1766ba127..a6ee0c926 100644 --- a/api/funkwhale_api/common/views.py +++ b/api/funkwhale_api/common/views.py @@ -1,3 +1,4 @@ +import logging import time from django.conf import settings @@ -23,6 +24,9 @@ from . import throttling from . import utils +logger = logging.getLogger(__name__) + + class SkipFilterForGetObject: def get_object(self, *args, **kwargs): setattr(self.request, "_skip_filters", True) @@ -172,7 +176,11 @@ class AttachmentViewSet( if size not in ["original", "medium_square_crop"]: size = "original" - tasks.fetch_remote_attachment(instance) + try: + tasks.fetch_remote_attachment(instance) + except Exception: + logger.exception("Error while fetching attachment %s", instance.url) + return response.Response(status=500) data = self.serializer_class(instance).data redirect = response.Response(status=302) redirect["Location"] = data["urls"][size]