diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 43a7a29f5..3237718e0 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -515,7 +515,6 @@ CACHES = { "LOCATION": "local-cache", }, } - CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache" CHANNEL_LAYERS = { @@ -530,7 +529,20 @@ CACHES["default"]["OPTIONS"] = { "IGNORE_EXCEPTIONS": True, # mimics memcache behavior. # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior } +CACHEOPS_DURATION = env("CACHEOPS_DURATION", default=0) +CACHEOPS_ENABLED = bool(CACHEOPS_DURATION) +if CACHEOPS_ENABLED: + INSTALLED_APPS += ("cacheops",) + CACHEOPS_REDIS = env("CACHE_URL", default=CACHE_DEFAULT) + CACHEOPS_PREFIX = lambda _: "cacheops" # noqa + CACHEOPS_DEFAULTS = {"timeout": CACHEOPS_DURATION} + CACHEOPS = { + "users.user": {"ops": "get"}, + "music.album": {"ops": "count"}, + "music.artist": {"ops": "count"}, + "music.track": {"ops": "count"}, + } # CELERY INSTALLED_APPS += ("funkwhale_api.taskapp.celery.CeleryConfig",) diff --git a/api/config/settings/local.py b/api/config/settings/local.py index 632eb3201..99ba6e23e 100644 --- a/api/config/settings/local.py +++ b/api/config/settings/local.py @@ -38,10 +38,26 @@ EMAIL_PORT = 1025 DEBUG_TOOLBAR_CONFIG = { "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"], "SHOW_TEMPLATE_CONTEXT": True, - "SHOW_TOOLBAR_CALLBACK": lambda request: True, + "SHOW_TOOLBAR_CALLBACK": lambda request: "debug" in request.GET, "JQUERY_URL": "/staticfiles/admin/js/vendor/jquery/jquery.js", } +DEBUG_TOOLBAR_PANELS = [ + # 'debug_toolbar.panels.versions.VersionsPanel', + "debug_toolbar.panels.timer.TimerPanel", + "debug_toolbar.panels.settings.SettingsPanel", + "debug_toolbar.panels.headers.HeadersPanel", + # 'debug_toolbar.panels.request.RequestPanel', + "debug_toolbar.panels.sql.SQLPanel", + # 'debug_toolbar.panels.staticfiles.StaticFilesPanel', + # 'debug_toolbar.panels.templates.TemplatesPanel', + "debug_toolbar.panels.cache.CachePanel", + # 'debug_toolbar.panels.signals.SignalsPanel', + # 'debug_toolbar.panels.logging.LoggingPanel', + # 'debug_toolbar.panels.redirects.RedirectsPanel', + # 'debug_toolbar.panels.profiling.ProfilingPanel', +] + # django-extensions # ------------------------------------------------------------------------------ # INSTALLED_APPS += ('django_extensions', ) @@ -69,4 +85,7 @@ if env.bool("WEAK_PASSWORDS", default=False): # Faster during tests PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",) -MIDDLEWARE = ("funkwhale_api.common.middleware.DevHttpsMiddleware",) + MIDDLEWARE +MIDDLEWARE = ( + "funkwhale_api.common.middleware.DevHttpsMiddleware", + "funkwhale_api.common.middleware.ProfilerMiddleware", +) + MIDDLEWARE diff --git a/api/funkwhale_api/common/middleware.py b/api/funkwhale_api/common/middleware.py index 6866148b9..d8573baaa 100644 --- a/api/funkwhale_api/common/middleware.py +++ b/api/funkwhale_api/common/middleware.py @@ -1,4 +1,5 @@ import html +import io import requests import time import xml.sax.saxutils @@ -242,3 +243,53 @@ class ThrottleStatusMiddleware: response["X-RateLimit-ResetSeconds"] = str(remaining) return response + + +class ProfilerMiddleware: + """ + from https://github.com/omarish/django-cprofile-middleware/blob/master/django_cprofile_middleware/middleware.py + Simple profile middleware to profile django views. To run it, add ?prof to + the URL like this: + http://localhost:8000/view/?prof + Optionally pass the following to modify the output: + ?sort => Sort the output by a given metric. Default is time. + See + http://docs.python.org/2/library/profile.html#pstats.Stats.sort_stats + for all sort options. + ?count => The number of rows to display. Default is 100. + ?download => Download profile file suitable for visualization. For example + in snakeviz or RunSnakeRun + This is adapted from an example found here: + http://www.slideshare.net/zeeg/django-con-high-performance-django-presentation. + """ + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if "prof" not in request.GET: + return self.get_response(request) + import profile + import pstats + + profiler = profile.Profile() + response = profiler.runcall(self.get_response, request) + profiler.create_stats() + if "prof-download" in request.GET: + import marshal + + output = marshal.dumps(profiler.stats) + response = http.HttpResponse( + output, content_type="application/octet-stream" + ) + response["Content-Disposition"] = "attachment; filename=view.prof" + response["Content-Length"] = len(output) + stream = io.StringIO() + stats = pstats.Stats(profiler, stream=stream) + + stats.sort_stats(request.GET.get("prof-sort", "cumtime")) + stats.print_stats(int(request.GET.get("count", 100))) + + response = http.HttpResponse("
%s
" % stream.getvalue()) + + return response diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index 49d22fa85..8832c74c7 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -436,7 +436,7 @@ class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet): return self.exclude(pk__in=matches) def with_playable_uploads(self, actor): - uploads = Upload.objects.playable_by(actor).select_related("track") + uploads = Upload.objects.playable_by(actor) return self.prefetch_related( models.Prefetch("uploads", queryset=uploads, to_attr="playable_uploads") ) @@ -594,7 +594,8 @@ class Track(APIModelMixin): @property def listen_url(self): - return reverse("api:v1:listen-detail", kwargs={"uuid": self.uuid}) + # Not using reverse because this is slow + return "/api/v1/listen/{}/".format(self.uuid) @property def local_license(self): diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index 77854d437..9ea2dd2cb 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -45,26 +45,21 @@ class LicenseSerializer(serializers.Serializer): return obj["identifiers"][0] -class ArtistAlbumSerializer(serializers.ModelSerializer): +class ArtistAlbumSerializer(serializers.Serializer): tracks_count = serializers.SerializerMethodField() cover = cover_field is_playable = serializers.SerializerMethodField() + is_local = serializers.BooleanField() + id = serializers.IntegerField() + fid = serializers.URLField() + mbid = serializers.UUIDField() + title = serializers.CharField() + artist = serializers.SerializerMethodField() + release_date = serializers.DateField() + creation_date = serializers.DateTimeField() - class Meta: - model = models.Album - fields = ( - "id", - "fid", - "mbid", - "title", - "artist", - "release_date", - "cover", - "creation_date", - "tracks_count", - "is_playable", - "is_local", - ) + def get_artist(self, o): + return o.artist_id def get_tracks_count(self, o): return o._tracks_count @@ -76,26 +71,20 @@ class ArtistAlbumSerializer(serializers.ModelSerializer): return None -class ArtistWithAlbumsSerializer(serializers.ModelSerializer): - albums = ArtistAlbumSerializer(many=True, read_only=True) +DATETIME_FIELD = serializers.DateTimeField() + + +class ArtistWithAlbumsSerializer(serializers.Serializer): + albums = ArtistAlbumSerializer(many=True) tags = serializers.SerializerMethodField() attributed_to = serializers.SerializerMethodField() tracks_count = serializers.SerializerMethodField() - - class Meta: - model = models.Artist - fields = ( - "id", - "fid", - "mbid", - "name", - "creation_date", - "albums", - "is_local", - "tags", - "attributed_to", - "tracks_count", - ) + id = serializers.IntegerField() + fid = serializers.URLField() + mbid = serializers.UUIDField() + name = serializers.CharField() + creation_date = serializers.DateTimeField() + is_local = serializers.BooleanField() def get_tags(self, obj): tagged_items = getattr(obj, "_prefetched_tagged_items", []) @@ -114,9 +103,7 @@ def serialize_artist_simple(artist): "fid": artist.fid, "mbid": str(artist.mbid), "name": artist.name, - "creation_date": serializers.DateTimeField().to_representation( - artist.creation_date - ), + "creation_date": DATETIME_FIELD.to_representation(artist.creation_date), "is_local": artist.is_local, } @@ -129,9 +116,7 @@ def serialize_album_track(track): "title": track.title, "artist": serialize_artist_simple(track.artist), "album": track.album_id, - "creation_date": serializers.DateTimeField().to_representation( - track.creation_date - ), + "creation_date": DATETIME_FIELD.to_representation(track.creation_date), "position": track.position, "disc_number": track.disc_number, "uploads": [ @@ -145,31 +130,22 @@ def serialize_album_track(track): } -class AlbumSerializer(serializers.ModelSerializer): +class AlbumSerializer(serializers.Serializer): tracks = serializers.SerializerMethodField() artist = serializers.SerializerMethodField() cover = cover_field is_playable = serializers.SerializerMethodField() tags = serializers.SerializerMethodField() attributed_to = serializers.SerializerMethodField() - - class Meta: - model = models.Album - fields = ( - "id", - "fid", - "mbid", - "title", - "artist", - "tracks", - "release_date", - "cover", - "creation_date", - "is_playable", - "is_local", - "tags", - "attributed_to", - ) + id = serializers.IntegerField() + fid = serializers.URLField() + mbid = serializers.UUIDField() + title = serializers.CharField() + artist = serializers.SerializerMethodField() + release_date = serializers.DateField() + creation_date = serializers.DateTimeField() + is_local = serializers.BooleanField() + is_playable = serializers.SerializerMethodField() get_attributed_to = serialize_attributed_to @@ -227,7 +203,7 @@ def serialize_upload(upload): } -class TrackSerializer(serializers.ModelSerializer): +class TrackSerializer(serializers.Serializer): artist = serializers.SerializerMethodField() album = TrackAlbumSerializer(read_only=True) uploads = serializers.SerializerMethodField() @@ -235,26 +211,17 @@ class TrackSerializer(serializers.ModelSerializer): tags = serializers.SerializerMethodField() attributed_to = serializers.SerializerMethodField() - class Meta: - model = models.Track - fields = ( - "id", - "fid", - "mbid", - "title", - "album", - "artist", - "creation_date", - "position", - "disc_number", - "uploads", - "listen_url", - "copyright", - "license", - "is_local", - "tags", - "attributed_to", - ) + id = serializers.IntegerField() + fid = serializers.URLField() + mbid = serializers.UUIDField() + title = serializers.CharField() + artist = serializers.SerializerMethodField() + creation_date = serializers.DateTimeField() + is_local = serializers.BooleanField() + position = serializers.IntegerField() + disc_number = serializers.IntegerField() + copyright = serializers.CharField() + license = serializers.SerializerMethodField() get_attributed_to = serialize_attributed_to @@ -271,6 +238,9 @@ class TrackSerializer(serializers.ModelSerializer): tagged_items = getattr(obj, "_prefetched_tagged_items", []) return [ti.tag.name for ti in tagged_items] + def get_license(self, o): + return o.license_id + @common_serializers.track_fields_for_update("name", "description", "privacy_level") class LibraryForOwnerSerializer(serializers.ModelSerializer): diff --git a/api/requirements/base.txt b/api/requirements/base.txt index 63b1063f6..4f87e1ba8 100644 --- a/api/requirements/base.txt +++ b/api/requirements/base.txt @@ -72,3 +72,4 @@ django-oauth-toolkit==1.2 django-storages==1.7.1 boto3<3 unicode-slugify +django-cacheops==4.2