From c20e4d7c9ab365918d4523692245faed82ce5eaf Mon Sep 17 00:00:00 2001
From: Eliot Berriot <contact@eliotberriot.com>
Date: Fri, 13 Apr 2018 21:31:40 +0200
Subject: [PATCH] Added task to delete unused cached files

---
 .../dynamic_preferences_registry.py           | 14 +++++++++
 api/funkwhale_api/federation/factories.py     |  5 ++++
 api/funkwhale_api/federation/tasks.py         | 24 +++++++++++++++
 api/tests/federation/test_tasks.py            | 30 +++++++++++++++++++
 4 files changed, 73 insertions(+)

diff --git a/api/funkwhale_api/federation/dynamic_preferences_registry.py b/api/funkwhale_api/federation/dynamic_preferences_registry.py
index c7cb015a8..43877c75c 100644
--- a/api/funkwhale_api/federation/dynamic_preferences_registry.py
+++ b/api/funkwhale_api/federation/dynamic_preferences_registry.py
@@ -4,3 +4,17 @@ from dynamic_preferences import types
 from dynamic_preferences.registries import global_preferences_registry
 
 federation = types.Section('federation')
+
+
+@global_preferences_registry.register
+class MusicCacheDuration(types.IntPreference):
+    show_in_api = True
+    section = federation
+    name = 'music_cache_duration'
+    default = 60 * 24 * 2
+    verbose_name = 'Music cache duration'
+    help_text = (
+        'How much minutes do you want to keep a copy of federated tracks'
+        'locally? Federated files that were not listened in this interval '
+        'will be erased and refetched from the remote on the next listening.'
+    )
diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py
index 1aeb733c8..0754c4b2f 100644
--- a/api/funkwhale_api/federation/factories.py
+++ b/api/funkwhale_api/federation/factories.py
@@ -185,6 +185,11 @@ class LibraryTrackFactory(factory.DjangoModelFactory):
     class Meta:
         model = models.LibraryTrack
 
+    class Params:
+        with_audio_file = factory.Trait(
+            audio_file=factory.django.FileField()
+        )
+
 
 @registry.register(name='federation.Note')
 class NoteFactory(factory.Factory):
diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py
index c6a70174f..adc354c4f 100644
--- a/api/funkwhale_api/federation/tasks.py
+++ b/api/funkwhale_api/federation/tasks.py
@@ -1,3 +1,4 @@
+import datetime
 import json
 import logging
 
@@ -5,8 +6,10 @@ from django.conf import settings
 from django.utils import timezone
 
 from requests.exceptions import RequestException
+from dynamic_preferences.registries import global_preferences_registry
 
 from funkwhale_api.common import session
+from funkwhale_api.history.models import Listening
 from funkwhale_api.taskapp import celery
 
 from . import actors
@@ -85,3 +88,24 @@ def scan_library_page(library, page_url, until=None):
     next_page = data.get('next')
     if next_page and next_page != page_url:
         scan_library_page.delay(library_id=library.id, page_url=next_page)
+
+
+@celery.app.task(name='federation.clean_music_cache')
+def clean_music_cache():
+    preferences = global_preferences_registry.manager()
+    delay = preferences['federation__music_cache_duration']
+    if delay < 1:
+        return  # cache clearing disabled
+
+    candidates = models.LibraryTrack.objects.filter(
+        audio_file__isnull=False
+    ).values_list('local_track_file__track', flat=True)
+    listenings = Listening.objects.filter(
+        creation_date__gte=timezone.now() - datetime.timedelta(minutes=delay),
+        track__pk__in=candidates).values_list('track', flat=True)
+    too_old = set(candidates) - set(listenings)
+
+    to_remove = models.LibraryTrack.objects.filter(
+        local_track_file__track__pk__in=too_old).only('audio_file')
+    for lt in to_remove:
+        lt.audio_file.delete()
diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py
index d164d62a6..506fbc1fe 100644
--- a/api/tests/federation/test_tasks.py
+++ b/api/tests/federation/test_tasks.py
@@ -1,3 +1,5 @@
+import datetime
+
 from django.core.paginator import Paginator
 from django.utils import timezone
 
@@ -108,3 +110,31 @@ def test_scan_page_stops_once_until_is_reached(
     assert len(lts) == 2
     for i, tf in enumerate(tfs[:1]):
         assert tf.creation_date == lts[i].published_date
+
+
+def test_clean_federation_music_cache_if_no_listen(preferences, factories):
+    preferences['federation__music_cache_duration'] = 60
+    lt1 = factories['federation.LibraryTrack'](with_audio_file=True)
+    lt2 = factories['federation.LibraryTrack'](with_audio_file=True)
+    lt3 = factories['federation.LibraryTrack'](with_audio_file=True)
+    tf1 = factories['music.TrackFile'](library_track=lt1)
+    tf2 = factories['music.TrackFile'](library_track=lt2)
+    tf3 = factories['music.TrackFile'](library_track=lt3)
+
+    # we listen to the first one, and the second one (but weeks ago)
+    listening1 = factories['history.Listening'](
+        track=tf1.track,
+        creation_date=timezone.now())
+    listening2 = factories['history.Listening'](
+        track=tf2.track,
+        creation_date=timezone.now() - datetime.timedelta(minutes=61))
+
+    tasks.clean_music_cache()
+
+    lt1.refresh_from_db()
+    lt2.refresh_from_db()
+    lt3.refresh_from_db()
+
+    assert bool(lt1.audio_file) is True
+    assert bool(lt2.audio_file) is False
+    assert bool(lt3.audio_file) is False