implement listening and favorite sync with listenbrainz

Part-of: <https://dev.funkwhale.audio/funkwhale/funkwhale/-/merge_requests/2658>

api/config/plugins.py

@@ -303,6 +303,23 @@ LISTENING_CREATED = "listening_created"
 """
 Called when a track is being listened
 """
+LISTENING_SYNC = "listening_sync"
+"""
+Called by the task manager to trigger listening sync
+"""
+FAVORITE_CREATED = "favorite_created"
+"""
+Called when a track is being liked
+"""
+FAVORITE_DELETED = "favorite_deleted"
+"""
+Called when a favorite track is being unliked
+"""
+FAVORITE_SYNC = "favorite_sync"
+"""
+Called by the task manager to trigger favorite sync
+"""
+
 SCAN = "scan"
 """
 

api/config/settings/common.py

@@ -276,6 +276,7 @@ LOCAL_APPS = (
     # Your stuff: custom apps go here
     "funkwhale_api.instance",
     "funkwhale_api.audio",
+    "funkwhale_api.contrib.listenbrainz",
     "funkwhale_api.music",
     "funkwhale_api.requests",
     "funkwhale_api.favorites",

api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py

@@ -2,23 +2,40 @@ import liblistenbrainz
 from django.utils import timezone
 
 import funkwhale_api
-from config import plugins
+import pylistenbrainz
 
+<<<<<<< HEAD
+=======
+from config import plugins
+from django.utils import timezone
+
+from . import tasks
+>>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz)
 from .funkwhale_startup import PLUGIN
 
+from funkwhale_api.history import models as history_models
+from funkwhale_api.favorites import models as favorites_models
+
 
 @plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN)
 def submit_listen(listening, conf, **kwargs):
     user_token = conf["user_token"]
-    if not user_token:
+    if not user_token and not conf["submit_listenings"]:
         return
 
     logger = PLUGIN["logger"]
     logger.info("Submitting listen to ListenBrainz")
+<<<<<<< HEAD
     client = liblistenbrainz.ListenBrainz()
     client.set_auth_token(user_token)
     listen = get_listen(listening.track)
 
+=======
+
+    listen = get_listen(listening.track)
+    client = pylistenbrainz.ListenBrainz()
+    client.set_auth_token(user_token)
+>>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz)
     client.submit_single_listen(listen)
 
 
@@ -48,10 +65,91 @@ def get_listen(track):
     if upload:
         additional_info["duration"] = upload.duration
 
+<<<<<<< HEAD
     return liblistenbrainz.Listen(
+=======
+    return pylistenbrainz.Listen(
+>>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz)
         track_name=track.title,
         artist_name=track.artist.name,
         listened_at=int(timezone.now()),
         release_name=release_name,
         additional_info=additional_info,
     )
+<<<<<<< HEAD
+=======
+
+
+@plugins.register_hook(plugins.FAVORITE_CREATED, PLUGIN)
+def submit_favorite_creation(track_favorite, conf, **kwargs):
+    user_token = conf["user_token"]
+    if not user_token or not conf["submit_favorites"]:
+        return
+    logger = PLUGIN["logger"]
+    logger.info("Submitting favorite to ListenBrainz")
+    client = pylistenbrainz.ListenBrainz()
+    track = get_listen(track_favorite.track)
+    if not track.mbid:
+        logger.warning(
+            "This tracks doesn't have a mbid. Feedback will not be sublited to Listenbrainz"
+        )
+        return
+    # client.feedback(track, 1)
+
+
+@plugins.register_hook(plugins.FAVORITE_DELETED, PLUGIN)
+def submit_favorite_deletion(track_favorite, conf, **kwargs):
+    user_token = conf["user_token"]
+    if not user_token or not conf["submit_favorites"]:
+        return
+    logger = PLUGIN["logger"]
+    logger.info("Submitting favorite deletion to ListenBrainz")
+    client = pylistenbrainz.ListenBrainz()
+    track = get_listen(track_favorite.track)
+    if not track.mbid:
+        logger.warning(
+            "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz"
+        )
+        return
+    # client.feedback(track, 0)
+
+
+@plugins.register_hook(plugins.LISTENING_SYNC, PLUGIN)
+def sync_listenings_from_listenbrainz(user, conf):
+    user_name = conf["user_name"]
+    user_token = conf["user_token"]
+
+    if not user_name or not conf["sync_listenings"]:
+        return
+    logger = PLUGIN["logger"]
+    logger.info("Getting listenings from ListenBrainz")
+    try:
+        last_ts = (
+            history_models.Listening.objects.filter(user=user)
+            .filter(from_listenbrainz=True)
+            .latest("creation_date")
+            .values_list("creation_date", flat=True)
+        )
+    except history_models.Listening.DoesNotExist:
+        tasks.import_listenbrainz_listenings(user, user_name, ts=0)
+
+    tasks.import_listenbrainz_listenings(user, user_name, ts=last_ts)
+
+
+@plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN)
+def sync_favorites_from_listenbrainz(user, conf):
+    user_name = conf["user_name"]
+    user_token = conf["user_token"]
+
+    if not user_name or not conf["sync_favorites"]:
+        return
+    try:
+        last_ts = (
+            favorites_models.TrackFavorite.objects.filter(user=user)
+            .filter(from_listenbrainz=True)
+            .latest("creation_date")
+            .values_list("creation_date", flat=True)
+        )
+    except history_models.Listening.DoesNotExist:
+        tasks.import_listenbrainz_favorites(user, user_name, last_ts)
+>>>>>>> bf0c861a0 (implement listening and favorite sync with listenbrainz)

api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py

@@ -3,7 +3,7 @@ from config import plugins
 PLUGIN = plugins.get_plugin_config(
     name="listenbrainz",
     label="ListenBrainz",
-    description="A plugin that allows you to submit your listens to ListenBrainz.",
+    description="A plugin that allows you to submit or sync your listens and favorites to ListenBrainz.",
     homepage="https://docs.funkwhale.audio/users/builtinplugins.html#listenbrainz-plugin",  # noqa
     version="0.3",
     user=True,
@@ -13,6 +13,45 @@ PLUGIN = plugins.get_plugin_config(
             "type": "text",
             "label": "Your ListenBrainz user token",
             "help": "You can find your user token in your ListenBrainz profile at https://listenbrainz.org/profile/",
-        }
+        },
+        {
+            "name": "user_name",
+            "type": "text",
+            "required": False,
+            "label": "Your ListenBrainz user name.",
+            "help": "It's needed for synchronisation with Listenbrainz (import listenings and favorites) \
+                but not to send activities",
+        },
+        {
+            "name": "submit_listenings",
+            "type": "boolean",
+            "default": True,
+            "label": "Enable listenings submission to Listenbrainz",
+            "help": "If enable, your listening from Funkwhale will be imported into ListenBrainz.",
+        },
+        {
+            "name": "sync_listenings",
+            "type": "boolean",
+            "default": False,
+            "label": "Enable listenings sync",
+            "help": "If enable, your listening from Listenbrainz will be imported into Funkwhale. This means they \
+                will be used has any other funkwhale listenings to filter out recently listened content or \
+                generate recomendations",
+        },
+        {
+            "name": "sync_facorites",
+            "type": "boolean",
+            "default": False,
+            "label": "Enable favorite sync",
+            "help": "If enable, your favorites from Listenbrainz will be imported into Funkwhale. This means they \
+                will be used has any other funkwhale favorites (Ui display, federatipon activity)",
+        },
+        {
+            "name": "submit_favorites",
+            "type": "boolean",
+            "default": False,
+            "label": "Enable favorite submition to Listenbrainz services",
+            "help": "If enable, your favorites from Funkwhale will be submit to Listenbrainz",
+        },
     ],
 )

api/funkwhale_api/contrib/listenbrainz/tasks.py

@@ -0,0 +1,90 @@
+import datetime
+import pylistenbrainz
+
+from config import plugins
+from django.utils import timezone
+
+from funkwhale_api.users import models
+from funkwhale_api.taskapp import celery
+from funkwhale_api.history import models as history_models
+from funkwhale_api.music import models as music_models
+
+
+@celery.app.task(name="listenbrainz.trigger_listening_sync_with_listenbrainz")
+def trigger_listening_sync_with_listenbrainz():
+    now = timezone.now()
+    active_month = now - datetime.timedelta(days=30)
+    users = (
+        models.User.objects.filter(plugins__code="listenbrainz")
+        .filter(plugins__conf__sync_listenings=True)
+        .filter(last_activity__gte=active_month)
+    )
+    for user in users:
+        plugins.trigger_hook(
+            plugins.LISTENING_SYNC,
+            user=user,
+            confs=plugins.get_confs(user),
+        )
+
+
+@celery.app.task(name="listenbrainz.trigger_favorite_sync_with_listenbrainz")
+def trigger_favorite_sync_with_listenbrainz():
+    now = timezone.now()
+    active_month = now - datetime.timedelta(days=30)
+    users = (
+        models.User.objects.filter(plugins__code="listenbrainz")
+        .filter(plugins__conf__sync_listenings=True)
+        .filter(last_activity__gte=active_month)
+    )
+    for user in users:
+        plugins.trigger_hook(
+            plugins.FAVORITE_SYNC,
+            user=user,
+            confs=plugins.get_confs(user),
+        )
+
+
+@celery.app.task(name="listenbrainz.import_listenbrainz_listenings")
+def import_listenbrainz_listenings(user, user_name, ts):
+    client = pylistenbrainz.ListenBrainz()
+    listens = client.get_listens(username=user_name, min_ts=ts, count=100)
+    add_lb_listenings_to_db(listens, user)
+    new_ts = 13
+    last_ts = 12
+    while new_ts != last_ts:
+        last_ts = listens[0].listened_at
+        listens = client.get_listens(username=user_name, min_ts=new_ts, count=100)
+        new_ts = listens[0].listened_at
+        add_lb_listenings_to_db(listens, user)
+
+
+def add_lb_listenings_to_db(listens, user):
+    fw_listens = []
+    for listen in listens:
+        if (
+            listen.additional_info.get("submission_client")
+            and listen.additional_info.get("submission_client")
+            == "Funkwhale ListenBrainz plugin"
+        ):
+            continue
+        try:
+            track = music_models.Track.objects.get(mbid=listen.recording_mbid)
+        except music_models.Track.DoesNotExist:
+            # to do : resolve non mbid listens ?
+            continue
+
+        user = user
+        fw_listen = history_models.Listening(
+            creation_date=listen.listened_at,
+            track=track,
+            user=user,
+            from_listenbrainz=True,
+        )
+        fw_listens.append(fw_listen)
+
+    history_models.Listening.objects.bulk_create(fw_listens)
+
+
+@celery.app.task(name="listenbrainz.import_listenbrainz_favorites")
+def import_listenbrainz_favorites():
+    return "to do"

api/funkwhale_api/favorites/migrations/0002_trackfavorite_from_listenbrainz.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-11-29 15:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('favorites', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='trackfavorite',
+            name='from_listenbrainz',
+            field=models.BooleanField(default=None, null=True),
+        ),
+    ]

api/funkwhale_api/favorites/models.py

@@ -12,6 +12,7 @@ class TrackFavorite(models.Model):
     track = models.ForeignKey(
         Track, related_name="track_favorites", on_delete=models.CASCADE
     )
+    from_listenbrainz = models.BooleanField(default=None, null=True)
 
     class Meta:
         unique_together = ("track", "user")

api/funkwhale_api/favorites/views.py

@@ -1,3 +1,5 @@
+from config import plugins
+
 from django.db.models import Prefetch
 from drf_spectacular.utils import extend_schema
 from rest_framework import mixins, status, viewsets
@@ -44,6 +46,11 @@ class TrackFavoriteViewSet(
         instance = self.perform_create(serializer)
         serializer = self.get_serializer(instance=instance)
         headers = self.get_success_headers(serializer.data)
+        plugins.trigger_hook(
+            plugins.FAVORITE_CREATED,
+            track_favorite=serializer.instance,
+            confs=plugins.get_confs(self.request.user),
+        )
         record.send(instance)
         return Response(
             serializer.data, status=status.HTTP_201_CREATED, headers=headers
@@ -76,6 +83,11 @@ class TrackFavoriteViewSet(
         except (AttributeError, ValueError, models.TrackFavorite.DoesNotExist):
             return Response({}, status=400)
         favorite.delete()
+        plugins.trigger_hook(
+            plugins.FAVORITE_DELETED,
+            track_favorite=favorite,
+            confs=plugins.get_confs(self.request.user),
+        )
         return Response([], status=status.HTTP_204_NO_CONTENT)
 
     @extend_schema(

api/funkwhale_api/history/migrations/0003_listening_from_listenbrainz.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-11-29 15:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('history', '0002_auto_20180325_1433'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='listening',
+            name='from_listenbrainz',
+            field=models.BooleanField(default=None, null=True),
+        ),
+    ]

api/funkwhale_api/history/models.py

@@ -17,6 +17,7 @@ class Listening(models.Model):
         on_delete=models.CASCADE,
     )
     session_key = models.CharField(max_length=100, null=True, blank=True)
+    from_listenbrainz = models.BooleanField(default=None, null=True)
 
     class Meta:
         ordering = ("-creation_date",)

api/funkwhale_api/typesense/utils.py

@@ -36,7 +36,6 @@ def delete_non_alnum_characters(text):
 def resolve_recordings_to_fw_track(recordings):
     """
     Tries to match a troi recording entity to a fw track using the typesense index.
-    It will save the results in the match_mbid attribute of the Track table.
     For test purposes : if multiple fw tracks are returned, we log the information
     but only keep the best result in db to avoid duplicates.
     """

api/tests/contrib/listenbrainz/test_listenbrainz.py

@@ -0,0 +1,35 @@
+import pytest
+from django.urls import reverse
+from config import plugins
+from funkwhale_api.history import models as history_models
+
+
+def test_listenbrainz_submit_listen(logged_in_client, mocker, factories):
+    plugin = plugins.get_plugin_config(
+        name="listenbrainz",
+        description="A plugin that allows you to submit or sync your listens and favorites to ListenBrainz.",
+        conf=[],
+        source=False,
+    )
+    handler = mocker.Mock()
+    plugins.register_hook(plugins.LISTENING_CREATED, plugin)(handler)
+    plugins.set_conf(
+        "listenbrainz",
+        {
+            "sync_listenings": True,
+            "sync_facorites": True,
+            "submit_favorites": True,
+            "sync_favorites": True,
+            "user_token": "blablabla",
+        },
+        user=logged_in_client.user,
+    )
+    plugins.enable_conf("listenbrainz", True, logged_in_client.user)
+
+    track = factories["music.Track"]()
+    url = reverse("api:v1:history:listenings-list")
+    logged_in_client.post(url, {"track": track.pk})
+    response = logged_in_client.get(url)
+    listening = history_models.Listening.objects.get(user=logged_in_client.user)
+    handler.assert_called_once_with(listening=listening, conf=None)
+    # why conf=none ?

api/tests/contrib/listenbrainz/test_tasks.py

@@ -0,0 +1,48 @@
+import datetime
+import pytest
+
+import pylistenbrainz
+from funkwhale_api.contrib.listenbrainz import tasks
+from funkwhale_api.history import models as history_models
+
+
+def test_import_listenbrainz_listenings(factories, mocker):
+    factories["music.Track"](mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476")
+    factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
+
+    listens = [
+        pylistenbrainz.utils.Listen(
+            track_name="test",
+            artist_name="artist_test",
+            recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476",
+            additional_info={"submission_client": "not funkwhale"},
+            listened_at=datetime.datetime.fromtimestamp(-3124224000),
+        ),
+        pylistenbrainz.utils.Listen(
+            track_name="test2",
+            artist_name="artist_test2",
+            recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559",
+            additional_info={"submission_client": "Funkwhale ListenBrainz plugin"},
+            listened_at=datetime.datetime.fromtimestamp(1871),
+        ),
+        pylistenbrainz.utils.Listen(
+            track_name="test3",
+            artist_name="artist_test3",
+            listened_at=0,
+        ),
+    ]
+
+    mocker.patch.object(
+        tasks.pylistenbrainz.ListenBrainz, "get_listens", return_value=listens
+    )
+    user = factories["users.User"]()
+
+    tasks.import_listenbrainz_listenings(user, "user_name", ts=0)
+
+    history_models.Listening.objects.filter(
+        track__mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476"
+    ).exists()
+
+    assert not history_models.Listening.objects.filter(
+        track__mbid="54c60860-f43d-484e-b691-7ab7ec8de559"
+    ).exists()