MirrorMirror
/
funkwhale
mirror of https://dev.funkwhale.audio/funkwhale/funkwhale.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add favorite sync 

Part-of: <https://dev.funkwhale.audio/funkwhale/funkwhale/-/merge_requests/2658>
This commit is contained in:
Petitminion 2024-02-05 18:15:22 +01:00 committed by Ciarán Ainsworth
parent 5bc0171694
commit 2a364d5785
6 changed files with 272 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
api/config/settings/common.py
View File

@ -950,6 +950,16 @@ CELERY_BEAT_SCHEDULE = {
),
"options": {"expires": 60 * 60},
},
"listenbrainz.trigger_listening_sync_with_listenbrainz": {
"task": "listenbrainz.trigger_listening_sync_with_listenbrainz",
"schedule": crontab(day_of_week="*", minute="0", hour="3"),
"options": {"expires": 60 * 60 * 24},
},
"listenbrainz.trigger_favorite_sync_with_listenbrainz": {
"task": "listenbrainz.trigger_favorite_sync_with_listenbrainz",
"schedule": crontab(day_of_week="*", minute="0", hour="3"),
"options": {"expires": 60 * 60 * 24},
},
}
if env.str("TYPESENSE_API_KEY", default=None):

43
api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py
View File

@ -1,14 +1,15 @@
import liblistenbrainz
from django.utils import timezone
import funkwhale_api
import pylistenbrainz
import liblistenbrainz
from .funkwhale_startup import PLUGIN
from config import plugins
from django.utils import timezone
from funkwhale_api.history import models as history_models
from funkwhale_api.favorites import models as favorites_models
from .funkwhale_startup import PLUGIN
from . import tasks
@plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN)
def submit_listen(listening, conf, **kwargs):
@ -20,12 +21,12 @@ def submit_listen(listening, conf, **kwargs):
logger.info("Submitting listen to ListenBrainz")
client = liblistenbrainz.ListenBrainz()
client.set_auth_token(user_token)
listen = get_listen(listening.track)
listen = get_lb_listen(listening)
client.submit_single_listen(listen)
def get_listen(listening):
def get_lb_listen(listening):
track = listening.track
additional_info = {
"media_player": "Funkwhale",
@ -68,11 +69,11 @@ def submit_favorite_creation(track_favorite, conf, **kwargs):
return
logger = PLUGIN["logger"]
logger.info("Submitting favorite to ListenBrainz")
client = pylistenbrainz.ListenBrainz()
track = get_listen(track_favorite.track)
client = liblistenbrainz.ListenBrainz()
track = track_favorite.track
if not track.mbid:
logger.warning(
"This tracks doesn't have a mbid. Feedback will not be sublited to Listenbrainz"
"This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz"
)
return
client.submit_user_feedback(1, track.mbid)
@ -85,8 +86,8 @@ def submit_favorite_deletion(track_favorite, conf, **kwargs):
return
logger = PLUGIN["logger"]
logger.info("Submitting favorite deletion to ListenBrainz")
client = pylistenbrainz.ListenBrainz()
track = get_listen(track_favorite.track)
client = liblistenbrainz.ListenBrainz()
track = track_favorite.track
if not track.mbid:
logger.warning(
"This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz"
@ -109,13 +110,12 @@ def sync_listenings_from_listenbrainz(user, conf):
.filter(source="Listenbrainz")
.latest("creation_date")
.values_list("creation_date", flat=True)
)
last_ts.timestamp()
except history_models.Listening.DoesNotExist:
tasks.import_listenbrainz_listenings(user, user_name, ts=0)
).timestamp()
except funkwhale_api.history.models.Listening.DoesNotExist:
tasks.import_listenbrainz_listenings(user, user_name, 0)
return
tasks.import_listenbrainz_listenings(user, user_name, ts=last_ts)
tasks.import_listenbrainz_listenings(user, user_name, last_ts)
@plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN)
@ -129,11 +129,10 @@ def sync_favorites_from_listenbrainz(user, conf):
favorites_models.TrackFavorite.objects.filter(user=user)
.filter(source="Listenbrainz")
.latest("creation_date")
.values_list("creation_date", flat=True)
.creation_date.timestamp()
)
last_ts.timestamp()
except history_models.Listening.DoesNotExist:
tasks.import_listenbrainz_favorites(user, user_name, ts=0)
except favorites_models.TrackFavorite.DoesNotExist:
tasks.import_listenbrainz_favorites(user, user_name, 0)
return
tasks.import_listenbrainz_favorites(user, user_name, ts=last_ts)
tasks.import_listenbrainz_favorites(user, user_name, last_ts)

70
api/funkwhale_api/contrib/listenbrainz/tasks.py
View File

@ -1,5 +1,5 @@
import datetime
import pylistenbrainz
import liblistenbrainz
from django.utils import timezone
from config import plugins
@ -47,27 +47,20 @@ def trigger_favorite_sync_with_listenbrainz():
@celery.app.task(name="listenbrainz.import_listenbrainz_listenings")
def import_listenbrainz_listenings(user, user_name, ts):
client = pylistenbrainz.ListenBrainz()
listens = client.get_listens(username=user_name, min_ts=ts, count=100)
add_lb_listenings_to_db(listens, user)
new_ts = 13
last_ts = 12
while new_ts != last_ts:
last_ts = max(
listens,
key=lambda obj: datetime.datetime.fromtimestamp(
obj.listened_at, timezone.utc
),
)
listens = client.get_listens(username=user_name, min_ts=new_ts, count=100)
def import_listenbrainz_listenings(user, user_name, since):
client = liblistenbrainz.ListenBrainz()
response = client.get_listens(username=user_name, min_ts=since, count=100)
listens = response["payload"]["listens"]
while listens:
add_lb_listenings_to_db(listens, user)
new_ts = max(
listens,
key=lambda obj: datetime.datetime.fromtimestamp(
obj.listened_at, timezone.utc
),
)
add_lb_listenings_to_db(listens, user)
response = client.get_listens(username=user_name, min_ts=new_ts, count=100)
listens = response["payload"]["listens"]
def add_lb_listenings_to_db(listens, user):
@ -119,46 +112,51 @@ def add_lb_listenings_to_db(listens, user):
@celery.app.task(name="listenbrainz.import_listenbrainz_favorites")
def import_listenbrainz_favorites(user, user_name, last_sync):
client = pylistenbrainz.ListenBrainz()
last_ts = int(datetime.datetime.now(timezone.utc).timestamp())
def import_listenbrainz_favorites(user, user_name, since):
client = liblistenbrainz.ListenBrainz()
response = client.get_user_feedback(username=user_name)
offset = 0
while last_ts >= last_sync:
feedbacks = client.get_user_feedback(username=user_name, offset=offset)
add_lb_feedback_to_db(feedbacks, user)
offset = feedbacks.count
last_ts = max(
feedbacks.feedback,
key=lambda obj: datetime.datetime.fromtimestamp(obj.created, timezone.utc),
)
# to do implement offset in pylb
while response["feedback"]:
count = response["count"]
offset = offset + count
last_sync = min(
response["feedback"],
key=lambda obj: datetime.datetime.fromtimestamp(
obj["created"], timezone.utc
),
)["created"]
add_lb_feedback_to_db(response["feedback"], user)
if last_sync <= since or count == 0:
return
response = client.get_user_feedback(username=user_name, offset=offset)
def add_lb_feedback_to_db(feedbacks, user):
logger = PLUGIN["logger"]
fw_listens = []
for feedback in feedbacks.feedback:
for feedback in feedbacks:
try:
track = music_models.Track.objects.get(mbid=feedback.recording_mbid)
track = music_models.Track.objects.get(mbid=feedback["recording_mbid"])
except music_models.Track.DoesNotExist:
logger.info(
"Received feedback track doesn't exist in fw database. Skipping..."
)
continue
if feedback.score == 1:
if feedback["score"] == 1:
favorites_models.TrackFavorite.objects.get_or_create(
user=user,
creation_date=datetime.datetime.fromtimestamp(
feedback.created, timezone.utc
feedback["created"], timezone.utc
),
track=track,
source="Listenbrainz",
)
elif feedback.score == 0:
elif feedback["score"] == 0:
try:
favorites_models.TrackFavorite.objects.delete(user=user, track=track)
favorites_models.TrackFavorite.objects.get(
user=user, track=track
).delete()
except favorites_models.TrackFavorite.DoesNotExist:
continue
elif feedback.score == -1:
elif feedback["score"] == -1:
logger.info("Funkwhale doesn't support hate yet <3")

2
api/funkwhale_api/favorites/factories.py
View File

@ -1,4 +1,5 @@
import factory
from django.utils import timezone
from funkwhale_api.factories import NoUpdateOnCreate, registry
from funkwhale_api.music.factories import TrackFactory
@ -9,6 +10,7 @@ from funkwhale_api.users.factories import UserFactory
class TrackFavorite(NoUpdateOnCreate, factory.django.DjangoModelFactory):
track = factory.SubFactory(TrackFactory)
user = factory.SubFactory(UserFactory)
creation_date = factory.Faker("date_time_this_decade", tzinfo=timezone.utc)
class Meta:
model = "favorites.TrackFavorite"

10
api/tests/conftest.py
View File

@ -277,16 +277,6 @@ def disabled_musicbrainz(mocker):
)
# @pytest.fixture()
# def disabled_listenbrainz(mocker):
# # we ensure no listenbrainz requests gets out
# yield mocker.patch.object(
# listenbrainz.client.ListenBrainzClient,
# "_submit",
# return_value=None,
# )
@pytest.fixture(autouse=True)
def r_mock(requests_mock):
"""

238
api/tests/contrib/listenbrainz/test_listenbrainz.py
View File

@ -2,7 +2,7 @@ import datetime
import logging
import pytest
import pylistenbrainz
import liblistenbrainz
from django.urls import reverse
from django.utils import timezone
@ -62,32 +62,47 @@ def test_sync_listenings_from_listenbrainz(factories, mocker, caplog):
"sync_listenings": True,
}
listens = [
pylistenbrainz.Listen(
track_name="test",
artist_name="artist_test",
recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476",
additional_info={"submission_client": "not funkwhale"},
listened_at=-3124224000,
),
pylistenbrainz.Listen(
track_name="test2",
artist_name="artist_test2",
recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559",
additional_info={"submission_client": "Funkwhale ListenBrainz plugin"},
listened_at=1871,
),
pylistenbrainz.Listen(
track_name="test3",
artist_name="artist_test3",
listened_at=0,
),
]
listens = {
"payload": {
"count": 25,
"user_id": "-- the MusicBrainz ID of the user --",
"listens": [
liblistenbrainz.Listen(
track_name="test",
artist_name="artist_test",
recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476",
additional_info={"submission_client": "not funkwhale"},
listened_at=-3124224000,
),
liblistenbrainz.Listen(
track_name="test2",
artist_name="artist_test2",
recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559",
additional_info={
"submission_client": "Funkwhale ListenBrainz plugin"
},
listened_at=1871,
),
liblistenbrainz.Listen(
track_name="test3",
artist_name="artist_test3",
listened_at=0,
),
],
}
}
no_more_listen = {
"payload": {
"count": 25,
"user_id": "Bilbo",
"listens": [],
}
}
mocker.patch.object(
funkwhale_ready.tasks.pylistenbrainz.ListenBrainz,
funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
"get_listens",
return_value=listens,
side_effect=[listens, no_more_listen],
)
funkwhale_ready.sync_listenings_from_listenbrainz(user, conf)
@ -105,13 +120,20 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
caplog.set_level(logging.INFO)
logger.addHandler(caplog.handler)
user = factories["users.User"]()
# track lb fav
factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
# random track
factories["music.Track"]()
# track lb neutral
track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
favorite = factories["favorites.TrackFavorite"](track=track)
favorite = factories["favorites.TrackFavorite"](track=track, user=user)
# last_sync
track_last_sync = factories["music.Track"](
mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7"
)
favorite_last_sync = factories["favorites.TrackFavorite"](
track=track_last_sync, source="Listenbrainz"
)
conf = {
"user_name": user.username,
@ -135,6 +157,7 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
"user_id": user.username,
},
{
# last sync
"created": 1690775094,
"recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7",
"score": -1,
@ -142,7 +165,7 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
},
{
"created": 1690775093,
"recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2 ",
"recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2",
"score": 1,
"user_id": user.username,
},
@ -150,10 +173,11 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
"offset": 0,
"total_count": 4,
}
empty_feedback = {"count": 0, "feedback": [], "offset": 0, "total_count": 0}
mocker.patch.object(
funkwhale_ready.tasks.pylistenbrainz.ListenBrainz,
funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
"get_user_feedback",
return_value=feedbacks,
side_effect=[feedbacks, empty_feedback],
)
funkwhale_ready.sync_favorites_from_listenbrainz(user, conf)
@ -161,5 +185,153 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
assert favorites_models.TrackFavorite.objects.filter(
track__mbid="195565db-65f9-4d0d-b347-5f0c85509528"
).exists()
with pytest.raises(deleted.DoesNotExist):
with pytest.raises(favorites_models.TrackFavorite.DoesNotExist):
favorite.refresh_from_db()
def test_sync_favorites_from_listenbrainz_since(factories, mocker, caplog):
logger = logging.getLogger("plugins")
caplog.set_level(logging.INFO)
logger.addHandler(caplog.handler)
user = factories["users.User"]()
# track lb fav
factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
# track lb neutral
track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
favorite = factories["favorites.TrackFavorite"](track=track, user=user)
# track should be not synced
factories["music.Track"](mbid="1fd02cf2-7247-4715-8862-c378ec196000")
# last_sync
track_last_sync = factories["music.Track"](
mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7"
)
factories["favorites.TrackFavorite"](
track=track_last_sync,
user=user,
source="Listenbrainz",
creation_date=datetime.datetime.fromtimestamp(1690775094),
)
conf = {
"user_name": user.username,
"user_token": "user_tolkien",
"sync_favorites": True,
}
feedbacks = {
"count": 5,
"feedback": [
{
"created": 1701116226,
"recording_mbid": "195565db-65f9-4d0d-b347-5f0c85509528",
"score": 1,
"user_id": user.username,
},
{
"created": 1701116214,
"recording_mbid": "c5af5351-dbbf-4481-b52e-a480b6c57986",
"score": 0,
"user_id": user.username,
},
{
# last sync
"created": 1690775094,
"recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7",
"score": -1,
"user_id": user.username,
},
{
"created": 1690775093,
"recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2",
"score": 1,
"user_id": user.username,
},
],
"offset": 0,
"total_count": 4,
}
second_feedback = {
"count": 0,
"feedback": [
{
"created": 0,
"recording_mbid": "1fd02cf2-7247-4715-8862-c378ec196000",
"score": 1,
"user_id": user.username,
},
],
"offset": 0,
"total_count": 0,
}
mocker.patch.object(
funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
"get_user_feedback",
side_effect=[feedbacks, second_feedback],
)
funkwhale_ready.sync_favorites_from_listenbrainz(user, conf)
assert favorites_models.TrackFavorite.objects.filter(
track__mbid="195565db-65f9-4d0d-b347-5f0c85509528"
).exists()
assert not favorites_models.TrackFavorite.objects.filter(
track__mbid="1fd02cf2-7247-4715-8862-c378ec196000"
).exists()
with pytest.raises(favorites_models.TrackFavorite.DoesNotExist):
favorite.refresh_from_db()
def test_submit_favorites_to_listenbrainz(factories, mocker, caplog):
logger = logging.getLogger("plugins")
caplog.set_level(logging.INFO)
logger.addHandler(caplog.handler)
user = factories["users.User"]()
factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
favorite = factories["favorites.TrackFavorite"](track=track)
conf = {
"user_name": user.username,
"user_token": "user_tolkien",
"submit_favorites": True,
}
patch = mocker.patch.object(
funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
"submit_user_feedback",
return_value="Success",
)
funkwhale_ready.submit_favorite_creation(favorite, conf)
patch.assert_called_once_with(1, track.mbid)
def test_submit_favorites_deletion(factories, mocker, caplog):
logger = logging.getLogger("plugins")
caplog.set_level(logging.INFO)
logger.addHandler(caplog.handler)
user = factories["users.User"]()
factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
favorite = factories["favorites.TrackFavorite"](track=track)
conf = {
"user_name": user.username,
"user_token": "user_tolkien",
"submit_favorites": True,
}
patch = mocker.patch.object(
funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
"submit_user_feedback",
return_value="Success",
)
funkwhale_ready.submit_favorite_deletion(favorite, conf)
patch.assert_called_once_with(0, track.mbid)