From 0cb34573787e1c70e832594a182184f98259fe8a Mon Sep 17 00:00:00 2001 From: Petitminion Date: Thu, 30 Jun 2022 18:04:04 +0200 Subject: [PATCH] New task checking if remote instance is reachable to avoid playback latence --- api/config/settings/common.py | 10 ++++++ api/funkwhale_api/federation/factories.py | 2 ++ .../migrations/0028_auto_20221027_1141.py | 23 +++++++++++++ api/funkwhale_api/federation/models.py | 3 +- api/funkwhale_api/federation/tasks.py | 32 +++++++++++++++++++ api/funkwhale_api/music/models.py | 3 ++ api/tests/federation/test_tasks.py | 19 +++++++++++ changes/changelog.d/1711.enhancement | 1 + 8 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 api/funkwhale_api/federation/migrations/0028_auto_20221027_1141.py create mode 100644 changes/changelog.d/1711.enhancement diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 2b82688d2..b24cad2c0 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -862,6 +862,16 @@ CELERY_BEAT_SCHEDULE = { "schedule": crontab(day_of_week="1", minute="0", hour="2"), "options": {"expires": 60 * 60 * 24}, }, + "federation.check_remote_instance_availability": { + "task": "federation.check_remote_instance_availability", + "schedule": crontab( + **env.dict( + "SCHEDULE_FEDERATION_CHECK_INTANCES_AVAILABILITY", + default={"minute": "0", "hour": "*"}, + ) + ), + "options": {"expires": 60 * 60}, + }, } if env.bool("ADD_ALBUM_TAGS_FROM_TRACKS", default=True): diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index 2d2a2abae..852998c9e 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -70,6 +70,8 @@ class DomainFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): name = factory.Faker("domain_name") nodeinfo_fetch_date = factory.LazyFunction(lambda: timezone.now()) allowed = None + reachable = True + last_successful_contact = None class Meta: model = "federation.Domain" diff --git a/api/funkwhale_api/federation/migrations/0028_auto_20221027_1141.py b/api/funkwhale_api/federation/migrations/0028_auto_20221027_1141.py new file mode 100644 index 000000000..aaa275d56 --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0028_auto_20221027_1141.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.16 on 2022-10-27 11:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0027_auto_20220627_1915'), + ] + + operations = [ + migrations.AddField( + model_name='domain', + name='last_successful_contact', + field=models.DateTimeField(default=None, null=True), + ), + migrations.AddField( + model_name='domain', + name='reachable', + field=models.BooleanField(default=True), + ), + ] diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index 125b61353..58a6acf54 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -123,7 +123,8 @@ class Domain(models.Model): ) # are interactions with this domain allowed (only applies when allow-listing is on) allowed = models.BooleanField(default=None, null=True) - + reachable = models.BooleanField(default=True) + last_successful_contact = models.DateTimeField(default=None, null=True) objects = DomainQuerySet.as_manager() def __str__(self): diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index bf3440524..8c94ead41 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -625,3 +625,35 @@ def fetch_collection(url, max_pages, channel, is_page=False): results["errored"], ) return results + + +@celery.app.task(name="federation.check_all_remote_instance_availability") +def check_all_remote_instance_availability(): + domains = models.Domain.objects.all().prefetch_related() + for domain in domains: + check_single_remote_instance_availability(domain) + + +@celery.app.task(name="federation.check_single_remote_instance_availability") +def check_single_remote_instance_availability(domain): + try: + response = requests.get(f"https://{domain.name}/api/v1/instance/nodeinfo/2.0/") + nodeinfo = response.json() + except Exception as e: + logger.info( + f"Domain {domain.name} could not be reached because of the following error : {e}. \ + Setting domain as unreacheable." + ) + domain.reachable = False + domain.save() + + if "version" in nodeinfo.keys(): + domain.reachable = True + domain.last_successful_contact = datetime.datetime.now() + domain.save() + else: + logger.info( + f"Domain {domain.name} is not reacheable at the moment. Setting domain as unreacheable." + ) + domain.reachable = False + domain.save() diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index 076239935..37a9a5343 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -1184,12 +1184,15 @@ class LibraryQuerySet(models.QuerySet): ) .values_list("target__channel__library", flat=True) ) + domains_reachable = federation_models.Domain.objects.filter(reachable=True) + return self.filter( me_query | instance_query | models.Q(privacy_level="everyone") | models.Q(pk__in=followed_libraries) | models.Q(pk__in=followed_channels_libraries) + & models.Q(actor__domain__in=domains_reachable) ) diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py index 5ab001298..f878b7ad3 100644 --- a/api/tests/federation/test_tasks.py +++ b/api/tests/federation/test_tasks.py @@ -670,3 +670,22 @@ def test_fetch_collection(mocker, r_mock): assert result["seen"] == 7 assert result["total"] == 27094 assert result["next_page"] == payloads["page2"]["next"] + + +def test_check_all_remote_instance_reachable(factories, r_mock): + domain = factories["federation.Domain"]() + r_mock.get( + f"https://{domain.name}/api/v1/instance/nodeinfo/2.0/", json={"version": "2"} + ) + tasks.check_all_remote_instance_availability() + domain = models.Domain.objects.get(name=domain.name) + assert domain.reachable is True + + +def test_check_remote_instance_unreachable(factories, r_mock): + domain = factories["federation.Domain"]() + + r_mock.get(f"https://{domain.name}/api/v1/instance/nodeinfo/2.0/", json={}) + tasks.check_all_remote_instance_availability() + domain = models.Domain.objects.get(name=domain.name) + assert domain.reachable is False diff --git a/changes/changelog.d/1711.enhancement b/changes/changelog.d/1711.enhancement new file mode 100644 index 000000000..8d13a6f80 --- /dev/null +++ b/changes/changelog.d/1711.enhancement @@ -0,0 +1 @@ +New task checking if remote instance is reachable to avoid playback latence (#1711) \ No newline at end of file