MirrorMirror
/
funkwhale
mirror of https://dev.funkwhale.audio/funkwhale/funkwhale.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

See #170: fetching remote objects

This commit is contained in:
Eliot Berriot 2020-03-02 17:23:03 +01:00
parent 65097f6297
commit c2eeee5eb1
30 changed files with 1173 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
api/config/settings/common.py
View File

@ -833,6 +833,10 @@ THROTTLING_RATES = {
"rate": THROTTLING_USER_RATES.get("password-reset-confirm", "20/h"),
"description": "Password reset confirmation",
},
"fetch": {
"rate": THROTTLING_USER_RATES.get("fetch", "200/d"),
"description": "Fetch remote objects",
},
}
@ -906,7 +910,7 @@ ACCOUNT_USERNAME_BLACKLIST = [
] + env.list("ACCOUNT_USERNAME_BLACKLIST", default=[])
EXTERNAL_REQUESTS_VERIFY_SSL = env.bool("EXTERNAL_REQUESTS_VERIFY_SSL", default=True)
EXTERNAL_REQUESTS_TIMEOUT = env.int("EXTERNAL_REQUESTS_TIMEOUT", default=5)
EXTERNAL_REQUESTS_TIMEOUT = env.int("EXTERNAL_REQUESTS_TIMEOUT", default=10)
# XXX: deprecated, see #186
API_AUTHENTICATION_REQUIRED = env.bool("API_AUTHENTICATION_REQUIRED", True)
@ -955,7 +959,11 @@ FEDERATION_OBJECT_FETCH_DELAY = env.int(
MODERATION_EMAIL_NOTIFICATIONS_ENABLED = env.bool(
"MODERATION_EMAIL_NOTIFICATIONS_ENABLED", default=True
)
FEDERATION_AUTHENTIFY_FETCHES = True
FEDERATION_SYNCHRONOUS_FETCH = env.bool("FEDERATION_SYNCHRONOUS_FETCH", default=True)
FEDERATION_DUPLICATE_FETCH_DELAY = env.int(
"FEDERATION_DUPLICATE_FETCH_DELAY", default=60 * 50
)
# Delay in days after signup before we show the "support us" messages
INSTANCE_SUPPORT_MESSAGE_DELAY = env.int("INSTANCE_SUPPORT_MESSAGE_DELAY", default=15)
FUNKWHALE_SUPPORT_MESSAGE_DELAY = env.int("FUNKWHALE_SUPPORT_MESSAGE_DELAY", default=15)

8
api/funkwhale_api/common/utils.py
View File

@ -234,9 +234,11 @@ def get_updated_fields(conf, data, obj):
data_value = data[data_field]
except KeyError:
continue
obj_value = getattr(obj, obj_field)
if obj_value != data_value:
if obj.pk:
obj_value = getattr(obj, obj_field)
if obj_value != data_value:
final_data[obj_field] = data_value
else:
final_data[obj_field] = data_value
return final_data

77
api/funkwhale_api/federation/api_serializers.py
View File

@ -1,7 +1,13 @@
import datetime
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.core import validators
from django.utils import timezone
from rest_framework import serializers
from funkwhale_api.common import fields as common_fields
from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.music import models as music_models
from funkwhale_api.users import serializers as users_serializers
@ -158,8 +164,21 @@ class InboxItemActionSerializer(common_serializers.ActionSerializer):
return objects.update(is_read=True)
FETCH_OBJECT_CONFIG = {
"artist": {"queryset": music_models.Artist.objects.all()},
"album": {"queryset": music_models.Album.objects.all()},
"track": {"queryset": music_models.Track.objects.all()},
"library": {"queryset": music_models.Library.objects.all(), "id_attr": "uuid"},
"upload": {"queryset": music_models.Upload.objects.all(), "id_attr": "uuid"},
"account": {"queryset": models.Actor.objects.all(), "id_attr": "full_username"},
}
FETCH_OBJECT_FIELD = common_fields.GenericRelation(FETCH_OBJECT_CONFIG)
class FetchSerializer(serializers.ModelSerializer):
actor = federation_serializers.APIActorSerializer()
actor = federation_serializers.APIActorSerializer(read_only=True)
object = serializers.CharField(write_only=True)
force = serializers.BooleanField(default=False, required=False, write_only=True)
class Meta:
model = models.Fetch
@ -171,7 +190,63 @@ class FetchSerializer(serializers.ModelSerializer):
"detail",
"creation_date",
"fetch_date",
"object",
"force",
]
read_only_fields = [
"id",
"url",
"actor",
"status",
"detail",
"creation_date",
"fetch_date",
]
def validate_object(self, value):
# if value is a webginfer lookup, we craft a special url
if value.startswith("@"):
value = value.lstrip("@")
validator = validators.EmailValidator()
try:
validator(value)
except validators.ValidationError:
return value
return "webfinger://{}".format(value)
def create(self, validated_data):
check_duplicates = not validated_data.get("force", False)
if check_duplicates:
# first we check for duplicates
duplicate = (
validated_data["actor"]
.fetches.filter(
status="finished",
url=validated_data["object"],
creation_date__gte=timezone.now()
- datetime.timedelta(
seconds=settings.FEDERATION_DUPLICATE_FETCH_DELAY
),
)
.order_by("-creation_date")
.first()
)
if duplicate:
return duplicate
fetch = models.Fetch.objects.create(
actor=validated_data["actor"], url=validated_data["object"]
)
return fetch
def to_representation(self, obj):
repr = super().to_representation(obj)
object_data = None
if obj.object:
object_data = FETCH_OBJECT_FIELD.to_representation(obj.object)
repr["object"] = object_data
return repr
class FullActorSerializer(serializers.Serializer):

22
api/funkwhale_api/federation/api_views.py
View File

@ -1,5 +1,6 @@
import requests.exceptions
from django.conf import settings
from django.db import transaction
from django.db.models import Count
@ -10,6 +11,7 @@ from rest_framework import response
from rest_framework import viewsets
from funkwhale_api.common import preferences
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common.permissions import ConditionalAuthentication
from funkwhale_api.music import models as music_models
from funkwhale_api.music import views as music_views
@ -22,6 +24,7 @@ from . import filters
from . import models
from . import routes
from . import serializers
from . import tasks
from . import utils
@ -195,11 +198,28 @@ class InboxItemViewSet(
return response.Response(result, status=200)
class FetchViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
class FetchViewSet(
mixins.CreateModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
):
queryset = models.Fetch.objects.select_related("actor")
serializer_class = api_serializers.FetchSerializer
permission_classes = [permissions.IsAuthenticated]
throttling_scopes = {"create": {"authenticated": "fetch"}}
def get_queryset(self):
return super().get_queryset().filter(actor=self.request.user.actor)
def perform_create(self, serializer):
fetch = serializer.save(actor=self.request.user.actor)
if fetch.status == "finished":
# a duplicate was returned, no need to fetch again
return
if settings.FEDERATION_SYNCHRONOUS_FETCH:
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
else:
common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk)
class DomainViewSet(

4
api/funkwhale_api/federation/factories.py
View File

@ -21,7 +21,7 @@ class SignatureAuthFactory(factory.Factory):
key = factory.LazyFunction(lambda: keys.get_key_pair()[0])
key_id = factory.Faker("url")
use_auth_header = False
headers = ["(request-target)", "user-agent", "host", "date", "content-type"]
headers = ["(request-target)", "user-agent", "host", "date", "accept"]
class Meta:
model = requests_http_signature.HTTPSignatureAuth
@ -42,7 +42,7 @@ class SignedRequestFactory(factory.Factory):
"User-Agent": "Test",
"Host": "test.host",
"Date": http_date(timezone.now().timestamp()),
"Content-Type": "application/activity+json",
"Accept": "application/activity+json",
}
if extracted:
default_headers.update(extracted)

6
api/funkwhale_api/federation/library.py
View File

@ -9,9 +9,7 @@ def get_library_data(library_url, actor):
auth = signing.get_auth(actor.private_key, actor.private_key_id)
try:
response = session.get_session().get(
library_url,
auth=auth,
headers={"Content-Type": "application/activity+json"},
library_url, auth=auth, headers={"Accept": "application/activity+json"},
)
except requests.ConnectionError:
return {"errors": ["This library is not reachable"]}
@ -32,7 +30,7 @@ def get_library_data(library_url, actor):
def get_library_page(library, page_url, actor):
auth = signing.get_auth(actor.private_key, actor.private_key_id)
response = session.get_session().get(
page_url, auth=auth, headers={"Content-Type": "application/activity+json"},
page_url, auth=auth, headers={"Accept": "application/activity+json"},
)
serializer = serializers.CollectionPageSerializer(
data=response.json(),

9
api/funkwhale_api/federation/models.py
View File

@ -372,7 +372,7 @@ class Fetch(models.Model):
objects = FetchQuerySet.as_manager()
def save(self, **kwargs):
if not self.url and self.object:
if not self.url and self.object and hasattr(self.object, "fid"):
self.url = self.object.fid
super().save(**kwargs)
@ -388,6 +388,11 @@ class Fetch(models.Model):
contexts.FW.Track: serializers.TrackSerializer,
contexts.AS.Audio: serializers.UploadSerializer,
contexts.FW.Library: serializers.LibrarySerializer,
contexts.AS.Group: serializers.ActorSerializer,
contexts.AS.Person: serializers.ActorSerializer,
contexts.AS.Organization: serializers.ActorSerializer,
contexts.AS.Service: serializers.ActorSerializer,
contexts.AS.Application: serializers.ActorSerializer,
}
@ -568,7 +573,7 @@ class LibraryTrack(models.Model):
auth=auth,
stream=True,
timeout=20,
headers={"Content-Type": "application/activity+json"},
headers={"Accept": "application/activity+json"},
)
with remote_response as r:
remote_response.raise_for_status()

140
api/funkwhale_api/federation/serializers.py
View File

@ -151,6 +151,10 @@ class ActorSerializer(jsonld.JsonLdSerializer):
)
class Meta:
# not strictly necessary because it's not a model serializer
# but used by tasks.py/fetch
model = models.Actor
jsonld_mapping = {
"outbox": jsonld.first_id(contexts.AS.outbox),
"inbox": jsonld.first_id(contexts.LDP.inbox),
@ -765,6 +769,10 @@ class LibrarySerializer(PaginatedCollectionSerializer):
)
class Meta:
# not strictly necessary because it's not a model serializer
# but used by tasks.py/fetch
model = music_models.Library
jsonld_mapping = common_utils.concat_dicts(
PAGINATED_COLLECTION_JSONLD_MAPPING,
{
@ -795,12 +803,15 @@ class LibrarySerializer(PaginatedCollectionSerializer):
return r
def create(self, validated_data):
actor = utils.retrieve_ap_object(
validated_data["attributedTo"],
actor=self.context.get("fetch_actor"),
queryset=models.Actor,
serializer_class=ActorSerializer,
)
if self.instance:
actor = self.instance.actor
else:
actor = utils.retrieve_ap_object(
validated_data["attributedTo"],
actor=self.context.get("fetch_actor"),
queryset=models.Actor,
serializer_class=ActorSerializer,
)
privacy = {"": "me", "./": "me", None: "me", contexts.AS.Public: "everyone"}
library, created = music_models.Library.objects.update_or_create(
fid=validated_data["id"],
@ -815,6 +826,9 @@ class LibrarySerializer(PaginatedCollectionSerializer):
)
return library
def update(self, instance, validated_data):
return self.create(validated_data)
class CollectionPageSerializer(jsonld.JsonLdSerializer):
type = serializers.ChoiceField(choices=[contexts.AS.CollectionPage])
@ -968,8 +982,13 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
allow_null=True,
)
@transaction.atomic
def update(self, instance, validated_data):
return self.update_or_create(validated_data)
@transaction.atomic
def update_or_create(self, validated_data):
instance = self.instance or self.Meta.model(fid=validated_data["id"])
creating = instance.pk is None
attributed_to_fid = validated_data.get("attributedTo")
if attributed_to_fid:
validated_data["attributedTo"] = actors.get_actor(attributed_to_fid)
@ -977,8 +996,11 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
self.updateable_fields, validated_data, instance
)
updated_fields = self.validate_updated_data(instance, updated_fields)
if updated_fields:
if creating:
instance, created = self.Meta.model.objects.get_or_create(
fid=validated_data["id"], defaults=updated_fields
)
else:
music_tasks.update_library_entity(instance, updated_fields)
tags = [t["name"] for t in validated_data.get("tags", []) or []]
@ -1064,6 +1086,8 @@ class ArtistSerializer(MusicEntitySerializer):
d["@context"] = jsonld.get_default_context()
return d
create = MusicEntitySerializer.update_or_create
class AlbumSerializer(MusicEntitySerializer):
released = serializers.DateField(allow_null=True, required=False)
@ -1074,10 +1098,11 @@ class AlbumSerializer(MusicEntitySerializer):
)
updateable_fields = [
("name", "title"),
("cover", "attachment_cover"),
("musicbrainzId", "mbid"),
("attributedTo", "attributed_to"),
("released", "release_date"),
("cover", "attachment_cover"),
("_artist", "artist"),
]
class Meta:
@ -1124,6 +1149,20 @@ class AlbumSerializer(MusicEntitySerializer):
d["@context"] = jsonld.get_default_context()
return d
def validate(self, data):
validated_data = super().validate(data)
if not self.parent:
validated_data["_artist"] = utils.retrieve_ap_object(
validated_data["artists"][0]["id"],
actor=self.context.get("fetch_actor"),
queryset=music_models.Artist,
serializer_class=ArtistSerializer,
)
return validated_data
create = MusicEntitySerializer.update_or_create
class TrackSerializer(MusicEntitySerializer):
position = serializers.IntegerField(min_value=0, allow_null=True, required=False)
@ -1293,39 +1332,66 @@ class UploadSerializer(jsonld.JsonLdSerializer):
return lb
actor = self.context.get("actor")
kwargs = {}
if actor:
kwargs["actor"] = actor
try:
return music_models.Library.objects.get(fid=v, **kwargs)
except music_models.Library.DoesNotExist:
library = utils.retrieve_ap_object(
v,
actor=self.context.get("fetch_actor"),
queryset=music_models.Library,
serializer_class=LibrarySerializer,
)
except Exception:
raise serializers.ValidationError("Invalid library")
if actor and library.actor != actor:
raise serializers.ValidationError("Invalid library")
return library
def update(self, instance, validated_data):
return self.create(validated_data)
@transaction.atomic
def create(self, validated_data):
try:
return music_models.Upload.objects.get(fid=validated_data["id"])
except music_models.Upload.DoesNotExist:
pass
instance = self.instance or None
if not self.instance:
try:
instance = music_models.Upload.objects.get(fid=validated_data["id"])
except music_models.Upload.DoesNotExist:
pass
track = TrackSerializer(
context={"activity": self.context.get("activity")}
).create(validated_data["track"])
if instance:
data = {
"mimetype": validated_data["url"]["mediaType"],
"source": validated_data["url"]["href"],
"creation_date": validated_data["published"],
"modification_date": validated_data.get("updated"),
"duration": validated_data["duration"],
"size": validated_data["size"],
"bitrate": validated_data["bitrate"],
"import_status": "finished",
}
return music_models.Upload.objects.update_or_create(
fid=validated_data["id"], defaults=data
)[0]
else:
track = TrackSerializer(
context={"activity": self.context.get("activity")}
).create(validated_data["track"])
data = {
"fid": validated_data["id"],
"mimetype": validated_data["url"]["mediaType"],
"source": validated_data["url"]["href"],
"creation_date": validated_data["published"],
"modification_date": validated_data.get("updated"),
"track": track,
"duration": validated_data["duration"],
"size": validated_data["size"],
"bitrate": validated_data["bitrate"],
"library": validated_data["library"],
"from_activity": self.context.get("activity"),
"import_status": "finished",
}
return music_models.Upload.objects.create(**data)
data = {
"fid": validated_data["id"],
"mimetype": validated_data["url"]["mediaType"],
"source": validated_data["url"]["href"],
"creation_date": validated_data["published"],
"modification_date": validated_data.get("updated"),
"track": track,
"duration": validated_data["duration"],
"size": validated_data["size"],
"bitrate": validated_data["bitrate"],
"library": validated_data["library"],
"from_activity": self.context.get("activity"),
"import_status": "finished",
}
return music_models.Upload.objects.create(**data)
def to_representation(self, instance):
track = instance.track

81
api/funkwhale_api/federation/tasks.py
View File

@ -14,6 +14,7 @@ from requests.exceptions import RequestException
from funkwhale_api.common import preferences
from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils
from funkwhale_api.moderation import mrf
from funkwhale_api.music import models as music_models
from funkwhale_api.taskapp import celery
@ -24,6 +25,7 @@ from . import models, signing
from . import serializers
from . import routes
from . import utils
from . import webfinger
logger = logging.getLogger(__name__)
@ -285,24 +287,45 @@ def rotate_actor_key(actor):
@celery.app.task(name="federation.fetch")
@transaction.atomic
@celery.require_instance(
models.Fetch.objects.filter(status="pending").select_related("actor"), "fetch"
models.Fetch.objects.filter(status="pending").select_related("actor"),
"fetch_obj",
"fetch_id",
)
def fetch(fetch):
actor = fetch.actor
auth = signing.get_auth(actor.private_key, actor.private_key_id)
def fetch(fetch_obj):
def error(code, **kwargs):
fetch.status = "errored"
fetch.fetch_date = timezone.now()
fetch.detail = {"error_code": code}
fetch.detail.update(kwargs)
fetch.save(update_fields=["fetch_date", "status", "detail"])
fetch_obj.status = "errored"
fetch_obj.fetch_date = timezone.now()
fetch_obj.detail = {"error_code": code}
fetch_obj.detail.update(kwargs)
fetch_obj.save(update_fields=["fetch_date", "status", "detail"])
url = fetch_obj.url
mrf_check_url = url
if not mrf_check_url.startswith("webfinger://"):
payload, updated = mrf.inbox.apply({"id": mrf_check_url})
if not payload:
return error("blocked", message="Blocked by MRF")
actor = fetch_obj.actor
if settings.FEDERATION_AUTHENTIFY_FETCHES:
auth = signing.get_auth(actor.private_key, actor.private_key_id)
else:
auth = None
try:
if url.startswith("webfinger://"):
# we first grab the correpsonding webfinger representation
# to get the ActivityPub actor ID
webfinger_data = webfinger.get_resource(
"acct:" + url.replace("webfinger://", "")
)
url = webfinger.get_ap_url(webfinger_data["links"])
if not url:
return error("webfinger", message="Invalid or missing webfinger data")
payload, updated = mrf.inbox.apply({"id": url})
if not payload:
return error("blocked", message="Blocked by MRF")
response = session.get_session().get(
auth=auth,
url=fetch.url,
headers={"Content-Type": "application/activity+json"},
auth=auth, url=url, headers={"Accept": "application/activity+json"},
)
logger.debug("Remote answered with %s", response.status_code)
response.raise_for_status()
@ -320,8 +343,19 @@ def fetch(fetch):
try:
payload = response.json()
except json.decoder.JSONDecodeError:
# we attempt to extract a <link rel=alternate> that points
# to an activity pub resource, if possible, and retry with this URL
alternate_url = utils.find_alternate(response.text)
if alternate_url:
fetch_obj.url = alternate_url
fetch_obj.save(update_fields=["url"])
return fetch(fetch_id=fetch_obj.pk)
return error("invalid_json")
payload, updated = mrf.inbox.apply(payload)
if not payload:
return error("blocked", message="Blocked by MRF")
try:
doc = jsonld.expand(payload)
except ValueError:
@ -332,13 +366,13 @@ def fetch(fetch):
except IndexError:
return error("missing_jsonld_type")
try:
serializer_class = fetch.serializers[type]
serializer_class = fetch_obj.serializers[type]
model = serializer_class.Meta.model
except (KeyError, AttributeError):
fetch.status = "skipped"
fetch.fetch_date = timezone.now()
fetch.detail = {"reason": "unhandled_type", "type": type}
return fetch.save(update_fields=["fetch_date", "status", "detail"])
fetch_obj.status = "skipped"
fetch_obj.fetch_date = timezone.now()
fetch_obj.detail = {"reason": "unhandled_type", "type": type}
return fetch_obj.save(update_fields=["fetch_date", "status", "detail"])
try:
id = doc.get("@id")
except IndexError:
@ -350,11 +384,14 @@ def fetch(fetch):
if not serializer.is_valid():
return error("validation", validation_errors=serializer.errors)
try:
serializer.save()
obj = serializer.save()
except Exception as e:
error("save", message=str(e))
raise
fetch.status = "finished"
fetch.fetch_date = timezone.now()
return fetch.save(update_fields=["fetch_date", "status"])
fetch_obj.object = obj
fetch_obj.status = "finished"
fetch_obj.fetch_date = timezone.now()
return fetch_obj.save(
update_fields=["fetch_date", "status", "object_id", "object_content_type"]
)

37
api/funkwhale_api/federation/utils.py
View File

@ -1,3 +1,4 @@
import html.parser
import unicodedata
import re
from django.conf import settings
@ -164,3 +165,39 @@ def get_actor_from_username_data_query(field, data):
"domain__name__iexact": data["domain"],
}
)
class StopParsing(Exception):
pass
class AlternateLinkParser(html.parser.HTMLParser):
def __init__(self, *args, **kwargs):
self.result = None
super().__init__(*args, **kwargs)
def handle_starttag(self, tag, attrs):
if tag != "link":
return
attrs_dict = dict(attrs)
if attrs_dict.get("rel") == "alternate" and attrs_dict.get(
"type", "application/activity+json"
):
self.result = attrs_dict.get("href")
raise StopParsing()
def handle_endtag(self, tag):
if tag == "head":
raise StopParsing()
def find_alternate(response_text):
if not response_text:
return
parser = AlternateLinkParser()
try:
parser.feed(response_text)
except StopParsing:
return parser.result

9
api/funkwhale_api/federation/webfinger.py
View File

@ -46,3 +46,12 @@ def get_resource(resource_string):
serializer = serializers.ActorWebfingerSerializer(data=response.json())
serializer.is_valid(raise_exception=True)
return serializer.validated_data
def get_ap_url(links):
for link in links:
if (
link.get("rel") == "self"
and link.get("type") == "application/activity+json"
):
return link["href"]

2
api/funkwhale_api/moderation/management/commands/mrf_check.py
View File

@ -82,7 +82,7 @@ class Command(BaseCommand):
content = models.Activity.objects.get(uuid=input).payload
elif is_url(input):
response = session.get_session().get(
input, headers={"Content-Type": "application/activity+json"},
input, headers={"Accept": "application/activity+json"},
)
response.raise_for_status()
content = response.json()

8
api/funkwhale_api/music/serializers.py
View File

@ -324,6 +324,7 @@ class TrackSerializer(OptionalDescriptionMixin, serializers.Serializer):
class LibraryForOwnerSerializer(serializers.ModelSerializer):
uploads_count = serializers.SerializerMethodField()
size = serializers.SerializerMethodField()
actor = serializers.SerializerMethodField()
class Meta:
model = models.Library
@ -336,6 +337,7 @@ class LibraryForOwnerSerializer(serializers.ModelSerializer):
"uploads_count",
"size",
"creation_date",
"actor",
]
read_only_fields = ["fid", "uuid", "creation_date", "actor"]
@ -350,6 +352,12 @@ class LibraryForOwnerSerializer(serializers.ModelSerializer):
{"type": "Update", "object": {"type": "Library"}}, context={"library": obj}
)
def get_actor(self, o):
# Import at runtime to avoid a circular import issue
from funkwhale_api.federation import serializers as federation_serializers
return federation_serializers.APIActorSerializer(o.actor).data
class UploadSerializer(serializers.ModelSerializer):
track = TrackSerializer(required=False, allow_null=True)

20
api/funkwhale_api/music/views.py
View File

@ -249,6 +249,7 @@ class LibraryViewSet(
queryset = (
models.Library.objects.all()
.filter(channel=None)
.select_related("actor")
.order_by("-creation_date")
.annotate(_uploads_count=Count("uploads"))
.annotate(_size=Sum("uploads__size"))
@ -261,11 +262,15 @@ class LibraryViewSet(
required_scope = "libraries"
anonymous_policy = "setting"
owner_field = "actor.user"
owner_checks = ["read", "write"]
owner_checks = ["write"]
def get_queryset(self):
qs = super().get_queryset()
return qs.filter(actor=self.request.user.actor)
# allow retrieving a single library by uuid if request.user isn't
# the owner. Any other get should be from the owner only
if self.action != "retrieve":
qs = qs.filter(actor=self.request.user.actor)
return qs
def perform_create(self, serializer):
serializer.save(actor=self.request.user.actor)
@ -599,7 +604,7 @@ class UploadViewSet(
models.Upload.objects.all()
.order_by("-creation_date")
.prefetch_related(
"library",
"library__actor",
"track__artist",
"track__album__artist",
"track__attachment_cover",
@ -613,7 +618,7 @@ class UploadViewSet(
required_scope = "libraries"
anonymous_policy = "setting"
owner_field = "library.actor.user"
owner_checks = ["read", "write"]
owner_checks = ["write"]
filterset_class = filters.UploadFilter
ordering_fields = (
"creation_date",
@ -628,7 +633,12 @@ class UploadViewSet(
if self.action in ["update", "partial_update"]:
# prevent updating an upload that is already processed
qs = qs.filter(import_status="draft")
return qs.filter(library__actor=self.request.user.actor)
if self.action != "retrieve":
qs = qs.filter(library__actor=self.request.user.actor)
else:
actor = utils.get_actor_from_request(self.request)
qs = qs.playable_by(actor)
return qs
@action(methods=["get"], detail=True, url_path="audio-file-metadata")
def audio_file_metadata(self, request, *args, **kwargs):

2
api/requirements/local.txt
View File

@ -5,7 +5,7 @@ django_coverage_plugin>=1.6,<1.7
factory_boy>=2.11.1
# django-debug-toolbar that works with Django 1.5+
django-debug-toolbar>=1.11,<1.12
django-debug-toolbar>=2.2,<2.3
# improved REPL
ipdb==0.11

62
api/tests/federation/test_api_serializers.py
View File

@ -141,3 +141,65 @@ def test_api_full_actor_serializer(factories, to_api_date):
serializer = api_serializers.FullActorSerializer(actor)
assert serializer.data == expected
def test_fetch_serializer_no_obj(factories, to_api_date):
fetch = factories["federation.Fetch"]()
expected = {
"id": fetch.pk,
"url": fetch.url,
"creation_date": to_api_date(fetch.creation_date),
"fetch_date": None,
"status": fetch.status,
"detail": fetch.detail,
"object": None,
"actor": serializers.APIActorSerializer(fetch.actor).data,
}
assert api_serializers.FetchSerializer(fetch).data == expected
@pytest.mark.parametrize(
"object_factory, expected_type, expected_id",
[
("music.Album", "album", "id"),
("music.Artist", "artist", "id"),
("music.Track", "track", "id"),
("music.Library", "library", "uuid"),
("music.Upload", "upload", "uuid"),
("federation.Actor", "account", "full_username"),
],
)
def test_fetch_serializer_with_object(
object_factory, expected_type, expected_id, factories, to_api_date
):
obj = factories[object_factory]()
fetch = factories["federation.Fetch"](object=obj)
expected = {
"id": fetch.pk,
"url": fetch.url,
"creation_date": to_api_date(fetch.creation_date),
"fetch_date": None,
"status": fetch.status,
"detail": fetch.detail,
"object": {"type": expected_type, expected_id: getattr(obj, expected_id)},
"actor": serializers.APIActorSerializer(fetch.actor).data,
}
assert api_serializers.FetchSerializer(fetch).data == expected
def test_fetch_serializer_unhandled_obj(factories, to_api_date):
fetch = factories["federation.Fetch"](object=factories["users.User"]())
expected = {
"id": fetch.pk,
"url": fetch.url,
"creation_date": to_api_date(fetch.creation_date),
"fetch_date": None,
"status": fetch.status,
"detail": fetch.detail,
"object": None,
"actor": serializers.APIActorSerializer(fetch.actor).data,
}
assert api_serializers.FetchSerializer(fetch).data == expected

79
api/tests/federation/test_api_views.py
View File

@ -1,9 +1,12 @@
import datetime
import pytest
from django.urls import reverse
from funkwhale_api.federation import api_serializers
from funkwhale_api.federation import serializers
from funkwhale_api.federation import tasks
from funkwhale_api.federation import views
@ -170,7 +173,8 @@ def test_user_can_update_read_status_of_inbox_item(factories, logged_in_api_clie
def test_can_detail_fetch(logged_in_api_client, factories):
fetch = factories["federation.Fetch"](url="http://test.object")
actor = logged_in_api_client.user.create_actor()
fetch = factories["federation.Fetch"](url="http://test.object", actor=actor)
url = reverse("api:v1:federation:fetches-detail", kwargs={"pk": fetch.pk})
response = logged_in_api_client.get(url)
@ -209,3 +213,76 @@ def test_can_retrieve_actor(factories, api_client, preferences):
expected = api_serializers.FullActorSerializer(actor).data
assert response.data == expected
@pytest.mark.parametrize(
"object_id, expected_url",
[
("https://fetch.url", "https://fetch.url"),
("name@domain.tld", "webfinger://name@domain.tld"),
("@name@domain.tld", "webfinger://name@domain.tld"),
],
)
def test_can_fetch_using_url_synchronous(
object_id, expected_url, factories, logged_in_api_client, mocker, settings
):
settings.FEDERATION_SYNCHRONOUS_FETCH = True
actor = logged_in_api_client.user.create_actor()
def fake_task(fetch_id):
actor.fetches.filter(id=fetch_id).update(status="finished")
fetch_task = mocker.patch.object(tasks, "fetch", side_effect=fake_task)
url = reverse("api:v1:federation:fetches-list")
data = {"object": object_id}
response = logged_in_api_client.post(url, data)
assert response.status_code == 201
fetch = actor.fetches.latest("id")
assert fetch.status == "finished"
assert fetch.url == expected_url
assert response.data == api_serializers.FetchSerializer(fetch).data
fetch_task.assert_called_once_with(fetch_id=fetch.pk)
def test_fetch_duplicate(factories, logged_in_api_client, settings, now):
object_id = "http://example.test"
settings.FEDERATION_DUPLICATE_FETCH_DELAY = 60
actor = logged_in_api_client.user.create_actor()
duplicate = factories["federation.Fetch"](
actor=actor,
status="finished",
url=object_id,
creation_date=now - datetime.timedelta(seconds=59),
)
url = reverse("api:v1:federation:fetches-list")
data = {"object": object_id}
response = logged_in_api_client.post(url, data)
assert response.status_code == 201
assert response.data == api_serializers.FetchSerializer(duplicate).data
def test_fetch_duplicate_bypass_with_force(
factories, logged_in_api_client, mocker, settings, now
):
fetch_task = mocker.patch.object(tasks, "fetch")
object_id = "http://example.test"
settings.FEDERATION_DUPLICATE_FETCH_DELAY = 60
actor = logged_in_api_client.user.create_actor()
duplicate = factories["federation.Fetch"](
actor=actor,
status="finished",
url=object_id,
creation_date=now - datetime.timedelta(seconds=59),
)
url = reverse("api:v1:federation:fetches-list")
data = {"object": object_id, "force": True}
response = logged_in_api_client.post(url, data)
fetch = actor.fetches.latest("id")
assert fetch != duplicate
assert response.status_code == 201
assert response.data == api_serializers.FetchSerializer(fetch).data
fetch_task.assert_called_once_with(fetch_id=fetch.pk)

195
api/tests/federation/test_serializers.py
View File

@ -580,6 +580,37 @@ def test_music_library_serializer_from_private(factories, mocker):
)
def test_music_library_serializer_from_ap_update(factories, mocker):
actor = factories["federation.Actor"]()
library = factories["music.Library"]()
data = {
"@context": jsonld.get_default_context(),
"audience": "https://www.w3.org/ns/activitystreams#Public",
"name": "Hello",
"summary": "World",
"type": "Library",
"id": library.fid,
"followers": "https://library.id/followers",
"attributedTo": actor.fid,
"totalItems": 12,
"first": "https://library.id?page=1",
"last": "https://library.id?page=2",
}
serializer = serializers.LibrarySerializer(library, data=data)
assert serializer.is_valid(raise_exception=True)
serializer.save()
library.refresh_from_db()
assert library.uploads_count == data["totalItems"]
assert library.privacy_level == "everyone"
assert library.name == "Hello"
assert library.description == "World"
assert library.followers_url == data["followers"]
def test_activity_pub_artist_serializer_to_ap(factories):
content = factories["common.Content"]()
artist = factories["music.Artist"](
@ -610,6 +641,86 @@ def test_activity_pub_artist_serializer_to_ap(factories):
assert serializer.data == expected
def test_activity_pub_artist_serializer_from_ap_create(factories, faker, now, mocker):
actor = factories["federation.Actor"]()
mocker.patch(
"funkwhale_api.federation.utils.retrieve_ap_object", return_value=actor
)
payload = {
"@context": jsonld.get_default_context(),
"type": "Artist",
"id": "https://test.artist",
"name": "Art",
"musicbrainzId": faker.uuid4(),
"published": now.isoformat(),
"attributedTo": actor.fid,
"content": "Summary",
"image": {
"type": "Image",
"mediaType": "image/jpeg",
"url": "https://attachment.file",
},
"tag": [
{"type": "Hashtag", "name": "#Punk"},
{"type": "Hashtag", "name": "#Rock"},
],
}
serializer = serializers.ArtistSerializer(data=payload)
assert serializer.is_valid(raise_exception=True) is True
artist = serializer.save()
assert artist.fid == payload["id"]
assert artist.attributed_to == actor
assert artist.name == payload["name"]
assert str(artist.mbid) == payload["musicbrainzId"]
assert artist.description.text == payload["content"]
assert artist.description.content_type == "text/html"
assert artist.attachment_cover.url == payload["image"]["url"]
assert artist.attachment_cover.mimetype == payload["image"]["mediaType"]
assert artist.get_tags() == ["Punk", "Rock"]
def test_activity_pub_artist_serializer_from_ap_update(factories, faker, now, mocker):
artist = factories["music.Artist"]()
actor = factories["federation.Actor"]()
mocker.patch(
"funkwhale_api.federation.utils.retrieve_ap_object", return_value=actor
)
payload = {
"@context": jsonld.get_default_context(),
"type": "Artist",
"id": artist.fid,
"name": "Art",
"musicbrainzId": faker.uuid4(),
"published": now.isoformat(),
"attributedTo": actor.fid,
"content": "Summary",
"image": {
"type": "Image",
"mediaType": "image/jpeg",
"url": "https://attachment.file",
},
"tag": [
{"type": "Hashtag", "name": "#Punk"},
{"type": "Hashtag", "name": "#Rock"},
],
}
serializer = serializers.ArtistSerializer(artist, data=payload)
assert serializer.is_valid(raise_exception=True) is True
serializer.save()
artist.refresh_from_db()
assert artist.attributed_to == actor
assert artist.name == payload["name"]
assert str(artist.mbid) == payload["musicbrainzId"]
assert artist.description.text == payload["content"]
assert artist.description.content_type == "text/html"
assert artist.attachment_cover.url == payload["image"]["url"]
assert artist.attachment_cover.mimetype == payload["image"]["mediaType"]
assert artist.get_tags() == ["Punk", "Rock"]
def test_activity_pub_album_serializer_to_ap(factories):
content = factories["common.Content"]()
album = factories["music.Album"](
@ -652,39 +763,42 @@ def test_activity_pub_album_serializer_to_ap(factories):
assert serializer.data == expected
def test_activity_pub_artist_serializer_from_ap_update(factories, faker):
artist = factories["music.Artist"](attributed=True)
def test_activity_pub_album_serializer_from_ap_create(factories, faker, now):
actor = factories["federation.Actor"]()
artist = factories["music.Artist"]()
released = faker.date_object()
payload = {
"@context": jsonld.get_default_context(),
"type": "Artist",
"id": artist.fid,
"type": "Album",
"id": "https://album.example",
"name": faker.sentence(),
"cover": {"type": "Link", "mediaType": "image/jpeg", "href": faker.url()},
"musicbrainzId": faker.uuid4(),
"published": artist.creation_date.isoformat(),
"attributedTo": artist.attributed_to.fid,
"mediaType": "text/html",
"content": common_utils.render_html(faker.sentence(), "text/html"),
"image": {"type": "Image", "mediaType": "image/jpeg", "url": faker.url()},
"published": now.isoformat(),
"released": released.isoformat(),
"artists": [
serializers.ArtistSerializer(
artist, context={"include_ap_context": False}
).data
],
"attributedTo": actor.fid,
"tag": [
{"type": "Hashtag", "name": "#Punk"},
{"type": "Hashtag", "name": "#Rock"},
],
}
serializer = serializers.ArtistSerializer(artist, data=payload)
serializer = serializers.AlbumSerializer(data=payload)
assert serializer.is_valid(raise_exception=True) is True
serializer.save()
album = serializer.save()
artist.refresh_from_db()
assert artist.name == payload["name"]
assert str(artist.mbid) == payload["musicbrainzId"]
assert artist.attachment_cover.url == payload["image"]["url"]
assert artist.attachment_cover.mimetype == payload["image"]["mediaType"]
assert artist.description.text == payload["content"]
assert artist.description.content_type == "text/html"
assert sorted(artist.tagged_items.values_list("tag__name", flat=True)) == [
assert album.title == payload["name"]
assert str(album.mbid) == payload["musicbrainzId"]
assert album.release_date == released
assert album.artist == artist
assert album.attachment_cover.url == payload["cover"]["href"]
assert album.attachment_cover.mimetype == payload["cover"]["mediaType"]
assert sorted(album.tagged_items.values_list("tag__name", flat=True)) == [
"Punk",
"Rock",
]
@ -1062,6 +1176,43 @@ def test_activity_pub_upload_serializer_from_ap(factories, mocker, r_mock):
assert upload.modification_date == updated
def test_activity_pub_upload_serializer_from_ap_update(factories, mocker, now, r_mock):
library = factories["music.Library"]()
upload = factories["music.Upload"](library=library)
data = {
"@context": jsonld.get_default_context(),
"type": "Audio",
"id": upload.fid,
"name": "Ignored",
"published": now.isoformat(),
"updated": now.isoformat(),
"duration": 42,
"bitrate": 42,
"size": 66,
"url": {
"href": "https://audio.file/url",
"type": "Link",
"mediaType": "audio/mp3",
},
"library": library.fid,
"track": serializers.TrackSerializer(upload.track).data,
}
r_mock.get(data["track"]["album"]["cover"]["href"], body=io.BytesIO(b"coucou"))
serializer = serializers.UploadSerializer(upload, data=data)
assert serializer.is_valid(raise_exception=True)
serializer.save()
upload.refresh_from_db()
assert upload.fid == data["id"]
assert upload.duration == data["duration"]
assert upload.size == data["size"]
assert upload.bitrate == data["bitrate"]
assert upload.source == data["url"]["href"]
assert upload.mimetype == data["url"]["mediaType"]
def test_activity_pub_upload_serializer_validtes_library_actor(factories, mocker):
library = factories["music.Library"]()
usurpator = factories["federation.Actor"]()
@ -1201,7 +1352,7 @@ def test_track_serializer_update_license(factories):
obj = factories["music.Track"](license=None)
serializer = serializers.TrackSerializer()
serializer = serializers.TrackSerializer(obj)
serializer.update(obj, {"license": "http://creativecommons.org/licenses/by/2.0/"})
obj.refresh_from_db()

153
api/tests/federation/test_tasks.py
View File

@ -395,3 +395,156 @@ def test_fetch_success(factories, r_mock, mocker):
assert init.call_args[0][1] == artist
assert init.call_args[1]["data"] == payload
assert save.call_count == 1
def test_fetch_webfinger(factories, r_mock, mocker):
actor = factories["federation.Actor"]()
fetch = factories["federation.Fetch"](
url="webfinger://{}".format(actor.full_username)
)
payload = serializers.ActorSerializer(actor).data
init = mocker.spy(serializers.ActorSerializer, "__init__")
save = mocker.spy(serializers.ActorSerializer, "save")
webfinger_payload = {
"subject": "acct:{}".format(actor.full_username),
"aliases": ["https://test.webfinger"],
"links": [
{"rel": "self", "type": "application/activity+json", "href": actor.fid}
],
}
webfinger_url = "https://{}/.well-known/webfinger?resource={}".format(
actor.domain_id, webfinger_payload["subject"]
)
r_mock.get(actor.fid, json=payload)
r_mock.get(webfinger_url, json=webfinger_payload)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
payload["@context"].append("https://funkwhale.audio/ns")
assert fetch.status == "finished"
assert fetch.object == actor
assert init.call_count == 1
assert init.call_args[0][1] == actor
assert init.call_args[1]["data"] == payload
assert save.call_count == 1
def test_fetch_rel_alternate(factories, r_mock, mocker):
actor = factories["federation.Actor"]()
fetch = factories["federation.Fetch"](url="http://example.page")
html_text = """
<html>
<head>
<link rel="alternate" type="application/activity+json" href="{}" />
</head>
</html>
""".format(
actor.fid
)
ap_payload = serializers.ActorSerializer(actor).data
init = mocker.spy(serializers.ActorSerializer, "__init__")
save = mocker.spy(serializers.ActorSerializer, "save")
r_mock.get(fetch.url, text=html_text)
r_mock.get(actor.fid, json=ap_payload)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
ap_payload["@context"].append("https://funkwhale.audio/ns")
assert fetch.status == "finished"
assert fetch.object == actor
assert init.call_count == 1
assert init.call_args[0][1] == actor
assert init.call_args[1]["data"] == ap_payload
assert save.call_count == 1
@pytest.mark.parametrize(
"factory_name, serializer_class",
[
("federation.Actor", serializers.ActorSerializer),
("music.Library", serializers.LibrarySerializer),
("music.Artist", serializers.ArtistSerializer),
("music.Album", serializers.AlbumSerializer),
("music.Track", serializers.TrackSerializer),
],
)
def test_fetch_url(factory_name, serializer_class, factories, r_mock, mocker):
obj = factories[factory_name]()
fetch = factories["federation.Fetch"](url=obj.fid)
payload = serializer_class(obj).data
init = mocker.spy(serializer_class, "__init__")
save = mocker.spy(serializer_class, "save")
r_mock.get(obj.fid, json=payload)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
payload["@context"].append("https://funkwhale.audio/ns")
assert fetch.status == "finished"
assert fetch.object == obj
assert init.call_count == 1
assert init.call_args[0][1] == obj
assert init.call_args[1]["data"] == payload
assert save.call_count == 1
def test_fetch_honor_instance_policy_domain(factories):
domain = factories["moderation.InstancePolicy"](
block_all=True, for_domain=True
).target_domain
fid = "https://{}/test".format(domain.name)
fetch = factories["federation.Fetch"](url=fid)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
assert fetch.status == "errored"
assert fetch.detail["error_code"] == "blocked"
def test_fetch_honor_mrf_inbox_before_http(mrf_inbox_registry, factories, mocker):
apply = mocker.patch.object(mrf_inbox_registry, "apply", return_value=(None, False))
fid = "http://domain/test"
fetch = factories["federation.Fetch"](url=fid)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
assert fetch.status == "errored"
assert fetch.detail["error_code"] == "blocked"
apply.assert_called_once_with({"id": fid})
def test_fetch_honor_mrf_inbox_after_http(
r_mock, mrf_inbox_registry, factories, mocker
):
apply = mocker.patch.object(
mrf_inbox_registry, "apply", side_effect=[(True, False), (None, False)]
)
payload = {"id": "http://domain/test", "actor": "hello"}
r_mock.get(payload["id"], json=payload)
fetch = factories["federation.Fetch"](url=payload["id"])
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
assert fetch.status == "errored"
assert fetch.detail["error_code"] == "blocked"
apply.assert_any_call({"id": payload["id"]})
apply.assert_any_call(payload)
def test_fetch_honor_instance_policy_different_url_and_id(r_mock, factories):
domain = factories["moderation.InstancePolicy"](
block_all=True, for_domain=True
).target_domain
fid = "https://ok/test"
r_mock.get(fid, json={"id": "http://{}/test".format(domain.name)})
fetch = factories["federation.Fetch"](url=fid)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
assert fetch.status == "errored"
assert fetch.detail["error_code"] == "blocked"

29
api/tests/music/test_views.py
View File

@ -640,6 +640,16 @@ def test_user_can_list_their_library(factories, logged_in_api_client):
assert response.data["results"][0]["uuid"] == str(library.uuid)
def test_user_can_retrieve_another_user_library(factories, logged_in_api_client):
library = factories["music.Library"]()
url = reverse("api:v1:libraries-detail", kwargs={"uuid": library.uuid})
response = logged_in_api_client.get(url)
assert response.status_code == 200
assert response.data["uuid"] == str(library.uuid)
def test_library_list_excludes_channel_library(factories, logged_in_api_client):
actor = logged_in_api_client.user.create_actor()
factories["audio.Channel"](attributed_to=actor)
@ -670,9 +680,11 @@ def test_library_delete_via_api_triggers_outbox(factories, mocker):
)
def test_user_cannot_get_other_actors_uploads(factories, logged_in_api_client):
def test_user_cannot_get_other_not_playable_uploads(factories, logged_in_api_client):
logged_in_api_client.user.create_actor()
upload = factories["music.Upload"]()
upload = factories["music.Upload"](
import_status="finished", library__privacy_level="private"
)
url = reverse("api:v1:uploads-detail", kwargs={"uuid": upload.uuid})
response = logged_in_api_client.get(url)
@ -680,6 +692,19 @@ def test_user_cannot_get_other_actors_uploads(factories, logged_in_api_client):
assert response.status_code == 404
def test_user_can_get_retrieve_playable_uploads(factories, logged_in_api_client):
logged_in_api_client.user.create_actor()
upload = factories["music.Upload"](
import_status="finished", library__privacy_level="everyone"
)
url = reverse("api:v1:uploads-detail", kwargs={"uuid": upload.uuid})
response = logged_in_api_client.get(url)
assert response.status_code == 200
assert response.data["uuid"] == str(upload.uuid)
def test_user_cannot_delete_other_actors_uploads(factories, logged_in_api_client):
logged_in_api_client.user.create_actor()
upload = factories["music.Upload"]()

47
docs/conf.py
View File

@ -95,18 +95,16 @@ html_theme = "sphinx_rtd_theme"
# further. For a list of options available for each theme, see the
# documentation.
#
html_theme_options = {
'gitlab_url': 'https://dev.funkwhale.audio/funkwhale/funkwhale'
}
html_theme_options = {"gitlab_url": "https://dev.funkwhale.audio/funkwhale/funkwhale"}
html_context = {
'display_gitlab': True,
'gitlab_host': 'dev.funkwhale.audio',
'gitlab_repo': 'funkwhale',
'gitlab_user': 'funkwhale',
'gitlab_version': 'master',
'conf_py_path': '/docs/',
"display_gitlab": True,
"gitlab_host": "dev.funkwhale.audio",
"gitlab_repo": "funkwhale",
"gitlab_user": "funkwhale",
"gitlab_version": "master",
"conf_py_path": "/docs/",
}
html_logo = 'logo.svg'
html_logo = "logo.svg"
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
@ -173,15 +171,13 @@ texinfo_documents = [
# Define list of redirect files to be build in the Sphinx build process
redirect_files = [
('importing-music.html', 'admin/importing-music.html'),
('architecture.html', 'developers/architecture.html'),
('troubleshooting.html', 'admin/troubleshooting.html'),
('configuration.html', 'admin/configuration.html'),
('upgrading/index.html', '../admin/upgrading.html'),
('upgrading/0.17.html', '../admin/0.17.html'),
('users/django.html', '../admin/django.html'),
("importing-music.html", "admin/importing-music.html"),
("architecture.html", "developers/architecture.html"),
("troubleshooting.html", "admin/troubleshooting.html"),
("configuration.html", "admin/configuration.html"),
("upgrading/index.html", "../admin/upgrading.html"),
("upgrading/0.17.html", "../admin/0.17.html"),
("users/django.html", "../admin/django.html"),
]
# Generate redirect template
@ -199,16 +195,17 @@ redirect_template = """\
# Tell Sphinx to copy the files
def copy_legacy_redirects(app, docname):
if app.builder.name == 'html':
if app.builder.name == "html":
for html_src_path, new in redirect_files:
page = redirect_template.format(new=new)
target_path = app.outdir + '/' + html_src_path
target_path = app.outdir + "/" + html_src_path
if not os.path.exists(os.path.dirname(target_path)):
os.makedirs(os.path.dirname(target_path))
with open(target_path, 'w') as f:
f.write(page)
os.makedirs(os.path.dirname(target_path))
with open(target_path, "w") as f:
f.write(page)
def setup(app):
app.connect('build-finished', copy_legacy_redirects)
app.connect("build-finished", copy_legacy_redirects)

9
docs/serve.py
View File

@ -1,13 +1,10 @@
#!/usr/bin/env python
from subprocess import call
# initial make
call(["python", "-m", "sphinx", ".", "/tmp/_build"])
from livereload import Server, shell
server = Server()
server.watch('.', shell('python -m sphinx . /tmp/_build'))
server.serve(
root='/tmp/_build/',
liveport=35730,
port=8001,
host='0.0.0.0')
server.watch(".", shell("python -m sphinx . /tmp/_build"))
server.serve(root="/tmp/_build/", liveport=35730, port=8001, host="0.0.0.0")

16
front/scripts/print-duplicates-source.py
View File

@ -9,21 +9,17 @@ def print_duplicates(path):
contexts_by_id = collections.defaultdict(list)
for e in pofile:
contexts_by_id[e.msgid].append(e.msgctxt)
count = collections.Counter(
[e.msgid for e in pofile]
)
duplicates = [
(k, v) for k, v in count.items()
if v > 1
]
count = collections.Counter([e.msgid for e in pofile])
duplicates = [(k, v) for k, v in count.items() if v > 1]
for k, v in sorted(duplicates, key=lambda r: r[1], reverse=True):
print('{} entries - {}:'.format(v, k))
print("{} entries - {}:".format(v, k))
for ctx in contexts_by_id[k]:
print(' - {}'.format(ctx))
print(" - {}".format(ctx))
print()
total_duplicates = sum([v - 1 for _, v in duplicates])
print('{} total duplicates'.format(total_duplicates))
print("{} total duplicates".format(total_duplicates))
if __name__ == "__main__":
parser = argparse.ArgumentParser()

72
front/src/components/audio/SearchBar.vue
View File

@ -15,6 +15,7 @@
<script>
import jQuery from 'jquery'
import router from '@/router'
import lodash from '@/lodash'
import GlobalEvents from "@/components/utils/global-events"
export default {
@ -55,7 +56,11 @@ export default {
noResults: this.$pgettext('Sidebar/Search/Error.Label', 'Sorry, there are no results for this search')
},
onSelect (result, response) {
jQuery(self.$el).search("set value", searchQuery)
console.log('SELECTEING', result)
router.push(result.routerUrl)
jQuery(self.$el).search("hide results")
return false
},
onSearchQuery (query) {
self.$emit('search')
@ -70,9 +75,14 @@ export default {
return xhrObject
},
onResponse: function (initialResponse) {
let objId = self.extractObjId(searchQuery)
let results = {}
let isEmptyResults = true
let categories = [
{
code: 'federation',
name: self.$pgettext('*/*/*', 'Federation'),
},
{
code: 'artists',
route: 'library.artists.detail',
@ -139,21 +149,42 @@ export default {
name: category.name,
results: []
}
initialResponse[category.code].forEach(result => {
isEmptyResults = false
let id = category.getId(result)
results[category.code].results.push({
title: category.getTitle(result),
id,
routerUrl: {
name: category.route,
params: {
id
}
},
description: category.getDescription(result)
if (category.code === 'federation') {
if (objId) {
isEmptyResults = false
let searchMessage = self.$pgettext('Search/*/*', 'Search on the fediverse')
results['federation'] = {
name: self.$pgettext('*/*/*', 'Federation'),
results: [{
title: searchMessage,
routerUrl: {
name: 'search',
query: {
id: objId,
}
}
}]
}
}
}
else {
initialResponse[category.code].forEach(result => {
isEmptyResults = false
let id = category.getId(result)
results[category.code].results.push({
title: category.getTitle(result),
id,
routerUrl: {
name: category.route,
params: {
id
}
},
description: category.getDescription(result)
})
})
})
}
})
return {
results: isEmptyResults ? {} : results
@ -167,6 +198,19 @@ export default {
focusSearch () {
this.$refs.search.focus()
},
extractObjId (query) {
query = lodash.trim(query)
query = lodash.trim(query, '@')
if (query.indexOf(' ') > -1) {
return
}
if (query.startsWith('http://') || query.startsWith('https://')) {
return query
}
if (query.split('@').length > 1) {
return query
}
}
}
}
</script>

5
front/src/components/common/ActorAvatar.vue
View File

@ -1,5 +1,6 @@
<template>
<span :style="defaultAvatarStyle" class="ui avatar circular label">{{ actor.preferred_username[0]}}</span>
<img v-if="actor.icon && actor.icon.original" :src="actor.icon.small_square_crop" class="ui avatar circular image" />
<span v-else :style="defaultAvatarStyle" class="ui avatar circular label">{{ actor.preferred_username[0]}}</span>
</template>
<script>
@ -20,7 +21,7 @@ export default {
}
</script>
<style lang="scss">
.ui.circular.avatar.label {
.ui.circular.avatar {
width: 28px;
height: 28px;
font-size: 1em !important;

2
front/src/components/common/ActorLink.vue
View File

@ -19,7 +19,7 @@ export default {
if (this.actor.is_local) {
return {name: 'profile.overview', params: {username: this.actor.preferred_username}}
} else {
return {name: 'profile.overview', params: {username: this.actor.full_username}}
return {name: 'profile.full.overview', params: {username: this.actor.preferred_username, domain: this.actor.domain}}
}
},
repr () {

1
front/src/lodash.js
View File

@ -16,4 +16,5 @@ export default {
isEqual: require('lodash/isEqual'),
sum: require('lodash/sum'),
startCase: require('lodash/startCase'),
trim: require('lodash/trim'),
}

19
front/src/router/index.js
View File

@ -74,6 +74,15 @@ export default new Router({
defaultKey: route.query.key
})
},
{
path: "/search",
name: "search",
component: () =>
import(/* webpackChunkName: "core" */ "@/views/Search"),
props: route => ({
initialId: route.query.id
})
},
{
path: "/auth/password/reset/confirm",
name: "auth.password-reset-confirm",
@ -143,17 +152,17 @@ export default new Router({
),
props: true
},
...['/@:username', '/@:username@:domain'].map((path) => {
...[{suffix: '.full', path: '/@:username@:domain'}, {suffix: '', path: '/@:username'}].map((route) => {
return {
path: path,
name: "profile",
path: route.path,
name: `profile${route.suffix}`,
component: () =>
import(/* webpackChunkName: "core" */ "@/views/auth/ProfileBase"),
props: true,
children: [
{
path: "",
name: "profile.overview",
name: `profile${route.suffix}.overview`,
component: () =>
import(
/* webpackChunkName: "core" */ "@/views/auth/ProfileOverview"
@ -161,7 +170,7 @@ export default new Router({
},
{
path: "activity",
name: "profile.activity",
name: `profile${route.suffix}.activity`,
component: () =>
import(
/* webpackChunkName: "core" */ "@/views/auth/ProfileActivity"

207
front/src/views/Search.vue Normal file
View File

@ -0,0 +1,207 @@
<template>
<main class="main pusher" v-title="labels.title">
<section class="ui vertical stripe segment">
<div class="ui small text container">
<form :class="['ui', {loading: isLoading}, 'form']" @submit.stop.prevent="createFetch">
<h2><translate translate-context="Content/Fetch/Title">Retrieve a remote object</translate></h2>
<p>
<translate translate-context="Content/Fetch/Paragraph">Use this form to retrieve an object hosted somewhere else in the fediverse.</translate>
</p>
<div v-if="errors.length > 0" class="ui negative message">
<div class="header"><translate translate-context="Content/*/Error message.Title">Error while fetching object</translate></div>
<ul class="list">
<li v-for="error in errors">{{ error }}</li>
</ul>
</div>
<div class="ui required field">
<label for="object-id">
{{ labels.fieldLabel }}
</label>
<input type="text" name="object-id" id="object-id" v-model="id" required>
</div>
<button type="submit" :class="['ui', 'primary', {loading: isLoading}, 'button']" :disabled="isLoading || !id || id.length === 0">
<translate translate-context="Content/Search/Input.Label/Noun">Search</translate>
</button>
</form>
<div v-if="!isLoading && fetch && fetch.status === 'finished'">
<div class="ui hidden divider"></div>
<h2><translate translate-context="Content/Fetch/Title/Noun">Result</translate></h2>
<div class="ui hidden divider"></div>
<div v-if="objComponent" class="ui app-cards cards">
<component v-bind="objComponent.props" :is="objComponent.type"></component>
</div>
<div v-else class="ui warning message">
<p><translate translate-context="Content/*/Error message.Title">This kind of object isn't supported yet</translate></p>
</div>
</div>
</div>
</section>
</main>
</template>
<script>
import axios from 'axios'
import AlbumCard from '@/components/audio/album/Card'
import ArtistCard from '@/components/audio/artist/Card'
import LibraryCard from '@/views/content/remote/Card'
import ChannelEntryCard from '@/components/audio/ChannelEntryCard'
export default {
props: {
initialId: { type: String, required: false}
},
components: {
ActorLink: () => import(/* webpackChunkName: "common" */ "@/components/common/ActorLink"),
ArtistCard,
AlbumCard,
LibraryCard,
ChannelEntryCard,
},
data () {
return {
id: this.initialId,
fetch: null,
obj: null,
isLoading: false,
errors: [],
}
},
created () {
if (this.id) {
this.createFetch()
}
},
computed: {
labels() {
return {
title: this.$pgettext('Head/Fetch/Title', "Search a remote object"),
fieldLabel: this.$pgettext('Head/Fetch/Field.Placeholder', "URL or @username"),
}
},
objInfo () {
if (this.fetch && this.fetch.status === 'finished') {
return this.fetch.object
}
},
objComponent () {
if (!this.obj) {
return
}
switch (this.objInfo.type) {
case "account":
return {
type: "actor-link",
props: {actor: this.obj}
}
case "library":
return {
type: "library-card",
props: {library: this.obj}
}
case "album":
return {
type: "album-card",
props: {album: this.obj}
}
case "artist":
return {
type: "artist-card",
props: {artist: this.obj}
}
case "upload":
return {
type: "channel-entry-card",
props: {entry: this.obj.track}
}
default:
return
}
}
},
methods: {
createFetch () {
if (!this.id) {
return
}
this.$router.replace({name: "search", query: {id: this.id}})
this.fetch = null
let self = this
self.errors = []
self.isLoading = true
let payload = {
object: this.id
}
axios.post('federation/fetches/', payload).then((response) => {
self.isLoading = false
self.fetch = response.data
if (self.fetch.status === 'errored' || self.fetch.status === 'skipped') {
self.errors.push(
self.$pgettext("Content/*/Error message.Title", "This object cannot be retrieved")
)
}
}, error => {
self.isLoading = false
self.errors = error.backendErrors
})
},
getObj (objInfo) {
if (!this.id) {
return
}
let self = this
self.isLoading = true
let url = null
switch (objInfo.type) {
case 'account':
url = `federation/actors/${objInfo.full_username}/`
break;
case 'library':
url = `libraries/${objInfo.uuid}/`
break;
case 'artist':
url = `artists/${objInfo.id}/`
break;
case 'album':
url = `albums/${objInfo.id}/`
break;
case 'upload':
url = `uploads/${objInfo.uuid}/`
break;
default:
break;
}
if (!url) {
this.errors.push(
self.$pgettext("Content/*/Error message.Title", "This kind of object isn't supported yet")
)
this.isLoading = false
return
}
axios.get(url).then((response) => {
self.obj = response.data
self.isLoading = false
}, error => {
self.isLoading = false
self.errors = error.backendErrors
})
}
},
watch: {
initialId (v) {
this.id = v
this.createFetch()
},
objInfo (v) {
this.obj = null
if (v) {
this.getObj(v)
}
}
}
}
</script>

9
front/src/views/auth/ProfileBase.vue
View File

@ -105,6 +105,7 @@ export default {
methods: {
fetch () {
let self = this
self.object = null
self.isLoading = true
axios.get(`federation/actors/${this.fullUsername}/`).then((response) => {
self.object = response.data
@ -139,6 +140,14 @@ export default {
displayName () {
return this.object.name || this.object.preferred_username
}
},
watch: {
domain () {
this.fetch()
},
username () {
this.fetch()
}
}
}
</script>