From 07c8a0c24c3707cd7bc5399074b0436fe61e50bd Mon Sep 17 00:00:00 2001 From: Petitminion Date: Sun, 9 Feb 2025 19:15:31 +0100 Subject: [PATCH] Add far_right attribute and filtering logic on the far_righ attribute --- api/funkwhale_api/music/models.py | 1 + api/funkwhale_api/music/tasks.py | 143 ++++++++++++++++----------- api/tests/music/test_tasks.py | 68 +++++++++++++ docs/specs/far-right-filter/index.md | 63 ++++++++++++ 4 files changed, 219 insertions(+), 56 deletions(-) create mode 100644 docs/specs/far-right-filter/index.md diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index fe1e2b0c9..a4c3b9561 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -247,6 +247,7 @@ class Artist(APIModelMixin): ) modification_date = models.DateTimeField(default=timezone.now, db_index=True) api = musicbrainz.api.artists + far_right = models.CharField(max_length=100, null=True, blank=True) objects = ArtistQuerySet.as_manager() def __str__(self): diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py index cc0fcaaf2..5d815fad1 100644 --- a/api/funkwhale_api/music/tasks.py +++ b/api/funkwhale_api/music/tasks.py @@ -20,6 +20,7 @@ from funkwhale_api.federation import library as lb from funkwhale_api.federation import routes from funkwhale_api.federation import utils as federation_utils from funkwhale_api.music.management.commands import import_files +from funkwhale_api.music.models import Artist from funkwhale_api.tags import models as tags_models from funkwhale_api.tags import tasks as tags_tasks from funkwhale_api.taskapp import celery @@ -164,6 +165,13 @@ class UploadImportError(ValueError): super().__init__(code) +class FarRightError(ValueError): + def __init__(self, code, detail): + self.code = code + self.detail = detail + super().__init__(code) + + def fail_import(upload, error_code, detail=None, **fields): old_status = upload.import_status upload.import_status = "errored" @@ -271,6 +279,8 @@ def process_upload(upload, update_denormalization=True): ) except UploadImportError as e: return fail_import(upload, e.code) + except FarRightError as e: + return fail_import(upload, e.code, detail=e.detail) except Exception as e: fail_import(upload, "unknown_error", e) raise @@ -475,7 +485,15 @@ def get_best_candidate_or_create(model, query, defaults, sort_fields): """ candidates = model.objects.filter(query) if candidates: - return sort_candidates(candidates, sort_fields)[0], False + sorted_candidates = sort_candidates(candidates, sort_fields) + if model == Artist and sorted_candidates[0].far_right: + raise FarRightError( + code="Far right artist detected", + detail=f"The artist name has been matched with this wikidata entity \ + {sorted_candidates[0].far_right}. This artist will not be saved. No pasaran. \ + You can checkout our coc at https://www.funkwhale.audio/code-of-conduct/", + ) + return sorted_candidates[0], False return model.objects.create(**defaults), True @@ -530,6 +548,61 @@ def truncate(v, length): return v[:length] +def get_or_create_album_artist_credit( + data, attributed_to, from_activity_id, query_mb, track_artists_credits, album_mbid +): + mbid = query_mb and (data.get("musicbrainz_albumid", None) or album_mbid) + try: + return get_or_create_artists_credits_from_musicbrainz( + "release", + mbid, + attributed_to=attributed_to, + from_activity_id=from_activity_id, + ) + except (NoMbid, ResponseError, NetworkError): + if album_artists := getter(data, "album", "artist_credit", default=None): + return get_or_create_artists_credits_from_artist_credit_metadata( + album_artists, + attributed_to=attributed_to, + from_activity_id=from_activity_id, + ) + else: + return track_artists_credits + + +def get_or_create_track_artist_credit( + data, forced_values, attributed_to, from_activity_id, query_mb +): + artist_credit_data = getter(data, "artist_credit", default=[]) + if "artist" in forced_values: + artist = forced_values["artist"] + query = Q(artist=artist) + defaults = { + "artist": artist, + "joinphrase": "", + "credit": artist.name, + } + track_artist_credit, created = get_best_candidate_or_create( + models.ArtistCredit, query, defaults=defaults, sort_fields=["mbid", "fid"] + ) + return [track_artist_credit] + else: + mbid = query_mb and (data.get("musicbrainz_id", None) or data.get("mbid", None)) + try: + return get_or_create_artists_credits_from_musicbrainz( + "recording", + mbid, + attributed_to=attributed_to, + from_activity_id=from_activity_id, + ) + except (NoMbid, ResponseError, NetworkError): + return get_or_create_artists_credits_from_artist_credit_metadata( + artist_credit_data, + attributed_to=attributed_to, + from_activity_id=from_activity_id, + ) + + def _get_track(data, attributed_to=None, query_mb=True, **forced_values): sync_mb_tag = preferences.get("music__sync_musicbrainz_tags") track_uuid = getter(data, "funkwhale", "track", "uuid") @@ -575,66 +648,26 @@ def _get_track(data, attributed_to=None, query_mb=True, **forced_values): pass # get / create artist, artist_credit - album_artists_credits = None - artist_credit_data = getter(data, "artist_credit", default=[]) - if "artist" in forced_values: - artist = forced_values["artist"] - query = Q(artist=artist) - defaults = { - "artist": artist, - "joinphrase": "", - "credit": artist.name, - } - track_artist_credit, created = get_best_candidate_or_create( - models.ArtistCredit, query, defaults=defaults, sort_fields=["mbid", "fid"] - ) - track_artists_credits = [track_artist_credit] - else: - mbid = query_mb and (data.get("musicbrainz_id", None) or data.get("mbid", None)) - try: - track_artists_credits = get_or_create_artists_credits_from_musicbrainz( - "recording", - mbid, - attributed_to=attributed_to, - from_activity_id=from_activity_id, - ) - except (NoMbid, ResponseError, NetworkError): - track_artists_credits = ( - get_or_create_artists_credits_from_artist_credit_metadata( - artist_credit_data, - attributed_to=attributed_to, - from_activity_id=from_activity_id, - ) - ) + track_artists_credits = get_or_create_track_artist_credit( + data, forced_values, attributed_to, from_activity_id, query_mb + ) # get / create album artist, album artist_credit if "album" in forced_values: album = forced_values["album"] album_artists_credits = track_artists_credits else: - if album_artists_credits: - pass - mbid = query_mb and (data.get("musicbrainz_albumid", None) or album_mbid) - try: - album_artists_credits = get_or_create_artists_credits_from_musicbrainz( - "release", - mbid, - attributed_to=attributed_to, - from_activity_id=from_activity_id, - ) - except (NoMbid, ResponseError, NetworkError): - if album_artists := getter(data, "album", "artist_credit", default=None): - album_artists_credits = ( - get_or_create_artists_credits_from_artist_credit_metadata( - album_artists, - attributed_to=attributed_to, - from_activity_id=from_activity_id, - ) - ) - else: - album_artists_credits = track_artists_credits + album_artists_credits = get_or_create_album_artist_credit( + data, + attributed_to, + from_activity_id, + query_mb, + track_artists_credits, + album_mbid, + ) - # get / create album + # get / create album + if "album" not in forced_values: if "album" in data: album_data = data["album"] album_title = album_data["title"] @@ -677,7 +710,6 @@ def _get_track(data, attributed_to=None, query_mb=True, **forced_values): if sync_mb_tag and album_mbid: tags_tasks.sync_fw_item_tag_with_musicbrainz_tags(album) - else: album = None # get / create track @@ -789,7 +821,6 @@ def get_or_create_artist_from_ac(ac_data, attributed_to, from_activity_id): } if ac_data.get("fdate"): defaults["creation_date"] = ac_data.get("fdate") - artist, created = get_best_candidate_or_create( models.Artist, query, defaults=defaults, sort_fields=["mbid", "fid"] ) diff --git a/api/tests/music/test_tasks.py b/api/tests/music/test_tasks.py index de058ea41..6313bb3cf 100644 --- a/api/tests/music/test_tasks.py +++ b/api/tests/music/test_tasks.py @@ -1507,6 +1507,74 @@ def test_can_download_image_file_for_album_mbid(binary_cover, mocker, factories) assert album.attachment_cover.mimetype == "image/jpeg" +def test_import_track_filter_far_right(factories, mocker): + artist = factories["music.Artist"](far_right="QDKFZ585") + artist_credit = factories["music.ArtistCredit"](artist=artist) + upload = factories["music.Upload"]( + playable=True, + track__artist_credit=artist_credit, + track__album__artist_credit=artist_credit, + ) + assert upload.track.mbid is not None + data = { + "title": upload.track.title, + "artist_credit": [{"credit": artist.name, "mbid": artist.mbid}], + "album": { + "title": "The Slip", + "mbid": uuid.UUID("12b57d46-a192-499e-a91f-7da66790a1c1"), + "release_date": datetime.date(2008, 5, 5), + "artist_credit": [{"credit": artist.name, "mbid": artist.mbid}], + }, + "position": 1, + "disc_number": 1, + "mbid": upload.track.mbid, + } + + mb_ac = { + "artist-credit": [ + { + "artist": { + "id": artist.mbid, + "name": artist.name, + }, + "joinphrase": "", + "name": artist.name, + }, + ] + } + mb_ac_album = { + "artist-credit": [ + { + "artist": { + "id": artist.mbid, + "name": artist.name, + }, + "name": artist.name, + "joinphrase": "", + }, + ] + } + + mocker.patch.object( + tasks.musicbrainz.api.recordings, "get", return_value={"recording": mb_ac} + ) + mocker.patch.object( + tasks.musicbrainz.api.releases, "get", return_value={"recording": mb_ac_album} + ) + mocker.patch.object(metadata.TrackMetadataSerializer, "validated_data", data) + mocker.patch.object(tasks, "populate_album_cover") + + new_upload = factories["music.Upload"](library=upload.library) + + tasks.process_upload(upload_id=new_upload.pk) + + new_upload.refresh_from_db() + + assert new_upload.import_status == "errored" + assert new_upload.import_details["error_code"] == "Far right artist detected" + assert "QDKFZ585" in new_upload.import_details["detail"] + + def test_can_import_track_with_same_mbid_in_different_albums(factories, mocker): artist = factories["music.Artist"]() artist_credit = factories["music.ArtistCredit"](artist=artist) diff --git a/docs/specs/far-right-filter/index.md b/docs/specs/far-right-filter/index.md new file mode 100644 index 000000000..88cba019a --- /dev/null +++ b/docs/specs/far-right-filter/index.md @@ -0,0 +1,63 @@ +# Far right filter + +## The issue + +Has an user and an admin I don't want far right content to be displayed in my pod. + +Has a Funkwhale community following this [code of conduct](https://www.funkwhale.audio/code-of-conduct/) I don't want my software to be used to listen to music going against our principles. + +## The solution + +Hard code a filter against far right artists preventing far right movement to use the software, our builds and resources to promote their ideology. + +## Feature behavior + +To find a common consensus/definition of far right ideology we will use wikidata. Moderation and debates can happen on their infrastructure. To be transparent about why an artist in being censored, the backend should display the reference of the wikidata object being used to classify the artist has a far right defender. + +### Backend behavior + +### Backend + +- [ ] a cli tool to display the list of right wing artists that display the name, the mbid and the wikidata ref. Prompt a warning if mbid is missing so admins can add a mbid. +- [ ] A new database table to save the list of artists OR a new artist attribute `right_wing_extremism` displaying the wikidata id ? since we don't want to bother with moderation, we can only add an attribute. +- [ ] Display an explicit api error response that explain why the artist in banned (link the feature documentation and the artist wikidata id : + - [ ] on the import process : display an explicit error during import. + - [ ] on the federation artist serializers : + +workflow : querying wikidata -> create or update artist entries with the new `far_right` attribute -> filter out the artist based on the attribute and display logging info explaining why + +#### Wikidata query + +``` +SELECT DISTINCT ?item ?itemLabel WHERE { + SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE]". } + + ?item wdt:P31 wd:Q215380. + # Match items with the relevant properties and ensure they have references + { + VALUES ?genre { + wd:Q533914 # NSBM + wd:Q224694 # whit power music + wd:Q113084468 # nazi rock + wd:Q121411631 # neonazi music + wd:Q1547998 # rock identitaire francai + wd:Q602498 # nazi punk + wd:Q3328582 # italian right wing alternative + wd:Q828181 # rock against communism + } + + ?item p:P136 ?statement. + ?statement ps:P136 ?genre. + + + # Ensure that these statements have references + FILTER EXISTS { ?statement prov:wasDerivedFrom ?reference. } + } +} +``` + +#### Import + +get_or_create_artists_credits_from_musicbrainz + +### Frontend behavior