import logging import os from django.conf import settings from django.core.files.base import ContentFile from musicbrainzngs import ResponseError from funkwhale_api.common import preferences from funkwhale_api.federation import activity, actors from funkwhale_api.federation import serializers as federation_serializers from funkwhale_api.providers.acoustid import get_acoustid_client from funkwhale_api.providers.audiofile import tasks as audiofile_tasks from funkwhale_api.taskapp import celery from . import lyrics as lyrics_utils from . import models from . import utils as music_utils logger = logging.getLogger(__name__) @celery.app.task(name="acoustid.set_on_track_file") @celery.require_instance(models.TrackFile, "track_file") def set_acoustid_on_track_file(track_file): client = get_acoustid_client() result = client.get_best_match(track_file.audio_file.path) def update(id): track_file.acoustid_track_id = id track_file.save(update_fields=["acoustid_track_id"]) return id if result: return update(result["id"]) def import_track_from_remote(library_track): metadata = library_track.metadata try: track_mbid = metadata["recording"]["musicbrainz_id"] assert track_mbid # for null/empty values except (KeyError, AssertionError): pass else: return models.Track.get_or_create_from_api(mbid=track_mbid)[0] try: album_mbid = metadata["release"]["musicbrainz_id"] assert album_mbid # for null/empty values except (KeyError, AssertionError): pass else: album, _ = models.Album.get_or_create_from_api(mbid=album_mbid) return models.Track.get_or_create_from_title( library_track.title, artist=album.artist, album=album )[0] try: artist_mbid = metadata["artist"]["musicbrainz_id"] assert artist_mbid # for null/empty values except (KeyError, AssertionError): pass else: artist, _ = models.Artist.get_or_create_from_api(mbid=artist_mbid) album, _ = models.Album.get_or_create_from_title( library_track.album_title, artist=artist ) return models.Track.get_or_create_from_title( library_track.title, artist=artist, album=album )[0] # worst case scenario, we have absolutely no way to link to a # musicbrainz resource, we rely on the name/titles artist, _ = models.Artist.get_or_create_from_name(library_track.artist_name) album, _ = models.Album.get_or_create_from_title( library_track.album_title, artist=artist ) return models.Track.get_or_create_from_title( library_track.title, artist=artist, album=album )[0] def _do_import(import_job, use_acoustid=False): logger.info("[Import Job %s] starting job", import_job.pk) from_file = bool(import_job.audio_file) mbid = import_job.mbid replace = import_job.replace_if_duplicate acoustid_track_id = None duration = None track = None # use_acoustid = use_acoustid and preferences.get('providers_acoustid__api_key') # Acoustid is not reliable, we disable it for now. use_acoustid = False if not mbid and use_acoustid and from_file: # we try to deduce mbid from acoustid client = get_acoustid_client() match = client.get_best_match(import_job.audio_file.path) if match: duration = match["recordings"][0]["duration"] mbid = match["recordings"][0]["id"] acoustid_track_id = match["id"] if mbid: logger.info( "[Import Job %s] importing track from musicbrainz recording %s", import_job.pk, str(mbid), ) track, _ = models.Track.get_or_create_from_api(mbid=mbid) elif import_job.audio_file: logger.info( "[Import Job %s] importing track from uploaded track data at %s", import_job.pk, import_job.audio_file.path, ) track = audiofile_tasks.import_track_data_from_path(import_job.audio_file.path) elif import_job.library_track: logger.info( "[Import Job %s] importing track from federated library track %s", import_job.pk, import_job.library_track.pk, ) track = import_track_from_remote(import_job.library_track) elif import_job.source.startswith("file://"): tf_path = import_job.source.replace("file://", "", 1) logger.info( "[Import Job %s] importing track from local track data at %s", import_job.pk, tf_path, ) track = audiofile_tasks.import_track_data_from_path(tf_path) else: raise ValueError( "Not enough data to process import, " "add a mbid, an audio file or a library track" ) track_file = None if replace: logger.info("[Import Job %s] deleting existing audio file", import_job.pk) track.files.all().delete() elif track.files.count() > 0: logger.info( "[Import Job %s] skipping, we already have a file for this track", import_job.pk, ) if import_job.audio_file: import_job.audio_file.delete() import_job.status = "skipped" import_job.save() return track_file = track_file or models.TrackFile(track=track, source=import_job.source) track_file.acoustid_track_id = acoustid_track_id if from_file: track_file.audio_file = ContentFile(import_job.audio_file.read()) track_file.audio_file.name = import_job.audio_file.name track_file.duration = duration elif import_job.library_track: track_file.library_track = import_job.library_track track_file.mimetype = import_job.library_track.audio_mimetype if import_job.library_track.library.download_files: raise NotImplementedError() else: # no downloading, we hotlink pass elif not import_job.audio_file and not import_job.source.startswith("file://"): # not an inplace import, and we have a source, so let's download it logger.info("[Import Job %s] downloading audio file from remote", import_job.pk) track_file.download_file() elif not import_job.audio_file and import_job.source.startswith("file://"): # in place import, we set mimetype from extension path, ext = os.path.splitext(import_job.source) track_file.mimetype = music_utils.get_type_from_ext(ext) track_file.set_audio_data() track_file.save() # if no cover is set on track album, we try to update it as well: if not track.album.cover: logger.info("[Import Job %s] retrieving album cover", import_job.pk) update_album_cover(track.album, track_file) import_job.status = "finished" import_job.track_file = track_file if import_job.audio_file: # it's imported on the track, we don't need it anymore import_job.audio_file.delete() import_job.save() logger.info("[Import Job %s] job finished", import_job.pk) return track_file def update_album_cover(album, track_file, replace=False): if album.cover and not replace: return if track_file: # maybe the file has a cover embedded? try: metadata = track_file.get_metadata() except FileNotFoundError: metadata = None if metadata: cover = metadata.get_picture("cover_front") if cover: # best case scenario, cover is embedded in the track logger.info("[Album %s] Using cover embedded in file", album.pk) return album.get_image(data=cover) if track_file.source and track_file.source.startswith("file://"): # let's look for a cover in the same directory path = os.path.dirname(track_file.source.replace("file://", "", 1)) logger.info("[Album %s] scanning covers from %s", album.pk, path) cover = get_cover_from_fs(path) if cover: return album.get_image(data=cover) if not album.mbid: return try: logger.info( "[Album %s] Fetching cover from musicbrainz release %s", album.pk, str(album.mbid), ) return album.get_image() except ResponseError as exc: logger.warning( "[Album %s] cannot fetch cover from musicbrainz: %s", album.pk, str(exc) ) IMAGE_TYPES = [("jpg", "image/jpeg"), ("png", "image/png")] def get_cover_from_fs(dir_path): if os.path.exists(dir_path): for e, m in IMAGE_TYPES: cover_path = os.path.join(dir_path, "cover.{}".format(e)) if not os.path.exists(cover_path): logger.debug("Cover %s does not exists", cover_path) continue with open(cover_path, "rb") as c: logger.info("Found cover at %s", cover_path) return {"mimetype": m, "content": c.read()} @celery.app.task(name="ImportJob.run", bind=True) @celery.require_instance( models.ImportJob.objects.filter(status__in=["pending", "errored"]), "import_job" ) def import_job_run(self, import_job, use_acoustid=False): def mark_errored(exc): logger.error("[Import Job %s] Error during import: %s", import_job.pk, str(exc)) import_job.status = "errored" import_job.save(update_fields=["status"]) try: tf = _do_import(import_job, use_acoustid=use_acoustid) return tf.pk if tf else None except Exception as exc: if not settings.DEBUG: try: self.retry(exc=exc, countdown=30, max_retries=3) except Exception: mark_errored(exc) raise mark_errored(exc) raise @celery.app.task(name="ImportBatch.run") @celery.require_instance(models.ImportBatch, "import_batch") def import_batch_run(import_batch): for job_id in import_batch.jobs.order_by("id").values_list("id", flat=True): import_job_run.delay(import_job_id=job_id) @celery.app.task(name="Lyrics.fetch_content") @celery.require_instance(models.Lyrics, "lyrics") def fetch_content(lyrics): html = lyrics_utils._get_html(lyrics.url) content = lyrics_utils.extract_content(html) cleaned_content = lyrics_utils.clean_content(content) lyrics.content = cleaned_content lyrics.save(update_fields=["content"]) @celery.app.task(name="music.import_batch_notify_followers") @celery.require_instance( models.ImportBatch.objects.filter(status="finished"), "import_batch" ) def import_batch_notify_followers(import_batch): if not preferences.get("federation__enabled"): return if import_batch.source == "federation": return library_actor = actors.SYSTEM_ACTORS["library"].get_actor_instance() followers = library_actor.get_approved_followers() jobs = import_batch.jobs.filter( status="finished", library_track__isnull=True, track_file__isnull=False ).select_related("track_file__track__artist", "track_file__track__album__artist") track_files = [job.track_file for job in jobs] collection = federation_serializers.CollectionSerializer( { "actor": library_actor, "id": import_batch.get_federation_url(), "items": track_files, "item_serializer": federation_serializers.AudioSerializer, } ).data for f in followers: create = federation_serializers.ActivitySerializer( { "type": "Create", "id": collection["id"], "object": collection, "actor": library_actor.url, "to": [f.url], } ).data activity.deliver(create, on_behalf_of=library_actor, to=[f.url])