Brand new file importer

CHANGELOG

@@ -2,7 +2,15 @@ Changelog
 =========
 
 
-0.2.7 (Unreleased)
+0.3 (Unreleased)
+------------------
+
+- Revamped all import logic, everything is more tested and consistend
+- Can now use Acoustid in file imports to automatically grab metadata from musicbrainz
+- Brand new file import wizard
+
+
+0.2.7
 ------------------
 
 - Shortcuts: can now use the ``f`` shortcut to toggle the currently playing track

api/config/api_urls.py

@@ -15,6 +15,7 @@ router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles')
 router.register(r'artists', views.ArtistViewSet, 'artists')
 router.register(r'albums', views.AlbumViewSet, 'albums')
 router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches')
+router.register(r'import-jobs', views.ImportJobViewSet, 'import-jobs')
 router.register(r'submit', views.SubmitViewSet, 'submit')
 router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists')
 router.register(

api/funkwhale_api/common/permissions.py

@@ -1,6 +1,6 @@
 from django.conf import settings
 
-from rest_framework.permissions import BasePermission
+from rest_framework.permissions import BasePermission, DjangoModelPermissions
 
 
 class ConditionalAuthentication(BasePermission):
@@ -9,3 +9,14 @@ class ConditionalAuthentication(BasePermission):
         if settings.API_AUTHENTICATION_REQUIRED:
             return request.user and request.user.is_authenticated
         return True
+
+
+class HasModelPermission(DjangoModelPermissions):
+    """
+    Same as DjangoModelPermissions, but we pin the model:
+
+        class MyModelPermission(HasModelPermission):
+            model = User
+    """
+    def get_required_permissions(self, method, model_cls):
+        return super().get_required_permissions(method, self.model)

api/funkwhale_api/music/models.py

@@ -405,8 +405,11 @@ class ImportBatch(models.Model):
     @property
     def status(self):
         pending = any([job.status == 'pending' for job in self.jobs.all()])
+        errored = any([job.status == 'errored' for job in self.jobs.all()])
         if pending:
             return 'pending'
+        if errored:
+            return 'errored'
         return 'finished'
 
 class ImportJob(models.Model):
@@ -418,7 +421,7 @@ class ImportJob(models.Model):
         null=True,
         blank=True,
         on_delete=models.CASCADE)
-    source = models.URLField()
+    source = models.CharField(max_length=500)
     mbid = models.UUIDField(editable=False, null=True, blank=True)
     STATUS_CHOICES = (
         ('pending', 'Pending'),

api/funkwhale_api/music/serializers.py

@@ -113,7 +113,8 @@ class ImportJobSerializer(serializers.ModelSerializer):
     track_file = TrackFileSerializer(read_only=True)
     class Meta:
         model = models.ImportJob
-        fields = ('id', 'mbid', 'source', 'status', 'track_file')
+        fields = ('id', 'mbid', 'batch', 'source', 'status', 'track_file', 'audio_file')
+        read_only_fields = ('status', 'track_file')
 
 
 class ImportBatchSerializer(serializers.ModelSerializer):
@@ -121,3 +122,4 @@ class ImportBatchSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.ImportBatch
         fields = ('id', 'jobs', 'status', 'creation_date')
+        read_only_fields = ('creation_date',)

api/funkwhale_api/music/tasks.py

@@ -2,6 +2,7 @@ from django.core.files.base import ContentFile
 
 from funkwhale_api.taskapp import celery
 from funkwhale_api.providers.acoustid import get_acoustid_client
+from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path
 
 from django.conf import settings
 from . import models

api/funkwhale_api/music/views.py

@@ -6,7 +6,7 @@ from django.urls import reverse
 from django.db import models, transaction
 from django.db.models.functions import Length
 from django.conf import settings
-from rest_framework import viewsets, views
+from rest_framework import viewsets, views, mixins
 from rest_framework.decorators import detail_route, list_route
 from rest_framework.response import Response
 from rest_framework import permissions
@@ -15,7 +15,8 @@ from django.contrib.auth.decorators import login_required
 from django.utils.decorators import method_decorator
 
 from funkwhale_api.musicbrainz import api
-from funkwhale_api.common.permissions import ConditionalAuthentication
+from funkwhale_api.common.permissions import (
+    ConditionalAuthentication, HasModelPermission)
 from taggit.models import Tag
 
 from . import models
@@ -71,16 +72,45 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
     ordering_fields = ('creation_date',)
 
 
-class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet):
+class ImportBatchViewSet(
+        mixins.CreateModelMixin,
+        mixins.ListModelMixin,
+        mixins.RetrieveModelMixin,
+        viewsets.GenericViewSet):
     queryset = (
         models.ImportBatch.objects.all()
                           .prefetch_related('jobs__track_file')
                           .order_by('-creation_date'))
     serializer_class = serializers.ImportBatchSerializer
+    permission_classes = (permissions.DjangoModelPermissions, )
 
     def get_queryset(self):
         return super().get_queryset().filter(submitted_by=self.request.user)
 
+    def perform_create(self, serializer):
+        serializer.save(submitted_by=self.request.user)
+
+
+class ImportJobPermission(HasModelPermission):
+    # not a typo, perms on import job is proxied to import batch
+    model = models.ImportBatch
+
+
+class ImportJobViewSet(
+        mixins.CreateModelMixin,
+        viewsets.GenericViewSet):
+    queryset = (models.ImportJob.objects.all())
+    serializer_class = serializers.ImportJobSerializer
+    permission_classes = (ImportJobPermission, )
+
+    def get_queryset(self):
+        return super().get_queryset().filter(batch__submitted_by=self.request.user)
+
+    def perform_create(self, serializer):
+        source = 'file://' + serializer.validated_data['audio_file'].name
+        serializer.save(source=source)
+        tasks.import_job_run.delay(import_job_id=serializer.instance.pk)
+
 
 class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
     """

api/tests/music/test_api.py

@@ -1,4 +1,5 @@
 import json
+import os
 import pytest
 from django.urls import reverse
 
@@ -8,6 +9,8 @@ from funkwhale_api.music import serializers
 
 from . import data as api_data
 
+DATA_DIR = os.path.dirname(os.path.abspath(__file__))
+
 
 def test_can_submit_youtube_url_for_track_import(mocker, superuser_client):
     mocker.patch(
@@ -189,6 +192,48 @@ def test_user_can_query_api_for_his_own_batches(client, factories):
     assert results['results'][0]['jobs'][0]['mbid'] == job.mbid
 
 
+def test_user_can_create_an_empty_batch(client, factories):
+    user = factories['users.SuperUser']()
+    url = reverse('api:v1:import-batches-list')
+    client.login(username=user.username, password='test')
+    response = client.post(url)
+
+    assert response.status_code == 201
+
+    batch = user.imports.latest('id')
+
+    assert batch.submitted_by == user
+    assert batch.source == 'api'
+
+
+def test_user_can_create_import_job_with_file(client, factories, mocker):
+    path = os.path.join(DATA_DIR, 'test.ogg')
+    m = mocker.patch('funkwhale_api.music.tasks.import_job_run.delay')
+    user = factories['users.SuperUser']()
+    batch = factories['music.ImportBatch'](submitted_by=user)
+    url = reverse('api:v1:import-jobs-list')
+    client.login(username=user.username, password='test')
+    with open(path, 'rb') as f:
+        content = f.read()
+        f.seek(0)
+        response = client.post(url, {
+            'batch': batch.pk,
+            'audio_file': f,
+            'source': 'file://'
+        }, format='multipart')
+
+    assert response.status_code == 201
+
+    job = batch.jobs.latest('id')
+
+    assert job.status == 'pending'
+    assert job.source.startswith('file://')
+    assert 'test.ogg' in job.source
+    assert job.audio_file.read() == content
+
+    m.assert_called_once_with(import_job_id=job.pk)
+
+
 def test_can_search_artist(factories, client):
     artist1 = factories['music.Artist']()
     artist2 = factories['music.Artist']()

deploy/nginx.conf

@@ -47,6 +47,8 @@ server {
         rewrite ^(.+)$ /index.html last;
     }
     location /api/ {
+        # this is needed if you have file import via upload enabled
+        client_max_body_size 30M;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

docker/nginx/conf.dev

@@ -30,7 +30,7 @@ http {
     server {
         listen 6001;
         charset     utf-8;
-
+        client_max_body_size 20M;
         location /_protected/media {
             internal;
             alias   /protected/media;

front/package.json

@@ -23,6 +23,7 @@
     "vue-lazyload": "^1.1.4",
     "vue-resource": "^1.3.4",
     "vue-router": "^2.3.1",
+    "vue-upload-component": "^2.7.4",
     "vuedraggable": "^2.14.1",
     "vuex": "^3.0.1",
     "vuex-persistedstate": "^2.4.2"

front/src/components/library/import/BatchDetail.vue

@@ -8,6 +8,7 @@
         ['ui',
         {'active': batch.status === 'pending'},
         {'warning': batch.status === 'pending'},
+        {'error': batch.status === 'errored'},
         {'success': batch.status === 'finished'},
         'progress']">
         <div class="bar" :style="progressBarStyle">
@@ -37,7 +38,7 @@
             </td>
             <td>
               <span
-                :class="['ui', {'yellow': job.status === 'pending'}, {'green': job.status === 'finished'}, 'label']">{{ job.status }}</span>
+                :class="['ui', {'yellow': job.status === 'pending'}, {'red': job.status === 'errored'}, {'green': job.status === 'finished'}, 'label']">{{ job.status }}</span>
             </td>
             <td>
               <router-link v-if="job.track_file" :to="{name: 'library.tracks.detail', params: {id: job.track_file.track }}">{{ job.track_file.track }}</router-link>
@@ -89,7 +90,7 @@ export default {
   computed: {
     progress () {
       return this.batch.jobs.filter(j => {
-        return j.status === 'finished'
+        return j.status !== 'pending'
       }).length * 100 / this.batch.jobs.length
     },
     progressBarStyle () {

front/src/components/library/import/BatchList.vue

@@ -32,7 +32,7 @@
             <td>{{ result.jobs.length }}</td>
             <td>
               <span
-                :class="['ui', {'yellow': result.status === 'pending'}, {'green': result.status === 'finished'}, 'label']">{{ result.status }}</span>
+                :class="['ui', {'yellow': result.status === 'pending'}, {'red': result.status === 'errored'}, {'green': result.status === 'finished'}, 'label']">{{ result.status }}</span>
               </td>
             </tr>
           </tbody>

front/src/components/library/import/FileUpload.vue

@@ -0,0 +1,140 @@
+<template>
+  <div>
+    <div v-if="batch" class="ui two buttons">
+      <file-upload-widget
+        class="ui icon button"
+        :post-action="uploadUrl"
+        :multiple="true"
+        :size="1024 * 1024 * 30"
+        :data="uploadData"
+        :drop="true"
+        extensions="ogg,mp3"
+        accept="audio/*"
+        v-model="files"
+        name="audio_file"
+        :thread="3"
+        @input-filter="inputFilter"
+        @input-file="inputFile"
+        ref="upload">
+        <i class="upload icon"></i>
+        Select files to upload...
+    </file-upload-widget>
+      <button class="ui icon teal button" v-if="!$refs.upload || !$refs.upload.active" @click.prevent="$refs.upload.active = true">
+        <i class="play icon" aria-hidden="true"></i>
+        Start Upload
+      </button>
+      <button type="button" class="ui icon yellow button" v-else @click.prevent="$refs.upload.active = false">
+        <i class="pause icon" aria-hidden="true"></i>
+        Stop Upload
+      </button>
+    </div>
+    <div class="ui hidden divider"></div>
+    <p>
+      Once all your files are uploaded, simply head over  <router-link :to="{name: 'library.import.batches.detail', params: {id: batch.id }}">import detail page</router-link> to check the import status.
+    </p>
+    <table class="ui single line table">
+      <thead>
+        <tr>
+          <th>File name</th>
+          <th>Size</th>
+          <th>Status</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr v-for="(file, index) in files" :key="file.id">
+          <td>{{ file.name }}</td>
+          <td>{{ file.size }}</td>
+          <td>
+            <span v-if="file.error" class="ui red label">
+              {{ file.error }}
+            </span>
+            <span v-else-if="file.success" class="ui green label">Success</span>
+            <span v-else-if="file.active" class="ui yellow label">Uploading...</span>
+            <template v-else>
+              <span class="ui label">Pending</span>
+              <button class="ui tiny basic red icon button" @click.prevent="$refs.upload.remove(file)"><i class="delete icon"></i></button>
+            </template>
+          </td>
+        </tr>
+      </tbody>
+    </table>
+  </div>
+</template>
+
+<script>
+import Vue from 'vue'
+import logger from '@/logging'
+import FileUploadWidget from './FileUploadWidget'
+import config from '@/config'
+
+export default {
+  components: {
+    FileUploadWidget
+  },
+  data () {
+    return {
+      files: [],
+      uploadUrl: config.API_URL + 'import-jobs/',
+      batch: null
+    }
+  },
+  mounted: function () {
+    this.createBatch()
+  },
+  methods: {
+    inputFilter (newFile, oldFile, prevent) {
+      if (newFile && !oldFile) {
+        let extension = newFile.name.split('.').pop()
+        if (['ogg', 'mp3'].indexOf(extension) < 0) {
+          prevent()
+        }
+      }
+    },
+    inputFile (newFile, oldFile) {
+      if (newFile && !oldFile) {
+        // add
+        console.log('add', newFile)
+        if (!this.batch) {
+          this.createBatch()
+        }
+      }
+      if (newFile && oldFile) {
+        // update
+        console.log('update', newFile)
+      }
+      if (!newFile && oldFile) {
+        // remove
+        console.log('remove', oldFile)
+      }
+    },
+    createBatch () {
+      let self = this
+      let url = config.API_URL + 'import-batches/'
+      let resource = Vue.resource(url)
+      resource.save({}, {}).then((response) => {
+        self.batch = response.data
+      }, (response) => {
+        logger.default.error('error while launching creating batch')
+      })
+    }
+  },
+  computed: {
+    batchId: function () {
+      if (this.batch) {
+        return this.batch.id
+      }
+      return null
+    },
+    uploadData: function () {
+      return {
+        'batch': this.batchId,
+        'source': 'file://'
+      }
+    }
+  }
+}
+</script>
+
+<!-- Add "scoped" attribute to limit CSS to this component only -->
+<style scoped>
+</style>

front/src/components/library/import/FileUploadWidget.vue

@@ -0,0 +1,34 @@
+<script>
+import FileUpload from 'vue-upload-component'
+
+export default {
+  extends: FileUpload,
+  methods: {
+    uploadHtml5 (file) {
+      let form = new window.FormData()
+      let value
+      for (let key in file.data) {
+        value = file.data[key]
+        if (value && typeof value === 'object' && typeof value.toString !== 'function') {
+          if (value instanceof File) {
+            form.append(key, value, value.name)
+          } else {
+            form.append(key, JSON.stringify(value))
+          }
+        } else if (value !== null && value !== undefined) {
+          form.append(key, value)
+        }
+      }
+      form.append(this.name, file.file, file.file.filename || file.name)
+      let xhr = new XMLHttpRequest()
+      xhr.open('POST', file.postAction)
+      xhr.setRequestHeader('Authorization', this.$store.getters['auth/header'])
+      return this.uploadXhr(xhr, file, form)
+    }
+  }
+}
+</script>
+
+<!-- Add "scoped" attribute to limit CSS to this component only -->
+<style scoped>
+</style>

front/src/components/library/import/Main.vue

@@ -39,8 +39,8 @@
               </div>
             </div>
             <div class="field">
-              <div class="ui disabled radio checkbox">
+              <div class="ui radio checkbox">
-                <input type="radio" id="upload" value="upload" v-model="currentSource" disabled>
+                <input type="radio" id="upload" value="upload" v-model="currentSource">
                 <label for="upload">File upload</label>
               </div>
             </div>
@@ -84,8 +84,14 @@
           </div>
         </div>
         <div v-if="currentStep === 2">
+          <file-upload
+            ref="import"
+            v-if="currentSource == 'upload'"
+            ></file-upload>
+
           <component
             ref="import"
+            v-if="currentSource == 'external'"
             :metadata="metadata"
             :is="importComponent"
             :backends="backends"
@@ -119,6 +125,7 @@ import MetadataSearch from '@/components/metadata/Search'
 import ReleaseCard from '@/components/metadata/ReleaseCard'
 import ArtistCard from '@/components/metadata/ArtistCard'
 import ReleaseImport from './ReleaseImport'
+import FileUpload from './FileUpload'
 import ArtistImport from './ArtistImport'
 
 import router from '@/router'
@@ -130,7 +137,8 @@ export default {
     ArtistCard,
     ReleaseCard,
     ArtistImport,
-    ReleaseImport
+    ReleaseImport,
+    FileUpload
   },
   props: {
     mbType: {type: String, required: false},
@@ -142,7 +150,7 @@ export default {
       currentType: this.mbType || 'artist',
       currentId: this.mbId,
       currentStep: 0,
-      currentSource: this.source || 'external',
+      currentSource: '',
       metadata: {},
       isImporting: false,
       importData: {