Merge branch '149-better-demo' into 'develop'

Resolve "Improve the demo catalog"

Closes #149

See merge request funkwhale/funkwhale!336

api/demo/demo-user.py

@@ -1,7 +0,0 @@
-from funkwhale_api.users.models import User
-
-
-u = User.objects.create(email="demo@demo.com", username="demo", is_staff=True)
-u.set_password("demo")
-u.subsonic_api_token = "demo"
-u.save()

api/demo/load-demo-data.sh

@@ -1,13 +0,0 @@
-#! /bin/bash
-
-echo "Loading demo data..."
-
-python manage.py migrate --noinput
-
-echo "Creating demo user..."
-
-cat demo/demo-user.py | python manage.py shell -i python
-
-echo "Importing demo tracks..."
-
-python manage.py import_files "/music/**/*.ogg" --recursive --noinput --username demo

api/funkwhale_api/common/preferences.py

@@ -14,6 +14,11 @@ def get(pref):
     return manager[pref]
 
 
+def set(pref, value):
+    manager = global_preferences_registry.manager()
+    manager[pref] = value
+
+
 class StringListSerializer(serializers.BaseSerializer):
     separator = ","
     sort = True

demo/README.md

@@ -0,0 +1,61 @@
+# Setup the demo
+
+We assume you want to store the demo data in `/srv/funkwhale-demo`.
+This setup requires Docker and docker-compose.
+
+## Create the necessary directories
+
+`mkdir /srv/funkwhale-demo`
+
+## Get some music
+
+You can use your own music (put it in `/usr/share/music`, this is the directory the demo will look into by default).
+
+If you don't have any music, you can use the repository https://code.eliotberriot.com/funkwhale/catalog, which
+requires Git LFS.
+
+## Create an env file
+
+Copy the `env.sample` file to ``/srv/funkwhale-demo/.env`.
+
+Edit the file according to your needs.
+
+## Copy the setup script
+
+Copy the `setup.sh` script to ``/srv/funkwhale-demo/setup.sh`.
+
+Ensure it's executable with `chmod +x setup.sh`.
+
+## Setup your nginx vhost
+
+Setup your reverse proxy for the demo as described in https://docs.funkwhale.audio/installation/index.html#nginx.
+
+This is outside of the scope of this guide, as you will probably want some SSL certificates, however,
+ensure you point the vhost configuration to the proper static files:
+
+- `root` should point to `/srv/funkwhale-demo/demo/front/dist`
+- `/media` and `/_protected/media` should point to `/srv/funkwhale-demo/demo/data/media/`
+- `/staticfiles` should point to `/srv/funkwhale-demo/demo/data/static`
+
+## Launch
+
+Setup the demo:
+
+```
+cd /srv/funkwhale-demo
+sudo ENV_FILE=/srv/funkwhale-demo/.env ./setup.sh
+```
+
+## Automate
+
+You'll probaby want to reset the demo every now and then. You can do that
+using a cronjob:
+
+```
+sudo crontab -e
+# in the crontab, put this:
+SHELL=/bin/bash
+0 */3 * * * cd /srv/funkwhale-demo && ENV_FILE=/srv/funkwhale-demo/env ./setup.sh > /srv/funkwhale-demo/crontab.log 2>&1
+```
+
+This will reset and restart the demo every 3 hours.

demo/download-tracks.sh

@@ -1,21 +0,0 @@
-#! /bin/bash
-set -e
-[ -z $1 ] && echo "Path to list file missing" && exit 1
-
-echo "This will download tracks from zip archives listed in $1"
-
-LIST_CONTENT=$(cat $1)
-mkdir -p data/music
-cd data/music
-
-echo "Downloading files..."
-echo "$LIST_CONTENT" | grep "^[^#;]" | xargs -n 1 curl -LO
-
-echo "Unzipping archives..."
-find . -name "*.zip" | while read filename; do
-    dirname="${filename%.*}"
-    mkdir $dirname
-    unzip -o -d "$dirname" "$filename";
-done;
-
-echo "Done!"

demo/env.sample

@@ -0,0 +1,6 @@
+FUNKWHALE_URL=https://demo.funkwhale.audio/
+DJANGO_ALLOWED_HOSTS=demo.funkwhale.audio
+FUNKWHALE_API_PORT=5001
+DJANGO_SECRET_KEY=demo
+DATABASE_URL=postgresql://postgres@postgres/postgres
+CACHE_URL=redis://redis:6379/0

demo/music.txt

@@ -1,8 +0,0 @@
-# You can get a pretty list of open music here: https://archive.org/details/jamendo-albums
-https://archive.org/compress/jamendo-069098/formats=OGG%20VORBIS&file=/jamendo-069098.zip
-https://archive.org/compress/jamendo-001144/formats=OGG%20VORBIS&file=/jamendo-001144.zip
-https://archive.org/compress/jamendo-027690/formats=OGG%20VORBIS&file=/jamendo-027690.zip
-https://archive.org/compress/jamendo-001469/formats=OGG%20VORBIS&file=/jamendo-001469.zip
-https://archive.org/compress/jamendo-106323/formats=OGG%20VORBIS&file=/jamendo-106323.zip
-https://archive.org/compress/jamendo-071149/formats=OGG%20VORBIS&file=/jamendo-071149.zip
-https://archive.org/compress/jamendo-085030/formats=OGG%20VORBIS&file=/jamendo-085030.zip

demo/setup.sh

@@ -1,30 +1,114 @@
 #!/bin/bash -eux
-version="develop"
-music_path="/usr/share/music"
-demo_path="/srv/funkwhale-demo/demo"
-
+version=${VERSION:-develop}
+music_path=${MUSIC_PATH:-/usr/share/music}
+demo_path=${DEMO_PATH:-/srv/funkwhale-demo/demo}
+env_file=${ENV_FILE}
 echo 'Cleaning everything...'
+mkdir -p $demo_path
 cd $demo_path
 /usr/local/bin/docker-compose down -v || echo 'Nothing to stop'
-rm -rf /srv/funkwhale-demo/demo/*
+sudo rm -rf $demo_path/*
 mkdir -p $demo_path
 echo 'Downloading demo files...'
 curl -L -o docker-compose.yml "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/docker-compose.yml"
 curl -L -o .env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/env.prod.sample"
 
 mkdir data/
-cp -r $music_path data/music
-
 curl -L -o front.zip "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/$version/download?job=build_front"
 unzip front.zip
 
-echo "FUNKWHALE_URL=https://demo.funkwhale.audio/" >> .env
-echo "DJANGO_SECRET_KEY=demo" >> .env
-echo "DJANGO_ALLOWED_HOSTS=demo.funkwhale.audio" >> .env
+cat $env_file >> .env
 echo "FUNKWHALE_VERSION=$version" >> .env
-echo "FUNKWHALE_API_PORT=5001" >> .env
-/usr/local/bin/docker-compose pull
+echo "MUSIC_DIRECTORY_SERVE_PATH=$music_path" >> .env
+echo "MUSIC_DIRECTORY_PATH=$music_path" >> .env
+echo "MEDIA_ROOT=$demo_path/data/media/" >> .env
+echo "STATIC_ROOT=$demo_path/data/static/" >> .env
+
+# /usr/local/bin/docker-compose pull
 /usr/local/bin/docker-compose up -d postgres redis
 sleep 5
-/usr/local/bin/docker-compose run --rm api demo/load-demo-data.sh
+cat .env
+cat <<EOF | /usr/local/bin/docker-compose run --rm api python manage.py shell -i python
+import subprocess
+subprocess.call("pip install factory-boy", shell=True)
+
+from django.core.management import call_command
+
+call_command("migrate", interactive=False)
+
+from funkwhale_api.users.models import User
+
+print("Creating dummy user")
+u = User.objects.create(email="demo@demo.com", username="demo", is_staff=True, is_superuser=True, privacy_level="everyone")
+u.set_password("demo")
+u.subsonic_api_token = "demo"
+u.save()
+
+from funkwhale_api.common import preferences
+
+manager = preferences.global_preferences_registry.manager()
+manager['common__api_authentication_required'] = False
+manager['federation__music_needs_approval'] = False
+manager['instance__name'] = "Login: demo / password: demo"
+
+paths = [
+    "$music_path/**/*.ogg",
+    "$music_path/**/*.mp3",
+    "$music_path/**/*.flac",
+]
+print(paths)
+call_command("import_files", *paths, username="demo", recursive=True, interactive=False)
+
+print('Creating some dummy data...')
+
+import random
+import datetime
+from funkwhale_api.music.models import Album, Track
+from funkwhale_api.history.factories import ListeningFactory
+from funkwhale_api.favorites.factories import TrackFavorite as TrackFavoriteFactory
+from funkwhale_api.users.factories import UserFactory
+from funkwhale_api.playlists.factories import PlaylistFactory
+
+users = UserFactory.create_batch(size=15, privacy_level="everyone")
+available_tracks = list(Track.objects.all())
+available_albums = list(Album.objects.all())
+
+def get_random_datetime():
+    from django.utils import timezone
+    import datetime
+    import random
+    now = timezone.now()
+    return now - datetime.timedelta(seconds=random.randint(1, 3600 * 24 * 7))
+
+print('Updating album dates to have random sorting...')
+for album in available_albums:
+    album.creation_date = get_random_datetime()
+    album.save(update_fields=['creation_date'])
+
+for i in range(30):
+    print('Creating playlist {}'.format(i))
+    playlist = PlaylistFactory(user=random.choice(users), privacy_level="everyone", creation_date=get_random_datetime())
+    tracks = set()
+
+    for i in range(random.randint(5, 35)):
+        tracks.add(random.choice(available_tracks))
+
+    playlist.insert_many(tracks)
+
+for user in users:
+    for i in range(random.randint(5, 35)):
+        print('Adding favorite {} for user {}'.format(i, user.username))
+        try:
+            TrackFavoriteFactory(user=user, track=random.choice(available_tracks), creation_date=get_random_datetime())
+        except:
+            pass
+    for i in range(random.randint(5, 35)):
+        print('Adding listening {} for user {}'.format(i, user.username))
+        try:
+            ListeningFactory(user=user, track=random.choice(available_tracks), creation_date=get_random_datetime())
+        except:
+            pass
+EOF
+
+chmod 777 -R front
 /usr/local/bin/docker-compose up -d

scripts/clean-unused-artifacts.py

@@ -0,0 +1,30 @@
+import gitlab
+
+TOKEN = "CHANGEME"
+CLEAN_BEFORE = "2018-07"
+gl = gitlab.Gitlab("https://code.eliotberriot.com", private_token=TOKEN, per_page=100)
+project = gl.projects.get("funkwhale/funkwhale")
+
+jobs = project.jobs.list(as_list=False)
+total = jobs.total
+
+for job in jobs:
+    if job.attributes["ref"] != "develop":
+        continue
+    if job.attributes["status"] != "success":
+        continue
+    if job.attributes["tag"] is True:
+        continue
+    if job.attributes["name"] not in ["build_api", "build_front", "pages"]:
+        continue
+    if job.attributes["created_at"].startswith(CLEAN_BEFORE):
+        continue
+    relevant = {
+        "ref": job.attributes["ref"],
+        "status": job.attributes["status"],
+        "tag": job.attributes["tag"],
+        "name": job.attributes["name"],
+        "created_at": job.attributes["created_at"],
+    }
+    print("Deleting job {}!".format(job.id), relevant)
+    job.erase()