From b70d04eb0bf7a7ce79ed60dea4bc1c8deffbffc4 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 16:45:33 +0200 Subject: [PATCH 01/14] Should now build / publish front assets with runner --- .gitlab-ci.yml | 23 +++++++++++++++++++++++ dev.yml | 18 +++++++++--------- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a63410119..836600d3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,28 @@ image: docker:latest +steps: + - test + - build + +test_api: + stage: test + script: + - ./api/runtests + + tags: + - dind + +build_front: + stage: build + script: + - docker-compose -f dev.yml front rm dist/* + - docker-compose -f dev.yml front npm run build + artifacts: + paths: + - front/dist + + tags: + - dind # When using dind, it's wise to use the overlayfs driver for # improved performance. # variables: diff --git a/dev.yml b/dev.yml index 2144d4b16..526ce1ba8 100644 --- a/dev.yml +++ b/dev.yml @@ -52,12 +52,12 @@ services: - redis - celeryworker - nginx: - env_file: .env.dev - build: ./api/compose/nginx - links: - - api - volumes: - - ./api/funkwhale_api/media:/staticfiles/media - ports: - - "0.0.0.0:6001:80" + # nginx: + # env_file: .env.dev + # build: ./api/compose/nginx + # links: + # - api + # volumes: + # - ./api/funkwhale_api/media:/staticfiles/media + # ports: + # - "0.0.0.0:6001:80" From 0327d72f881fc3ecc7617e6339994137f380fa77 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 16:46:59 +0200 Subject: [PATCH 02/14] Typo in yaml --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 836600d3f..c4e827c34 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ image: docker:latest -steps: +stages: - test - build @@ -20,7 +20,9 @@ build_front: artifacts: paths: - front/dist - + only: + - master + - develop tags: - dind # When using dind, it's wise to use the overlayfs driver for From 27ff3a4ce68798c89ddba0157f7c94835ad86150 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 17:12:41 +0200 Subject: [PATCH 03/14] Fixed failing tests --- api/config/settings/test.py | 1 + api/docker/Dockerfile.base | 10 ---------- api/docker/Dockerfile.local | 12 ------------ api/docker/Dockerfile.test | 10 ++++++++-- api/funkwhale_api/radios/radios.py | 2 +- api/funkwhale_api/radios/tests/test_radios.py | 3 +-- api/requirements/test.txt | 2 +- api/test.yml | 1 - 8 files changed, 12 insertions(+), 29 deletions(-) delete mode 100644 api/docker/Dockerfile.base delete mode 100644 api/docker/Dockerfile.local diff --git a/api/config/settings/test.py b/api/config/settings/test.py index 1323ff35a..b8dd89b04 100644 --- a/api/config/settings/test.py +++ b/api/config/settings/test.py @@ -32,3 +32,4 @@ CELERY_ALWAYS_EAGER = True ########## END CELERY # Your local stuff: Below this line define 3rd party library settings +API_AUTHENTICATION_REQUIRED = False diff --git a/api/docker/Dockerfile.base b/api/docker/Dockerfile.base deleted file mode 100644 index 2617c9587..000000000 --- a/api/docker/Dockerfile.base +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.5 - -ENV PYTHONUNBUFFERED 1 - -# Requirements have to be pulled and installed here, otherwise caching won't work -COPY ./requirements.apt /requirements.apt -COPY ./install_os_dependencies.sh /install_os_dependencies.sh -RUN bash install_os_dependencies.sh install -COPY ./requirements /requirements -RUN pip install -r /requirements/base.txt diff --git a/api/docker/Dockerfile.local b/api/docker/Dockerfile.local deleted file mode 100644 index b70410459..000000000 --- a/api/docker/Dockerfile.local +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.5 - -ENV PYTHONUNBUFFERED 1 - -# Requirements have to be pulled and installed here, otherwise caching won't work -COPY ./requirements.apt /requirements.apt -COPY ./install_os_dependencies.sh /install_os_dependencies.sh -RUN bash install_os_dependencies.sh install -COPY ./requirements /requirements -RUN pip install -r /requirements/local.txt - -WORKDIR /app diff --git a/api/docker/Dockerfile.test b/api/docker/Dockerfile.test index f2e2ef893..0d8224879 100644 --- a/api/docker/Dockerfile.test +++ b/api/docker/Dockerfile.test @@ -1,4 +1,4 @@ -FROM funkwhale/apibase +FROM python:3.5 ENV PYTHONUNBUFFERED 1 @@ -6,8 +6,14 @@ ENV PYTHONUNBUFFERED 1 COPY ./requirements.apt /requirements.apt COPY ./install_os_dependencies.sh /install_os_dependencies.sh RUN bash install_os_dependencies.sh install -COPY ./requirements /requirements + +RUN mkdir /requirements + +COPY ./requirements/base.txt /requirements +RUN pip install -r /requirements/base.txt +COPY ./requirements/local.txt /requirements RUN pip install -r /requirements/local.txt +COPY ./requirements/test.txt /requirements RUN pip install -r /requirements/test.txt WORKDIR /app diff --git a/api/funkwhale_api/radios/radios.py b/api/funkwhale_api/radios/radios.py index 0f2632fe9..43819b9c4 100644 --- a/api/funkwhale_api/radios/radios.py +++ b/api/funkwhale_api/radios/radios.py @@ -50,7 +50,7 @@ class SessionRadio(SimpleRadio): def filter_from_session(self, queryset): already_played = self.session.session_tracks.all().values_list('track', flat=True) - queryset = queryset.exclude(pk__in=already_played) + queryset = queryset.exclude(pk__in=list(already_played)) return queryset def pick(self, **kwargs): diff --git a/api/funkwhale_api/radios/tests/test_radios.py b/api/funkwhale_api/radios/tests/test_radios.py index eb839a426..7d069be9c 100644 --- a/api/funkwhale_api/radios/tests/test_radios.py +++ b/api/funkwhale_api/radios/tests/test_radios.py @@ -74,12 +74,11 @@ class TestRadios(TestCase): def test_can_use_radio_session_to_filter_choices(self): tracks = mommy.make('music.Track', _quantity=30) - radio = radios.RandomRadio() session = radio.start_session(self.user) for i in range(30): - radio.pick() + p = radio.pick() # ensure 30 differents tracks have been suggested tracks_id = [session_track.track.pk for session_track in session.session_tracks.all()] diff --git a/api/requirements/test.txt b/api/requirements/test.txt index e3540b72d..bcb6ef060 100644 --- a/api/requirements/test.txt +++ b/api/requirements/test.txt @@ -7,6 +7,6 @@ coverage==4.0.3 django_coverage_plugin==1.1 flake8==2.5.0 django-test-plus==1.0.11 -factory_boy==2.6.0 +factory_boy>=2.8.1 model_mommy tox diff --git a/api/test.yml b/api/test.yml index c28a8138c..dc50a9b54 100644 --- a/api/test.yml +++ b/api/test.yml @@ -6,4 +6,3 @@ test: - .:/app environment: - DJANGO_SETTINGS_MODULE=config.settings.test - - API_AUTHENTICATION_REQUIRED=False From 381aaf48db9288d7da390ab0ad5e5533945b789c Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 17:18:25 +0200 Subject: [PATCH 04/14] Should now avoid pyc removal warnings --- .gitlab-ci.yml | 3 ++- api/docker/Dockerfile.test | 3 ++- api/test.yml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c4e827c34..eddde7883 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,8 @@ stages: test_api: stage: test script: - - ./api/runtests + - docker-compose -f api/test.yml build + - docker-compose -f api/test.yml run test tags: - dind diff --git a/api/docker/Dockerfile.test b/api/docker/Dockerfile.test index 0d8224879..652470c1e 100644 --- a/api/docker/Dockerfile.test +++ b/api/docker/Dockerfile.test @@ -1,12 +1,13 @@ FROM python:3.5 ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE 1 # Requirements have to be pulled and installed here, otherwise caching won't work COPY ./requirements.apt /requirements.apt COPY ./install_os_dependencies.sh /install_os_dependencies.sh RUN bash install_os_dependencies.sh install - +RUN apt-get install py3clean RUN mkdir /requirements COPY ./requirements/base.txt /requirements diff --git a/api/test.yml b/api/test.yml index dc50a9b54..e95461a72 100644 --- a/api/test.yml +++ b/api/test.yml @@ -1,7 +1,7 @@ test: dockerfile: docker/Dockerfile.test build: . - command: python manage.py test + entrypoint: python manage.py test volumes: - .:/app environment: From 3cb2b31f2060fea8c8ad61209dd0fc1864c11a73 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 17:22:04 +0200 Subject: [PATCH 05/14] Removed failing step in test dockerfile --- api/docker/Dockerfile.test | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/docker/Dockerfile.test b/api/docker/Dockerfile.test index 652470c1e..eda9f9c8f 100644 --- a/api/docker/Dockerfile.test +++ b/api/docker/Dockerfile.test @@ -7,9 +7,8 @@ ENV PYTHONDONTWRITEBYTECODE 1 COPY ./requirements.apt /requirements.apt COPY ./install_os_dependencies.sh /install_os_dependencies.sh RUN bash install_os_dependencies.sh install -RUN apt-get install py3clean -RUN mkdir /requirements +RUN mkdir /requirements COPY ./requirements/base.txt /requirements RUN pip install -r /requirements/base.txt COPY ./requirements/local.txt /requirements From 385c7eb1fe47802b4ed405872d2e0db0f3b746fe Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 17:34:38 +0200 Subject: [PATCH 06/14] More file removal and tweaks to gitlab-ci --- .gitlab-ci.yml | 13 +++++++++---- api/test.yml | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eddde7883..c11016232 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,23 +1,28 @@ image: docker:latest stages: - - test - build + - test test_api: stage: test - script: + before_script: - docker-compose -f api/test.yml build + script: - docker-compose -f api/test.yml run test + after_script: + - docker-compose -f api/test.yml run rm -rf api/funkwhale_api/media/* tags: - dind build_front: stage: build + before_script: + - docker-compose -f dev.yml run front rm dist/* + - docker-compose -f dev.yml build front script: - - docker-compose -f dev.yml front rm dist/* - - docker-compose -f dev.yml front npm run build + - docker-compose -f dev.yml run front npm run build artifacts: paths: - front/dist diff --git a/api/test.yml b/api/test.yml index e95461a72..dc50a9b54 100644 --- a/api/test.yml +++ b/api/test.yml @@ -1,7 +1,7 @@ test: dockerfile: docker/Dockerfile.test build: . - entrypoint: python manage.py test + command: python manage.py test volumes: - .:/app environment: From 3664f47aab543a1cf396a2cb93fac8a9560481ce Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 17:55:07 +0200 Subject: [PATCH 07/14] Try to build front using docker runner instead of dind --- .gitlab-ci.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c11016232..aed400364 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,3 @@ -image: docker:latest - stages: - build - test @@ -18,11 +16,10 @@ test_api: build_front: stage: build - before_script: - - docker-compose -f dev.yml run front rm dist/* - - docker-compose -f dev.yml build front + image: node:6-alpine script: - - docker-compose -f dev.yml run front npm run build + - npm install + - npm run build artifacts: paths: - front/dist @@ -30,7 +27,8 @@ build_front: - master - develop tags: - - dind + - docker + # When using dind, it's wise to use the overlayfs driver for # improved performance. # variables: From 5bac0c9c8d96a6a1d28099e6bfd2283be51168d8 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 17:57:36 +0200 Subject: [PATCH 08/14] Ensure we are in the right directory before building --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aed400364..5eaaec1ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,8 @@ test_api: build_front: stage: build image: node:6-alpine + before_script: + - cd front script: - npm install - npm run build From b50d1c5919b4790f04ef37bb8d5493429f62aae7 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 18:05:29 +0200 Subject: [PATCH 09/14] Should cache node modules between jobs --- .gitlab-ci.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5eaaec1ee..058b505ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,12 +19,19 @@ build_front: image: node:6-alpine before_script: - cd front + script: - npm install - npm run build - artifacts: + - mv dist /dist + cache: + key: "$CI_COMMIT_REF_NAME" paths: - - front/dist + - front/node_modules + artifacts: + name: "front_${CI_COMMIT_REF_NAME}" + paths: + - /dist only: - master - develop From 22f3b42ab117cce524c7e62d3ba85ef69273f9cd Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 18:08:50 +0200 Subject: [PATCH 10/14] Fixing wrong artifact path --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 058b505ff..1fbf126de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,7 +31,7 @@ build_front: artifacts: name: "front_${CI_COMMIT_REF_NAME}" paths: - - /dist + - front/dist/ only: - master - develop From 3e7aaedfbedf724e9ae14354a9021f7420286b53 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 18:10:48 +0200 Subject: [PATCH 11/14] Removed uselss mv statement --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1fbf126de..bc8d4267d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,7 +23,6 @@ build_front: script: - npm install - npm run build - - mv dist /dist cache: key: "$CI_COMMIT_REF_NAME" paths: From 0692dde06d2ac8b6878ce05881bd87c6b7e0665e Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 18:22:17 +0200 Subject: [PATCH 12/14] Missing container name in gitlabci --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc8d4267d..b9f4c5250 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ stages: - - build - test + - build test_api: stage: test @@ -9,7 +9,7 @@ test_api: script: - docker-compose -f api/test.yml run test after_script: - - docker-compose -f api/test.yml run rm -rf api/funkwhale_api/media/* + - docker-compose -f api/test.yml run test rm -rf api/funkwhale_api/media/* tags: - dind @@ -37,6 +37,7 @@ build_front: tags: - docker + # When using dind, it's wise to use the overlayfs driver for # improved performance. # variables: From 2bb3e1d5c666031ff15ac1ed756fab9100c42014 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 18:24:32 +0200 Subject: [PATCH 13/14] Typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9f4c5250..678f8eb5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ test_api: script: - docker-compose -f api/test.yml run test after_script: - - docker-compose -f api/test.yml run test rm -rf api/funkwhale_api/media/* + - docker-compose -f api/test.yml run test rm -rf funkwhale_api/media/* tags: - dind From f7cf397723ebc3efb182a32d71d58331c626a3c9 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 25 Jun 2017 18:27:41 +0200 Subject: [PATCH 14/14] Docker compose / rm does not like wildcards --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 678f8eb5b..8742084d4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ test_api: script: - docker-compose -f api/test.yml run test after_script: - - docker-compose -f api/test.yml run test rm -rf funkwhale_api/media/* + - docker-compose -f api/test.yml run test rm -rf funkwhale_api/media/ tags: - dind