From 1ef252e29d0715c10998580ede147ee09e138015 Mon Sep 17 00:00:00 2001 From: Georg Krause Date: Fri, 28 May 2021 09:39:27 +0000 Subject: [PATCH] Implement several pipeline improvements * Build docker image to run tests in * Collect backend coverage reports * Update dependencies --- .gitlab-ci.yml | 24 ++++++++++++++++++------ api/Dockerfile | 6 ++++-- api/funkwhale_api/federation/models.py | 4 ++-- api/funkwhale_api/federation/tasks.py | 4 +++- api/requirements/base.txt | 18 +++++++++--------- api/requirements/local.txt | 2 +- api/requirements/test.txt | 8 ++++---- 7 files changed, 41 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9d4f7b9f1..ce9d4483e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,8 @@ variables: stages: - review - lint - - test - build + - test - deploy - deps @@ -123,7 +123,7 @@ test_api: - postgres:11 - redis:5 stage: test - image: funkwhale/funkwhale:develop + image: $CI_REGISTRY/$IMAGE cache: key: "$CI_PROJECT_ID__pip_cache" paths: @@ -136,20 +136,21 @@ test_api: only: - branches before_script: - - apk add make git gcc python3-dev musl-dev + - apk add make git gcc python3-dev musl-dev python3-dev openssl-dev cargo - apk add postgresql-dev py3-psycopg2 libldap libffi-dev make zlib-dev jpeg-dev openldap-dev - cd api - pip3 install -r requirements/base.txt - pip3 install -r requirements/local.txt - pip3 install -r requirements/test.txt script: - - pytest --cov=funkwhale_api --junitxml=report.xml tests/ + - pytest --cov-report xml --cov=funkwhale_api --junitxml=report.xml tests/ tags: - docker artifacts: when: always reports: junit: api/report.xml + cobertura: api/coverage.xml test_front: interruptible: true @@ -198,10 +199,21 @@ build_front: - tags@funkwhale/funkwhale - master@funkwhale/funkwhale - develop@funkwhale/funkwhale - tags: - docker +build_backend: + stage: build + image: bash + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + - cd api + - docker build -t $CI_REGISTRY/$IMAGE . + - docker push $CI_REGISTRY/$IMAGE + tags: + - docker-build + pages: stage: test image: python:3.6 @@ -296,7 +308,7 @@ build_api: check_api_dependencies: interruptible: true stage: deps - image: funkwhale/funkwhale:develop + image: $CI_REGISTRY/$IMAGE cache: key: "$CI_PROJECT_ID__pip_cache" paths: diff --git a/api/Dockerfile b/api/Dockerfile index d19bc602f..1ba704e38 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.10 as builder +FROM alpine:3.13 as builder RUN \ echo 'installing dependencies' && \ @@ -15,6 +15,8 @@ RUN \ zlib-dev \ jpeg-dev \ openldap-dev \ + openssl-dev \ + cargo \ && \ \ ln -s /usr/bin/python3 /usr/bin/python @@ -41,7 +43,7 @@ RUN \ if [ "$install_dev_deps" = "1" ] ; then echo "Installing dev dependencies" && pip3 install --no-cache-dir -r /requirements/local.txt -r /requirements/test.txt ; else echo "Skipping dev deps installation" ; fi -FROM alpine:3.10 as build-image +FROM alpine:3.13 as build-image COPY --from=builder /venv /venv # emulate activation by prefixing PATH diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index fa285f3cf..c3868d737 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -142,7 +142,6 @@ class Domain(models.Model): from funkwhale_api.music import models as music_models data = Domain.objects.filter(pk=self.pk).aggregate( - actors=models.Count("actors", distinct=True), outbox_activities=models.Count("actors__outbox_activities", distinct=True), libraries=models.Count("actors__libraries", distinct=True), channels=models.Count("actors__owned_channels", distinct=True), @@ -152,6 +151,7 @@ class Domain(models.Model): emitted_library_follows=models.Count( "actors__library_follows", distinct=True ), + actors=models.Count("actors", distinct=True), ) data["artists"] = music_models.Artist.objects.filter( from_activity__actor__domain_id=self.pk @@ -283,12 +283,12 @@ class Actor(models.Model): data = Actor.objects.filter(pk=self.pk).aggregate( outbox_activities=models.Count("outbox_activities", distinct=True), - libraries=models.Count("libraries", distinct=True), channels=models.Count("owned_channels", distinct=True), received_library_follows=models.Count( "libraries__received_follows", distinct=True ), emitted_library_follows=models.Count("library_follows", distinct=True), + libraries=models.Count("libraries", distinct=True), ) data["artists"] = music_models.Artist.objects.filter( from_activity__actor=self.pk diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index c2bdfad34..cf7d08ebc 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -461,7 +461,9 @@ class PreserveSomeDataCollector(Collector): def related_objects(self, related, *args, **kwargs): qs = super().related_objects(related, *args, **kwargs) - if related.name == "outbox_activities": + # We can only exclude the actions if these fields are available, most likely its a + # model.Activity than + if hasattr(related, "type") and hasattr(related, "creation_date"): # exclude the delete activity can be broadcasted properly qs = qs.exclude(type="Delete", creation_date__gte=self.creation_date) diff --git a/api/requirements/base.txt b/api/requirements/base.txt index dc0a7ca69..4f39ea827 100644 --- a/api/requirements/base.txt +++ b/api/requirements/base.txt @@ -1,5 +1,5 @@ -django~=3.0.8 -setuptools>=49 +django~=3.2.3 +setuptools~=56.2.0 # Configuration django-environ~=0.4.0 @@ -11,7 +11,7 @@ django-allauth~=0.42.0 psycopg2-binary~=2.8.0 # Time zones support -pytz==2020.1 +pytz==2021.1 # Redis support django-redis~=4.12.0 @@ -29,7 +29,7 @@ persisting-theory~=0.2.0 django-versatileimagefield~=2.0.0 django-filter~=2.4.0 django-rest-auth~=0.9.0 -ipython~=7.22.0 +ipython~=7.23.1 mutagen~=1.45.0 pymemoize~=1.0.0 @@ -41,13 +41,13 @@ channels_redis~=3.2.0 uvicorn[standard]~=0.13.4 gunicorn~=20.1.0 -cryptography~=2.9.0 +cryptography~=3.4.7 # requests-http-signature==0.0.3 # clone until the branch is merged and released upstream git+https://github.com/agateblue/requests-http-signature.git@signature-header-support django-cleanup~=5.2.0 requests~=2.25.1 -pyOpenSSL~=19.1.0 +pyOpenSSL~=20.0.1 # for LDAP authentication python-ldap~=3.3.0 @@ -61,14 +61,14 @@ django-oauth-toolkit~=1.3.0 django-storages~=1.11.1 boto3~=1.17.59 unicode-slugify~=0.1.0 -django-cacheops~=5.1.0 +django-cacheops~=6.0.0 +service_identity~=21.1.0 click~=7.1.0 -service_identity~=18.1.0 markdown~=3.3.4 bleach~=3.3.0 feedparser~=6.0.0 -watchdog~=2.0.3 +watchdog~=2.1.2 ## Pin third party dependency to avoid issue with latest version twisted==20.3.0 diff --git a/api/requirements/local.txt b/api/requirements/local.txt index e6d413f1f..f321c81c6 100644 --- a/api/requirements/local.txt +++ b/api/requirements/local.txt @@ -2,7 +2,7 @@ coverage~=5.5.0 django_coverage_plugin~=1.8.0 -factory_boy~=2.11.0 +factory_boy~=2.12.0 # django-debug-toolbar that works with Django 1.5+ django-debug-toolbar~=3.2.1 diff --git a/api/requirements/test.txt b/api/requirements/test.txt index 0f8b6255d..0f217da3d 100644 --- a/api/requirements/test.txt +++ b/api/requirements/test.txt @@ -2,11 +2,11 @@ flake8~=3.9.1 pytest~=6.2.3 -pytest-cov~=2.11.1 -pytest-django~=4.2.0 +pytest-cov~=2.12.0 +pytest-django~=4.3.0 pytest-env~=0.6.0 pytest-mock~=3.6.0 -pytest-randomly~=3.7.0 +pytest-randomly~=3.8.0 pytest-sugar~=0.9.0 requests-mock~=1.9.0 -faker~=8.1.1 +faker~=8.2.0