diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ecf364486..e27cdeecb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,7 @@ variables: DOCKER_HOST: tcp://docker:2375/ DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - BUILD_PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 + DOCKER_BUILD_PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 workflow: rules: @@ -28,447 +28,45 @@ stages: - build - publish -review_front: - interruptible: true - stage: deploy - image: node:18-alpine - when: manual - allow_failure: true - variables: - BASE_URL: /-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/ - VUE_APP_ROUTER_BASE_URL: /-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/ - VUE_APP_INSTANCE_URL: $REVIEW_INSTANCE_URL - NODE_ENV: review - before_script: - - apk add --no-cache jq bash coreutils python3 - - rm -rf front-review - - mkdir front-review - - cd front - script: - - yarn install - # this is to ensure we don't have any errors in the output, - # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169 - - yarn run build --base ./ | tee /dev/stderr | (! grep -i 'ERROR in') - - cp -r dist/* ../front-review - artifacts: - expire_in: 2 weeks - paths: - - front-review - cache: - key: "funkwhale__front_dependencies" - paths: - - front/node_modules - - front/yarn.lock - tags: - - docker - environment: - name: review/front/$CI_COMMIT_REF_NAME - url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/index.html - -review_docs: - interruptible: true - stage: deploy - allow_failure: true - image: python:3.10 - variables: - BUILD_PATH: "../docs-review" - before_script: - - rm -rf docs-review - - mkdir docs-review - - cd docs - - apt-get update - - apt-get install -y graphviz git - - pip install poetry - - poetry install - - git switch develop && git pull - - git switch stable && git pull - - git switch $CI_COMMIT_BRANCH && git pull - script: - - poetry run python -m sphinx . $BUILD_PATH - cache: - key: "$CI_PROJECT_ID__sphinx" - paths: - - "$PIP_CACHE_DIR" - artifacts: - expire_in: 2 weeks - paths: - - docs-review - tags: - - docker - environment: - name: review/docs/$CI_COMMIT_REF_NAME - url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/docs-review/index.html - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - docs/**/* - - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH - when: manual - -changelog_snippet: - interruptible: true - image: alpine:3.17 - stage: lint - before_script: - - apk add git - - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - script: - - git diff --name-only FETCH_HEAD | grep "changes/changelog.d/*" - rules: - - if: $CI_COMMIT_AUTHOR == 'Renovate Bot ' - when: never - - if: $CI_MERGE_REQUEST_TITLE =~ /NOCHANGELOG/ - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - - -black: - interruptible: true - image: python:3.10 - stage: lint - variables: - GIT_STRATEGY: fetch - before_script: - - pip install black - script: - - black --check --diff . - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - api/**/* - -flake8: - interruptible: true - image: python:3.7 - stage: lint - variables: - GIT_STRATEGY: fetch - before_script: - - pip install 'flake8<3.7' - script: - - flake8 -v api - cache: - key: "$CI_PROJECT_ID__flake8_pip_cache" - paths: - - "$PIP_CACHE_DIR" - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - api/**/* - -eslint: - interruptible: true - image: node:18-alpine - stage: lint - before_script: - - cd front - - apk add --no-cache jq bash coreutils python3 - - yarn install - script: - - yarn lint --max-warnings 0 - - yarn lint:tsc - cache: - key: "$CI_PROJECT_ID__eslint_npm_cache" - paths: - - front/node_modules - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - front/**/* - -test_api: - interruptible: true - services: - - postgres:15-alpine - - redis:7-alpine - stage: test - retry: 1 - cache: - key: "$CI_PROJECT_ID__pip_cache" - paths: - - "$PIP_CACHE_DIR" - variables: - DATABASE_URL: "postgresql://postgres@postgres/postgres" - FUNKWHALE_URL: "https://funkwhale.ci" - DJANGO_SETTINGS_MODULE: config.settings.local - POSTGRES_HOST_AUTH_METHOD: trust - CACHE_URL: "redis://redis:6379/0" - before_script: - - cd api - - poetry install --no-root - script: - - poetry run pytest --cov-report xml --cov-report term-missing:skip-covered --cov=funkwhale_api --junitxml=report.xml tests/ - tags: - - docker - artifacts: - expire_in: 2 weeks - reports: - junit: api/report.xml - coverage_report: - coverage_format: cobertura - path: api/coverage.xml +build_dep_wheels: + tags: [docker, multiarch] parallel: matrix: - - PY_VER: ["3.7", "3.8", "3.9", "3.10", "3.11"] - image: $CI_REGISTRY/funkwhale/backend-test-docker:$PY_VER - coverage: '/TOTAL\s*\d*\s*\d*\s*(\d*%)/' - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push" - changes: - - api/**/* - - if: $CI_COMMIT_REF_PROTECTED == "true" - when: always - -# Those tests are disabled for now since no vitest dom emulation is providing -# AudioContext, which is required for our HTML audio player -#test_front: -# interruptible: true -# stage: test -# image: node:18-alpine -# before_script: -# - cd front -# - apk add --no-cache jq bash coreutils python3 -# script: -# - yarn install --check-files -# - yarn test:unit -# cache: -# key: "funkwhale__front_dependencies" -# paths: -# - front/node_modules -# - front/yarn.lock -# artifacts: -# name: "front_${CI_COMMIT_REF_NAME}" -# paths: -# - front/dist/ -# reports: -# junit: front/coverage/cobertura-coverage.xml -# tags: -# - docker -# rules: -# - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push" -# changes: -# - front/**/* -# - if: $CI_COMMIT_REF_PROTECTED == "true" -# when: always - -build_docs: - stage: build - image: $CI_REGISTRY/funkwhale/backend-test-docker:3.10 - services: - - postgres:15-alpine - - redis:7-alpine - cache: - key: "$CI_PROJECT_ID__pip_cache" - paths: - - "$PIP_CACHE_DIR" - variables: - DATABASE_URL: "postgresql://postgres@postgres/postgres" - FUNKWHALE_URL: "https://funkwhale.ci" - DJANGO_SETTINGS_MODULE: config.settings.local - POSTGRES_HOST_AUTH_METHOD: trust - CACHE_URL: "redis://redis:6379/0" - API_TYPE: "v1" + - DOCKER_PLATFORM: [amd64, arm64v8] # arm32v7, riscv64] + image: ${DOCKER_PLATFORM}/alpine:3.17 before_script: - - cd api - - pip3 install poetry - - poetry install - - poetry run python manage.py migrate + -> + apk add + musl-dev + gcc + postgresql-dev + python3-dev + py3-psycopg2 + py3-cryptography + py3-lxml + py3-pillow + libldap + libffi-dev + make + zlib-dev + jpeg-dev + openldap-dev + openssl-dev + cargo + libxml2-dev + libxslt-dev + curl + poetry + pip + - poetry export --without-hashes > requirements.txt + - python -m venv venv + - source venv/bin/activate + - pip install --upgrade pip script: - - poetry run python manage.py spectacular --file ../docs/schema.yml - tags: - - docker + - source venv/bin/activate + - pip wheel --check-build-dependencies -w wheelhome -r requirements.txt + - rm wheelhome/*-none-any.whl wheelhome/*-musllinux_*.whl + - ls -l wheelhome artifacts: - expire_in: 2 weeks paths: - - docs/schema.yml - -build_front: - stage: build - image: node:18-alpine - before_script: - - apk add --no-cache jq bash coreutils python3 - - cd front - script: - - yarn install - - yarn run i18n-compile - # this is to ensure we don't have any errors in the output, - # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169 - - yarn run build:deployment | tee /dev/stderr | (! grep -i 'ERROR in') - - chmod -R 755 dist - artifacts: - name: "front_${CI_COMMIT_REF_NAME}" - paths: - - front/dist/ - only: - - tags@funkwhale/funkwhale - - stable@funkwhale/funkwhale - - develop@funkwhale/funkwhale - tags: - - docker - -build_documentation: - stage: build - image: python:3.10 - variables: - BUILD_PATH: "../public" - before_script: - - cd docs - - apt-get update - - apt-get install -y graphviz git - - pip install poetry - - poetry install - - git switch develop && git pull - - git switch stable && git pull - - git switch $CI_COMMIT_BRANCH && git pull - script: - - ./build_docs.sh - cache: - key: "$CI_PROJECT_ID__sphinx" - paths: - - "$PIP_CACHE_DIR" - artifacts: - expire_in: 2 weeks - paths: - - public - rules: - - if: $CI_COMMIT_BRANCH == "stable" - when: always - - if: $CI_COMMIT_BRANCH == "develop" - changes: - - docs/**/* - when: always - tags: - - docker - -deploy_documentation: - stage: publish - image: alpine - dependencies: - - build_documentation - before_script: - - apk add openssh-client rsync - - mkdir -p ~/.ssh - - echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - - eval `ssh-agent -s` - - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - - script: - - rsync -r -e "ssh -p 2282" $CI_PROJECT_DIR/public/ docs@docs.funkwhale.audio:/htdocs/ - only: - - stable - - -.docker_publish: - stage: publish - image: egon0/docker-with-buildx-and-git:bash - tags: - - multiarch - services: - - docker:20-dind - before_script: - - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - cache: - key: docker_public_${CI_COMMIT_REF_NAME} - paths: - - ~/.cargo - script: - -docker_publish_stable_release: - # Publish a docker image for releases - extends: .docker_publish - variables: - IMAGE_NAME: funkwhale/$COMPONENT - rules: - - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^[0-9]+(.[0-9]+){1,2}$/ - script: - # Check if this is the latest release - - ./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG - && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=; - - export major="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1)" - - export minor="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1,2)" - - cd $COMPONENT - - docker buildx create --use --name A$CI_COMMIT_SHORT_SHA - - docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE $DOCKER_LATEST_TAG -t $IMAGE_NAME:$major -t $IMAGE_NAME:$minor . - parallel: - matrix: - - COMPONENT: ["api", "front"] - -docker_publish_unstable_release: - # Publish a docker image for releases - extends: .docker_publish - variables: - IMAGE_NAME: funkwhale/$COMPONENT - rules: - - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME !~ /^[0-9]+(.[0-9]+){1,2}$/ - script: - # Check if this is the latest release - - cd $COMPONENT - - docker buildx create --use --name A$CI_COMMIT_SHORT_SHA - - docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE . - parallel: - matrix: - - COMPONENT: ["api", "front"] - -docker_publish_non-release: - # Publish a docker image for each commit on develop - extends: .docker_publish - variables: - IMAGE_NAME: funkwhale/$COMPONENT - only: - - develop@funkwhale/funkwhale - - stable@funkwhale/funkwhale - script: - - ./scripts/set-api-build-metadata.sh $CI_COMMIT_SHORT_SHA - - cd $COMPONENT - - docker buildx create --use --name A$CI_COMMIT_SHORT_SHA - - docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE . - parallel: - matrix: - - COMPONENT: ["api", "front"] - -docker_all_in_one_release: - stage: publish - image: egon0/docker-with-buildx-and-git:bash - allow_failure: true - services: - - docker:20-dind - variables: - ALL_IN_ONE_REF: main - ALL_IN_ONE_ARTIFACT_URL: https://dev.funkwhale.audio/funkwhale/funkwhale-docker-all-in-one/-/archive/$ALL_IN_ONE_REF/funkwhale-docker-all-in-one-$ALL_IN_ONE_REF.zip - BUILD_PATH: all_in_one - before_script: - - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "stable" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); - script: - - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi - - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip - - unzip -o all_in_one.zip -d tmpdir - - mv tmpdir/funkwhale-docker-all-in-one-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir - - cp -r api $BUILD_PATH/src/api - - cp -r front $BUILD_PATH/src/front - - cd $BUILD_PATH - - ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME - - docker build -t $ALL_IN_ONE_IMAGE $DOCKER_LATEST_TAG . - - docker push $ALL_IN_ONE_IMAGE - - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi - only: - - develop@funkwhale/funkwhale - - stable@funkwhale/funkwhale - - tags@funkwhale/funkwhale - -build_api: - # Simply publish a zip containing api/ directory - stage: publish - image: bash - artifacts: - name: "api_${CI_COMMIT_REF_NAME}" - paths: - - api - script: - - rm -rf api/tests - - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "stable" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); - - chmod -R 750 api - - echo Done! - only: - - tags@funkwhale/funkwhale - - stable@funkwhale/funkwhale - - develop@funkwhale/funkwhale + - wheelhome \ No newline at end of file