docs: replace scripts with makefile

Part-of: <https://dev.funkwhale.audio/funkwhale/funkwhale/-/merge_requests/2348>

.gitlab-ci.yml

@@ -118,21 +118,16 @@ review_docs:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes: [docs/**/*]
 
-  image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-api:3.11
+  image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-docs:3.11
-  variables:
-    BUILD_PATH: "../docs-review"
   environment:
     name: review/docs/$CI_COMMIT_REF_NAME
     url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/docs-review/index.html
   cache: *docs_cache
   before_script:
-    - mkdir docs-review
     - cd docs
-    - apt-get update
+    - make install
-    - apt-get install -y graphviz
-    - poetry install
   script:
-    - poetry run python3 -m sphinx . $BUILD_PATH
+    - make build BUILD_DIR=../docs-review
   artifacts:
     expire_in: 2 weeks
     paths:
@@ -372,19 +367,13 @@ build_docs:
     - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
     - changes: [docs/**/*]
 
-  image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-api:3.11
+  image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-docs:3.11
-  variables:
-    BUILD_PATH: "../public"
-    GIT_STRATEGY: clone
-    GIT_DEPTH: 0
   cache: *docs_cache
   before_script:
     - cd docs
-    - apt-get update
+    - make install
-    - apt-get install -y graphviz
-    - poetry install
   script:
-    - ./build_docs.sh
+    - make build-all BUILD_DIR=../public
   artifacts:
     expire_in: 2 weeks
     paths:

changes/changelog.d/replace-docs-scripts-with-makefile.misc

@@ -0,0 +1 @@
+Replace docs scripts with make

dev.yml

@@ -154,12 +154,14 @@ services:
 
   docs:
     build: docs
-    command: python3 serve.py
+    command: make clean dev
+    working_dir: /src/docs
+    user: 1000:1000
     volumes:
-      - ".:/app/"
+      - .:/src
     ports:
-      - "35730:35730"
+      - 8001:8001
-      - "8001:8001"
+      - 35730:35730
 
   api-docs:
     image: swaggerapi/swagger-ui:v3.37.2

docs/Dockerfile

@@ -1,5 +1,19 @@
 FROM python:3.11
 
-RUN apt-get update && apt-get install -y graphviz
+ENV PYTHONDONTWRITEBYTECODE=1
-RUN pip install sphinx livereload sphinx_rtd_theme django-environ django myst-parser sphinx-design sphinx-multiversion sphinxcontrib-mermaid sphinx-copybutton
+ENV PYTHONUNBUFFERED=1
-WORKDIR /app/docs
+ARG PIP_NO_CACHE_DIR=1
+
+RUN set -eux \
+    && DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    curl \
+    git \
+    graphviz \
+    make \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN set -eux \
+    && pip install --upgrade pip \
+    && pip install setuptools wheel pipx \
+    && PIPX_BIN_DIR=/usr/local/bin pipx install poetry

docs/Makefile

@@ -1,20 +1,77 @@
-# Minimal makefile for Sphinx documentation
+SHELL = bash
-#
+CPU_CORES = $(shell N=$$(nproc); echo $$(( $$N > 4 ? 4 : $$N )))
 
-# You can set these variables from the command line.
+# Install
-SPHINXOPTS    =
+VENV = .venv
-SPHINXBUILD   = python3 -msphinx
+export POETRY_VIRTUALENVS_IN_PROJECT=true
-SPHINXPROJ    = funkwhale
-SOURCEDIR     = .
-BUILDDIR      = _build
 
-# Put it first so that "make" without argument is like "make help".
+$(VENV):
-help:
+	$(MAKE) install
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-.PHONY: help Makefile
+install:
+	poetry install
+	poetry run pip install --no-deps --editable ../api
 
-# Catch-all target: route all unknown targets to Sphinx using the new
+clean:
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+	git clean -xdf .
-%: Makefile
+
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+# Sphinx
+SPHINX_OPTS = -j $(CPU_CORES)
+SOURCE_DIR = .
+BUILD_DIR = _build
+
+# Dev
+dev: $(VENV)
+	poetry run _scripts/serve.py
+
+# I18n
+LOCALES = en_GB en_US fr
+
+locale-generate: $(VENV)
+	poetry run sphinx-build -b gettext $(SOURCE_DIR) locales/gettext $(SPHINX_OPTS)
+
+locale-update: $(VENV)
+	poetry run sphinx-intl update -p locales/gettext $(foreach locale,$(LOCALES),-l $(locale))
+
+locale-prune-untranslated: $(VENV)
+	poetry run _scripts/locale-prune-untranslated.py
+
+# Swagger
+SWAGGER_VERSION = 5.1.2
+SWAGGER_RELEASE_URL = https://github.com/swagger-api/swagger-ui/archive/refs/tags/v$(SWAGGER_VERSION).tar.gz
+SWAGGER_BUILD_DIR = swagger
+
+swagger:
+	mkdir "$(SWAGGER_BUILD_DIR)"
+	curl -sSL "$(SWAGGER_RELEASE_URL)" | \
+		tar --extract \
+			--gzip \
+			--directory="$(SWAGGER_BUILD_DIR)" \
+			--strip-components=2 \
+			"swagger-ui-$(SWAGGER_VERSION)/dist"
+
+	sed -i \
+		"s#https://petstore.swagger.io/v2/swagger.json#schema.yml#g" \
+		"$(SWAGGER_BUILD_DIR)/swagger-initializer.js"
+
+	cp schema.yml "$(SWAGGER_BUILD_DIR)/schema.yml"
+
+# Releases
+$(BUILD_DIR)/releases.json:
+	../scripts/releases.py > "$@"
+
+$(BUILD_DIR)/latest.txt:
+	../scripts/releases.py -r -q latest.id > "$@"
+
+releases: $(BUILD_DIR)/releases.json $(BUILD_DIR)/latest.txt
+
+# Build
+build: $(VENV)
+	poetry run sphinx-build $(SOURCE_DIR) $(BUILD_DIR) $(SPHINX_OPTS)
+
+build-translated: $(VENV) locale-prune-untranslated
+	for locale in $(LOCALES); do \
+		poetry run sphinx-build $(SOURCE_DIR) $(BUILD_DIR)/$$locale $(SPHINX_OPTS) -D language=$$locale; \
+	done
+
+build-all: build build-translated releases swagger

docs/_scripts/locale-prune-untranslated.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+import os
+from subprocess import check_output
+
+
+def main() -> int:
+    output = check_output(["poetry", "run", "sphinx-intl", "stat"], text=True)
+    for line in output.splitlines():
+        path, _, comment = line.partition(":")
+        if "0 untranslated." in comment:
+            print(f"removing untranslated po file: {path}")
+            os.unlink(path)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

docs/_scripts/serve.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+from subprocess import check_call
+
+from livereload import Server, shell
+
+BUILD_DIR = "/tmp/_build"
+
+
+def main() -> int:
+    # initial make
+    check_call(["make", "build", f"BUILD_DIR={BUILD_DIR}"])
+
+    server = Server()
+    server.watch("..", shell(f"make build BUILD_DIR={BUILD_DIR}"))
+    server.serve(root=BUILD_DIR, liveport=35730, port=8001, host="0.0.0.0")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

docs/build_docs.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-set -eux
-
-# We clean up translations, only fully translated components are kept
-IFS=$'\n'
-
-for i in $(poetry run sphinx-intl stat); do
-  echo "$i"
-  if [[ "$i" != *" 0 untranslated." ]]; then
-    file=$(echo "$i" | cut -d: -f1)
-    echo "delete $file"
-    rm "$file"
-  fi
-done
-
-# Build sphinx
-poetry run sphinx-build . "$BUILD_PATH"
-for path in locales/*; do
-  lang="$(basename "$path")"
-  if [[ "$lang" != "gettext" ]]; then
-    poetry run sphinx-build -D language="$lang" . "$BUILD_PATH/$lang"
-  fi
-done
-
-# Build swagger
-TARGET_PATH="$BUILD_PATH/swagger" ./build_swagger.sh
-python3 ./get-releases-json.py > "$BUILD_PATH/releases.json"
-python3 ./get-releases-json.py --latest > "$BUILD_PATH/latest.txt"

docs/build_swagger.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-set -eux
-
-SWAGGER_VERSION="4.15.5"
-TARGET_PATH=${TARGET_PATH-"swagger"}
-
-rm -rf "$TARGET_PATH"
-tmpdir="$(mktemp -d)"
-trap 'rm -rf "$tmpdir"' EXIT
-
-pushd "$tmpdir"
-curl -sSL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/v$SWAGGER_VERSION.tar.gz" -o swagger-ui.tgz
-tar -xzf swagger-ui.tgz
-popd
-mv "$tmpdir/"*/dist "$TARGET_PATH"
-cp schema.yml "$TARGET_PATH"
-
-sed -i "s#https://petstore.swagger.io/v2/swagger.json#schema.yml#g" "$TARGET_PATH/swagger-initializer.js"

docs/conf.py

@@ -1,58 +1,70 @@
-#!/usr/bin/env python3
-#
-# funkwhale documentation build configuration file, created by
-# sphinx-quickstart on Sun Jun 25 18:49:23 2017.
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
 # All configuration values have a default; values that are commented out
 # serve to show the default.
-
-import datetime
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
 #
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
 import os
-import sys
+from datetime import datetime
+from pathlib import Path
 
-sys.path.insert(0, os.path.abspath("../api"))
+from sphinx.application import Sphinx
-sys.path.insert(0, os.path.abspath("../api/config"))
+from sphinx.util import logging
+
+import funkwhale_api
+
+logger = logging.getLogger(__name__)
 
-import funkwhale_api  # NOQA
 
 FUNKWHALE_CONFIG = {
-    "FUNKWHALE_URL": "mypod.funkwhale",
+    "FUNKWHALE_URL": "https://pod.funkwhale",
-    "FUNKWHAL_PROTOCOL": "https",
+    "DATABASE_URL": "postgres://localhost:5432/funkwhale",
-    "DATABASE_URL": "postgres://localhost:5432/db",
     "AWS_ACCESS_KEY_ID": "my_access_key",
     "AWS_SECRET_ACCESS_KEY": "my_secret_key",
     "AWS_STORAGE_BUCKET_NAME": "my_bucket",
 }
-for key, value in FUNKWHALE_CONFIG.items():
+os.environ.update(**FUNKWHALE_CONFIG)
-    os.environ[key] = value
+
 # -- General configuration ------------------------------------------------
 
-# If your documentation needs a minimal Sphinx version, state it here.
+# General information about the project.
-#
+year = datetime.now().year
-# needs_sphinx = '1.0'
+project = "funkwhale"
+copyright = f"{year}, The Funkwhale Collective"
+author = "The Funkwhale Collective"
+version = funkwhale_api.__version__
+release = version
 
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
 extensions = [
-    "sphinx.ext.graphviz",
-    "sphinx.ext.autodoc",
-    "myst_parser",
-    "sphinx_design",
-    "sphinxcontrib.mermaid",
     "sphinx_copybutton",
+    "sphinx_design",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.graphviz",
+    "sphinxcontrib.mermaid",
+    "myst_parser",
 ]
+
+source_suffix = ".md"
+include_patterns = [
+    "_static/**",
+    "_templates/**",
+    "*_documentation/**",
+    "*.md",
+    "*.rst",
+    "logo.svg",
+]
+exclude_patterns = [
+    "_build",
+    "_scripts",
+    ".venv",
+    ".DS_Store",
+    "Thumbs.db",
+    "*.py",
+    "*.sh",
+]
+
+root_doc = "index"
+
+# autodoc
 autodoc_mock_imports = [
     "celery",
     "django_auth_ldap",
@@ -61,20 +73,13 @@ autodoc_mock_imports = [
     "rest_framework",
     "drf_spectacular",
 ]
+
+# sphinx
+pygments_style = "sphinx"
 add_module_names = False
-# Add any paths that contain templates here, relative to this directory.
+todo_include_todos = False
-templates_path = ["_templates"]
 
-# The suffix(es) of source filenames.
+# myst
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = ".md"
-
-# The root toctree document.
-root_doc = "index"
-
-# Enable colon fences
 myst_enable_extensions = [
     "colon_fence",
     "attrs_block",
@@ -82,54 +87,21 @@ myst_enable_extensions = [
     "fieldlist",
     "deflist",
 ]
-
-# Autogenerate anchors
-
 myst_heading_anchors = 3
 
-# General information about the project.
-year = datetime.datetime.now().year
-project = "funkwhale"
-copyright = f"{year}, The Funkwhale Collective"
-author = "The Funkwhale Collective"
 
-# The version info for the project you're documenting, acts as replacement for
+# internationalization
-# |version| and |release|, also used in various other places throughout the
+locale_dirs = ["locales/"]
-# built documents.
+gettext_compact = False
-# Read version from the API
-version = funkwhale_api.__version__
-release = version
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
 language = "en"
 
-# List of patterns, relative to source directory, that match files and
+# copybutton
-# directories to ignore when looking for source files.
+copybutton_exclude = ".linenos, .gp"
-# This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".venv"]
 
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = "sphinx"
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = False
 
 # -- Options for HTML output ----------------------------------------------
 
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
 html_theme = "sphinx_rtd_theme"
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
 html_context = {
     "display_gitlab": True,
     "gitlab_host": "dev.funkwhale.audio",
@@ -141,61 +113,37 @@ html_context = {
 }
 html_logo = "logo.svg"
 html_favicon = "../front/public/favicon.ico"
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
-html_css_files = [
+html_css_files = ["css/translation-hint.css"]
-    "css/translation-hint.css",
+html_js_files = ["js/translation-hint.js"]
-]
-html_js_files = [
-    "js/translation-hint.js",
-]
 
 # -- Options for HTMLHelp output ------------------------------------------
 
-# Output file base name for HTML help builder.
 htmlhelp_basename = "funkwhaledoc"
 
-
 # -- Options for LaTeX output ---------------------------------------------
 
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
 latex_documents = [
     (
         root_doc,
         "funkwhale.tex",
-        "funkwhale Documentation",
+        "Funkwhale Documentation",
         "The Funkwhale Collective",
         "manual",
     )
 ]
 
-
 # -- Options for manual page output ---------------------------------------
 
-# One entry per manual page. List of tuples
+man_pages = [
-# (source start file, name, description, authors, manual section).
+    (
-man_pages = [(root_doc, "funkwhale", "funkwhale Documentation", [author], 1)]
+        root_doc,
-
+        "funkwhale",
+        "Funkwhale Documentation",
+        [author],
+        1,
+    )
+]
 
 # -- Options for Texinfo output -------------------------------------------
 
@@ -206,7 +154,7 @@ texinfo_documents = [
     (
         root_doc,
         "funkwhale",
-        "funkwhale Documentation",
+        "Funkwhale Documentation",
         author,
         "funkwhale",
         "One line description of project.",
@@ -214,49 +162,35 @@ texinfo_documents = [
     )
 ]
 
-# -- Build legacy redirect files -------------------------------------------
+# -- Setup legacy redirects -----------------------------------------------
 
-# Define list of redirect files to be build in the Sphinx build process
+REDIRECT_TEMPLATE = """\
-
-redirect_list = []
-with open("redirects.txt") as fp:
-    data_list = [tuple(line.strip().split(",")) for line in fp]
-
-# Generate redirect template
-
-redirect_template = """\
 <html>
   <head>
-    <meta http-equiv="refresh" content="1; url={new}" />
+    <meta http-equiv="refresh" content="1; url={url}" />
     <script>
-      window.location.href = "{new}"
+      window.location.href = "{url}"
     </script>
   </head>
 </html>
 """
 
-# Tell Sphinx to copy the files
+redirects_file = Path("redirects.txt")
+redirects = [
+    tuple(line.strip().split(", "))
+    for line in redirects_file.read_text(encoding="utf-8").splitlines()
+]
 
 
-def copy_legacy_redirects(app, docname):
+def copy_legacy_redirects(app: Sphinx, docname):
     if app.builder.name == "html":
-        for html_src_path, new in data_list:
+        for src_path, dest_url in redirects:
-            page = redirect_template.format(new=new)
+            content = REDIRECT_TEMPLATE.format(url=dest_url)
-            target_path = app.outdir + "/" + html_src_path
+
-            if not os.path.exists(os.path.dirname(target_path)):
+            redirect_path = Path(app.outdir) / src_path
-                os.makedirs(os.path.dirname(target_path))
+            redirect_path.parent.mkdir(parents=True, exist_ok=True)
-            with open(target_path, "w") as f:
+            redirect_path.write_text(content, encoding="utf-8")
-                f.write(page)
 
 
 def setup(app):
     app.connect("build-finished", copy_legacy_redirects)
-
-
-# Internationalization settings
-locale_dirs = ["locales/"]
-gettext_compact = False
-
-# Don't copy prompts with copybutton
-
-copybutton_exclude = ".linenos, .gp"

docs/i18n_generate.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-poetry run make -e BUILDDIR=locales gettext
-
-for path in locales/*; do
-  lang="$(basename "$path")"
-  if [[ "$lang" != "gettext" ]]; then
-    poetry run sphinx-intl update -p locales/gettext -l "$lang"
-  fi
-done

docs/pyproject.toml

@@ -10,6 +10,7 @@ python = "^3.8"
 
 sphinx = "==5.3.0"
 sphinx_design = "==0.3.0"
+sphinx-copybutton = "==0.5.2"
 sphinx-intl = "2.0.1"
 sphinx-rtd-theme = "==1.1.1"
 sphinxcontrib-mermaid = "0.7.1"
@@ -17,10 +18,9 @@ myst-parser = "1.0.0"
 
 django = "==3.2.20"
 django-environ = "==0.10.0"
-funkwhale-api = {path = "../api"}
-sphinx-copybutton = "==0.5.2"
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
+livereload = "^2.6.3"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

docs/serve.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-from subprocess import call
-
-# initial make
-call(["python3", "-m", "sphinx", ".", "/tmp/_build"])
-from livereload import Server, shell  # noqa: E402
-
-server = Server()
-server.watch("..", shell("python3 -m sphinx . /tmp/_build"))
-server.serve(root="/tmp/_build/", liveport=35730, port=8001, host="0.0.0.0")