👷 Refactor Docs CI, run in multiple workers with a dynamic matrix to optimize speed (#9732)

.github/workflows/build-docs.yml

@@ -28,10 +28,40 @@ jobs:
             - docs/**
             - docs_src/**
             - requirements-docs.txt
+  langs:
+    needs:
+      - changes
+    runs-on: ubuntu-latest
+    outputs:
+      langs: ${{ steps.show-langs.outputs.langs }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - uses: actions/cache@v3
+        id: cache
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-python-docs-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml', 'requirements-docs.txt') }}-v03
+      - name: Install docs extras
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: pip install -r requirements-docs.txt
+      - name: Export Language Codes
+        id: show-langs
+        run: |
+          echo "langs=$(python ./scripts/docs.py langs-json)" >> $GITHUB_OUTPUT
+
   build-docs:
-    needs: changes
+    needs:
+      - changes
+      - langs
     if: ${{ needs.changes.outputs.docs == 'true' }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        lang: ${{ fromJson(needs.langs.outputs.langs) }}
     steps:
       - name: Dump GitHub context
         env:
@@ -53,21 +83,24 @@ jobs:
       - name: Install Material for MkDocs Insiders
         if: ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false ) && steps.cache.outputs.cache-hit != 'true'
         run: pip install git+https://${{ secrets.ACTIONS_TOKEN }}@github.com/squidfunk/mkdocs-material-insiders.git
+      - name: Update Languages
+        run: python ./scripts/docs.py update-languages
       - name: Build Docs
-        run: python ./scripts/docs.py build-all
-      - name: Zip docs
-        run: bash ./scripts/zip-docs.sh
+        run: python ./scripts/docs.py build-lang ${{ matrix.lang }}
       - uses: actions/upload-artifact@v3
         with:
-          name: docs-zip
-          path: ./site/docs.zip
-      - name: Deploy to Netlify
-        uses: nwtgck/actions-netlify@v2.0.0
+          name: docs-site
+          path: ./site/**
+
+  # https://github.com/marketplace/actions/alls-green#why
+  docs-all-green:  # This job does nothing and is only used for the branch protection
+    if: always()
+    needs:
+      - build-docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Decide whether the needed jobs succeeded or failed
+        uses: re-actors/alls-green@release/v1
         with:
-          publish-dir: './site'
-          production-branch: master
-          github-token: ${{ secrets.FASTAPI_BUILD_DOCS_NETLIFY }}
-          enable-commit-comment: false
-        env:
-          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
-          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+          jobs: ${{ toJSON(needs) }}
+          allowed-skips: build-docs

.github/workflows/preview-docs.yml → .github/workflows/deploy-docs.yml

@@ -1,4 +1,4 @@
-name: Preview Docs
+name: Deploy Docs
 on:
   workflow_run:
     workflows:
@@ -7,9 +7,13 @@ on:
       - completed
 
 jobs:
-  preview-docs:
+  deploy-docs:
     runs-on: ubuntu-latest
     steps:
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v3
       - name: Clean site
         run: |
@@ -23,21 +27,15 @@ jobs:
           github_token: ${{ secrets.FASTAPI_PREVIEW_DOCS_DOWNLOAD_ARTIFACTS }}
           workflow: build-docs.yml
           run_id: ${{ github.event.workflow_run.id }}
-          name: docs-zip
+          name: docs-site
           path: ./site/
-      - name: Unzip docs
-        if: steps.download.outputs.found_artifact == 'true'
-        run: |
-          cd ./site
-          unzip docs.zip
-          rm -f docs.zip
       - name: Deploy to Netlify
         if: steps.download.outputs.found_artifact == 'true'
         id: netlify
         uses: nwtgck/actions-netlify@v2.0.0
         with:
           publish-dir: './site'
-          production-deploy: false
+          production-deploy: ${{ github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.head_branch == 'master' }}
           github-token: ${{ secrets.FASTAPI_PREVIEW_DOCS_NETLIFY }}
           enable-commit-comment: false
         env:

.gitignore

@@ -16,6 +16,7 @@ Pipfile.lock
 env3.*
 env
 docs_build
+site_build
 venv
 docs.zip
 archive.zip

scripts/docs.py

@@ -1,3 +1,4 @@
+import json
 import os
 import re
 import shutil
@@ -133,12 +134,15 @@ def build_lang(
     build_lang_path = build_dir_path / lang
     en_lang_path = Path("docs/en")
     site_path = Path("site").absolute()
+    build_site_path = Path("site_build").absolute()
+    build_site_dist_path = build_site_path / lang
     if lang == "en":
         dist_path = site_path
     else:
         dist_path: Path = site_path / lang
     shutil.rmtree(build_lang_path, ignore_errors=True)
     shutil.copytree(lang_path, build_lang_path)
+    if not lang == "en":
         shutil.copytree(en_docs_path / "data", build_lang_path / "data")
         overrides_src = en_docs_path / "overrides"
         overrides_dest = build_lang_path / "overrides"
@@ -147,7 +151,9 @@ def build_lang(
             if not dest_path.exists():
                 shutil.copy(path, dest_path)
         en_config_path: Path = en_lang_path / mkdocs_name
-    en_config: dict = mkdocs.utils.yaml_load(en_config_path.read_text(encoding="utf-8"))
+        en_config: dict = mkdocs.utils.yaml_load(
+            en_config_path.read_text(encoding="utf-8")
+        )
         nav = en_config["nav"]
         lang_config_path: Path = lang_path / mkdocs_name
         lang_config: dict = mkdocs.utils.yaml_load(
@@ -201,7 +207,10 @@ def build_lang(
         )
     current_dir = os.getcwd()
     os.chdir(build_lang_path)
-    subprocess.run(["mkdocs", "build", "--site-dir", dist_path], check=True)
+    shutil.rmtree(build_site_dist_path, ignore_errors=True)
+    shutil.rmtree(dist_path, ignore_errors=True)
+    subprocess.run(["mkdocs", "build", "--site-dir", build_site_dist_path], check=True)
+    shutil.copytree(build_site_dist_path, dist_path, dirs_exist_ok=True)
     os.chdir(current_dir)
     typer.secho(f"Successfully built docs for: {lang}", color=typer.colors.GREEN)
 
@@ -271,18 +280,8 @@ def build_all():
     Build mkdocs site for en, and then build each language inside, end result is located
     at directory ./site/ with each language inside.
     """
-    site_path = Path("site").absolute()
     update_languages(lang=None)
-    current_dir = os.getcwd()
-    os.chdir(en_docs_path)
-    typer.echo("Building docs for: en")
-    subprocess.run(["mkdocs", "build", "--site-dir", site_path], check=True)
-    os.chdir(current_dir)
-    langs = []
-    for lang in get_lang_paths():
-        if lang == en_docs_path or not lang.is_dir():
-            continue
-        langs.append(lang.name)
+    langs = [lang.name for lang in get_lang_paths() if lang.is_dir()]
     cpu_count = os.cpu_count() or 1
     process_pool_size = cpu_count * 4
     typer.echo(f"Using process pool size: {process_pool_size}")
@@ -397,6 +396,15 @@ def update_config(lang: str):
     )
 
 
+@app.command()
+def langs_json():
+    langs = []
+    for lang_path in get_lang_paths():
+        if lang_path.is_dir():
+            langs.append(lang_path.name)
+    print(json.dumps(langs))
+
+
 def get_key_section(
     *, key_to_section: Dict[Tuple[str, ...], list], key: Tuple[str, ...]
 ) -> list:

scripts/zip-docs.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-set -x
-set -e
-
-cd ./site
-
-if [ -f docs.zip ]; then
-    rm -rf docs.zip
-fi
-zip -r docs.zip ./