Merge branch 'master' into ru-prompt

.github/workflows/translate.yml

@@ -0,0 +1,77 @@
+name: Translate
+
+on:
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: Run with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)
+        required: false
+        default: "false"
+      command:
+        description: Command to run
+        type: choice
+        options:
+          - translate-page
+          - translate-lang
+          - update-outdated
+          - add-missing
+          - update-and-add
+          - remove-all-removable
+      language:
+        description: Language to translate to as a letter code (e.g. "es" for Spanish)
+        type: string
+        required: false
+        default: ""
+      en_path:
+        description: File path in English to translate (e.g. docs/en/docs/index.md)
+        type: string
+        required: false
+        default: ""
+
+env:
+  UV_SYSTEM_PYTHON: 1
+
+jobs:
+  job:
+    if: github.repository_owner == 'fastapi'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.4.15"
+          enable-cache: true
+          cache-dependency-glob: |
+            requirements**.txt
+            pyproject.toml
+      - name: Install Dependencies
+        run: uv pip install -r requirements-github-actions.txt -r requirements-translations.txt
+      # Allow debugging with tmate
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }}
+        with:
+          limit-access-to-actor: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.FASTAPI_TRANSLATIONS }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      - name: FastAPI Translate
+        run: |
+          python ./scripts/translate.py ${{ github.event.inputs.command }}
+          python ./scripts/translate.py make-pr
+        env:
+          GITHUB_TOKEN: ${{ secrets.FASTAPI_TRANSLATIONS }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          LANGUAGE: ${{ github.event.inputs.language }}
+          EN_PATH: ${{ github.event.inputs.en_path }}

docs/en/docs/release-notes.md

@@ -23,6 +23,8 @@ hide:
 
 ### Internal
 
+* ⚒️ Tweak translate script and CI. PR [#13939](https://github.com/fastapi/fastapi/pull/13939) by [@tiangolo](https://github.com/tiangolo).
+* 👷 Add CI to translate with LLMs. PR [#13937](https://github.com/fastapi/fastapi/pull/13937) by [@tiangolo](https://github.com/tiangolo).
 * ⚒️ Update translate script, show and update outdated translations. PR [#13933](https://github.com/fastapi/fastapi/pull/13933) by [@tiangolo](https://github.com/tiangolo).
 * 🔨 Refactor translate script with extra feedback (prints). PR [#13932](https://github.com/fastapi/fastapi/pull/13932) by [@tiangolo](https://github.com/tiangolo).
 * 🔨 Update translations script to remove old (removed) files. PR [#13928](https://github.com/fastapi/fastapi/pull/13928) by [@tiangolo](https://github.com/tiangolo).

scripts/translate.py

@@ -1,10 +1,13 @@
+import secrets
+import subprocess
 from functools import lru_cache
 from pathlib import Path
-from typing import Iterable
+from typing import Annotated, Iterable
 
 import git
 import typer
 import yaml
+from github import Github
 from pydantic_ai import Agent
 from rich import print
 
@@ -89,27 +92,31 @@ def generate_en_path(*, lang: str, path: Path) -> Path:
 
 
 @app.command()
-def translate_page(*, lang: str, path: Path) -> None:
+def translate_page(
+    *,
+    language: Annotated[str, typer.Option(envvar="LANGUAGE")],
+    en_path: Annotated[Path, typer.Option(envvar="EN_PATH")],
+) -> None:
     langs = get_langs()
-    language = langs[lang]
+    language_name = langs[language]
-    lang_path = Path(f"docs/{lang}")
+    lang_path = Path(f"docs/{language}")
     lang_path.mkdir(exist_ok=True)
     lang_prompt_path = lang_path / "llm-prompt.md"
     assert lang_prompt_path.exists(), f"Prompt file not found: {lang_prompt_path}"
     lang_prompt_content = lang_prompt_path.read_text()
 
     en_docs_path = Path("docs/en/docs")
-    assert str(path).startswith(str(en_docs_path)), (
+    assert str(en_path).startswith(str(en_docs_path)), (
         f"Path must be inside {en_docs_path}"
     )
-    out_path = generate_lang_path(lang=lang, path=path)
+    out_path = generate_lang_path(lang=language, path=en_path)
     out_path.parent.mkdir(parents=True, exist_ok=True)
-    original_content = path.read_text()
+    original_content = en_path.read_text()
     old_translation: str | None = None
     if out_path.exists():
         print(f"Found existing translation: {out_path}")
         old_translation = out_path.read_text()
-    print(f"Translating {path} to {lang} ({language})")
+    print(f"Translating {en_path} to {language} ({language_name})")
     agent = Agent("openai:gpt-4o")
 
     prompt_segments = [
@@ -128,7 +135,7 @@ def translate_page(*, lang: str, path: Path) -> None:
         )
     prompt_segments.extend(
         [
-            f"Translate to {language} ({lang}).",
+            f"Translate to {language} ({language_name}).",
             "Original content:",
             f"%%%\n{original_content}%%%",
         ]
@@ -173,7 +180,7 @@ def iter_en_paths_to_translate() -> Iterable[Path]:
 
 
 @app.command()
-def translate_all(lang: str) -> None:
+def translate_lang(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None:
     paths_to_process = list(iter_en_paths_to_translate())
     print("Original paths:")
     for p in paths_to_process:
@@ -182,7 +189,7 @@ def translate_all(lang: str) -> None:
     missing_paths: list[Path] = []
     skipped_paths: list[Path] = []
     for p in paths_to_process:
-        lang_path = generate_lang_path(lang=lang, path=p)
+        lang_path = generate_lang_path(lang=language, path=p)
         if lang_path.exists():
             skipped_paths.append(p)
             continue
@@ -197,16 +204,16 @@ def translate_all(lang: str) -> None:
     print(f"Total paths to process: {len(missing_paths)}")
     for p in missing_paths:
         print(f"Translating: {p}")
-        translate_page(lang="es", path=p)
+        translate_page(language="es", en_path=p)
         print(f"Done translating: {p}")
 
 
 @app.command()
-def list_removable(lang: str) -> list[Path]:
+def list_removable(language: str) -> list[Path]:
     removable_paths: list[Path] = []
-    lang_paths = Path(f"docs/{lang}").rglob("*.md")
+    lang_paths = Path(f"docs/{language}").rglob("*.md")
     for path in lang_paths:
-        en_path = generate_en_path(lang=lang, path=path)
+        en_path = generate_en_path(lang=language, path=path)
         if not en_path.exists():
             removable_paths.append(path)
     print(removable_paths)
@@ -227,8 +234,8 @@ def list_all_removable() -> list[Path]:
 
 
 @app.command()
-def remove_removable(lang: str) -> None:
+def remove_removable(language: str) -> None:
-    removable_paths = list_removable(lang)
+    removable_paths = list_removable(language)
     for path in removable_paths:
         path.unlink()
         print(f"Removed: {path}")
@@ -245,16 +252,27 @@ def remove_all_removable() -> None:
 
 
 @app.command()
-def list_outdated(lang: str) -> list[Path]:
+def list_missing(language: str) -> list[Path]:
+    missing_paths: list[Path] = []
+    en_lang_paths = list(iter_en_paths_to_translate())
+    for path in en_lang_paths:
+        lang_path = generate_lang_path(lang=language, path=path)
+        if not lang_path.exists():
+            missing_paths.append(path)
+    print(missing_paths)
+    return missing_paths
+
+
+@app.command()
+def list_outdated(language: str) -> list[Path]:
     dir_path = Path(__file__).absolute().parent.parent
     repo = git.Repo(dir_path)
 
     outdated_paths: list[Path] = []
     en_lang_paths = list(iter_en_paths_to_translate())
     for path in en_lang_paths:
-        lang_path = generate_lang_path(lang=lang, path=path)
+        lang_path = generate_lang_path(lang=language, path=path)
         if not lang_path.exists():
-            outdated_paths.append(path)
             continue
         en_commit_datetime = list(repo.iter_commits(paths=path, max_count=1))[
             0
@@ -269,14 +287,75 @@ def list_outdated(lang: str) -> list[Path]:
 
 
 @app.command()
-def update_outdated(lang: str) -> None:
+def update_outdated(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None:
-    outdated_paths = list_outdated(lang)
+    outdated_paths = list_outdated(language)
     for path in outdated_paths:
-        print(f"Updating lang: {lang} path: {path}")
+        print(f"Updating lang: {language} path: {path}")
-        translate_page(lang=lang, path=path)
+        translate_page(language=language, en_path=path)
         print(f"Done updating: {path}")
     print("Done updating all outdated paths")
 
 
+@app.command()
+def add_missing(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None:
+    missing_paths = list_missing(language)
+    for path in missing_paths:
+        print(f"Adding lang: {language} path: {path}")
+        translate_page(language=language, en_path=path)
+        print(f"Done adding: {path}")
+    print("Done adding all missing paths")
+
+
+@app.command()
+def update_and_add(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None:
+    print(f"Updating outdated translations for {language}")
+    update_outdated(language=language)
+    print(f"Adding missing translations for {language}")
+    add_missing(language=language)
+    print(f"Done updating and adding for {language}")
+
+
+@app.command()
+def make_pr(
+    *,
+    language: Annotated[str | None, typer.Option(envvar="LANGUAGE")] = None,
+    github_token: Annotated[str, typer.Option(envvar="GITHUB_TOKEN")],
+    github_repository: Annotated[str, typer.Option(envvar="GITHUB_REPOSITORY")],
+) -> None:
+    print("Setting up GitHub Actions git user")
+    repo = git.Repo(Path(__file__).absolute().parent.parent)
+    if not repo.is_dirty(untracked_files=True):
+        print("Repository is clean, no changes to commit")
+        return
+    subprocess.run(["git", "config", "user.name", "github-actions"], check=True)
+    subprocess.run(
+        ["git", "config", "user.email", "github-actions@github.com"], check=True
+    )
+    branch_name = "translate"
+    if language:
+        branch_name += f"-{language}"
+    branch_name += f"-{secrets.token_hex(4)}"
+    print(f"Creating a new branch {branch_name}")
+    subprocess.run(["git", "checkout", "-b", branch_name], check=True)
+    print("Adding updated files")
+    git_path = Path("docs")
+    subprocess.run(["git", "add", str(git_path)], check=True)
+    print("Committing updated file")
+    message = "🌐 Update translations"
+    if language:
+        message += f" for {language}"
+    subprocess.run(["git", "commit", "-m", message], check=True)
+    print("Pushing branch")
+    subprocess.run(["git", "push", "origin", branch_name], check=True)
+    print("Creating PR")
+    g = Github(github_token)
+    gh_repo = g.get_repo(github_repository)
+    pr = gh_repo.create_pull(
+        title=message, body=message, base="master", head=branch_name
+    )
+    print(f"Created PR: {pr.number}")
+    print("Finished")
+
+
 if __name__ == "__main__":
     app()