diff --git a/.github/workflows/translate.yml b/.github/workflows/translate.yml new file mode 100644 index 000000000..fc6b4d730 --- /dev/null +++ b/.github/workflows/translate.yml @@ -0,0 +1,77 @@ +name: Translate + +on: + workflow_dispatch: + inputs: + debug_enabled: + description: Run with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate) + required: false + default: "false" + command: + description: Command to run + type: choice + options: + - translate-page + - translate-lang + - update-outdated + - add-missing + - update-and-add + - remove-all-removable + language: + description: Language to translate to as a letter code (e.g. "es" for Spanish) + type: string + required: false + default: "" + en_path: + description: File path in English to translate (e.g. docs/en/docs/index.md) + type: string + required: false + default: "" + +env: + UV_SYSTEM_PYTHON: 1 + +jobs: + job: + if: github.repository_owner == 'fastapi' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.4.15" + enable-cache: true + cache-dependency-glob: | + requirements**.txt + pyproject.toml + - name: Install Dependencies + run: uv pip install -r requirements-github-actions.txt -r requirements-translations.txt + # Allow debugging with tmate + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }} + with: + limit-access-to-actor: true + env: + GITHUB_TOKEN: ${{ secrets.FASTAPI_TRANSLATIONS }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + - name: FastAPI Translate + run: | + python ./scripts/translate.py ${{ github.event.inputs.command }} + python ./scripts/translate.py make-pr + env: + GITHUB_TOKEN: ${{ secrets.FASTAPI_TRANSLATIONS }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LANGUAGE: ${{ github.event.inputs.language }} + EN_PATH: ${{ github.event.inputs.en_path }} diff --git a/docs/en/docs/release-notes.md b/docs/en/docs/release-notes.md index f3984fa9e..c7af5d6fa 100644 --- a/docs/en/docs/release-notes.md +++ b/docs/en/docs/release-notes.md @@ -23,6 +23,8 @@ hide: ### Internal +* ⚒️ Tweak translate script and CI. PR [#13939](https://github.com/fastapi/fastapi/pull/13939) by [@tiangolo](https://github.com/tiangolo). +* 👷 Add CI to translate with LLMs. PR [#13937](https://github.com/fastapi/fastapi/pull/13937) by [@tiangolo](https://github.com/tiangolo). * ⚒️ Update translate script, show and update outdated translations. PR [#13933](https://github.com/fastapi/fastapi/pull/13933) by [@tiangolo](https://github.com/tiangolo). * 🔨 Refactor translate script with extra feedback (prints). PR [#13932](https://github.com/fastapi/fastapi/pull/13932) by [@tiangolo](https://github.com/tiangolo). * 🔨 Update translations script to remove old (removed) files. PR [#13928](https://github.com/fastapi/fastapi/pull/13928) by [@tiangolo](https://github.com/tiangolo). diff --git a/scripts/translate.py b/scripts/translate.py index b78d7211b..e18449685 100644 --- a/scripts/translate.py +++ b/scripts/translate.py @@ -1,10 +1,13 @@ +import secrets +import subprocess from functools import lru_cache from pathlib import Path -from typing import Iterable +from typing import Annotated, Iterable import git import typer import yaml +from github import Github from pydantic_ai import Agent from rich import print @@ -89,27 +92,31 @@ def generate_en_path(*, lang: str, path: Path) -> Path: @app.command() -def translate_page(*, lang: str, path: Path) -> None: +def translate_page( + *, + language: Annotated[str, typer.Option(envvar="LANGUAGE")], + en_path: Annotated[Path, typer.Option(envvar="EN_PATH")], +) -> None: langs = get_langs() - language = langs[lang] - lang_path = Path(f"docs/{lang}") + language_name = langs[language] + lang_path = Path(f"docs/{language}") lang_path.mkdir(exist_ok=True) lang_prompt_path = lang_path / "llm-prompt.md" assert lang_prompt_path.exists(), f"Prompt file not found: {lang_prompt_path}" lang_prompt_content = lang_prompt_path.read_text() en_docs_path = Path("docs/en/docs") - assert str(path).startswith(str(en_docs_path)), ( + assert str(en_path).startswith(str(en_docs_path)), ( f"Path must be inside {en_docs_path}" ) - out_path = generate_lang_path(lang=lang, path=path) + out_path = generate_lang_path(lang=language, path=en_path) out_path.parent.mkdir(parents=True, exist_ok=True) - original_content = path.read_text() + original_content = en_path.read_text() old_translation: str | None = None if out_path.exists(): print(f"Found existing translation: {out_path}") old_translation = out_path.read_text() - print(f"Translating {path} to {lang} ({language})") + print(f"Translating {en_path} to {language} ({language_name})") agent = Agent("openai:gpt-4o") prompt_segments = [ @@ -128,7 +135,7 @@ def translate_page(*, lang: str, path: Path) -> None: ) prompt_segments.extend( [ - f"Translate to {language} ({lang}).", + f"Translate to {language} ({language_name}).", "Original content:", f"%%%\n{original_content}%%%", ] @@ -173,7 +180,7 @@ def iter_en_paths_to_translate() -> Iterable[Path]: @app.command() -def translate_all(lang: str) -> None: +def translate_lang(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None: paths_to_process = list(iter_en_paths_to_translate()) print("Original paths:") for p in paths_to_process: @@ -182,7 +189,7 @@ def translate_all(lang: str) -> None: missing_paths: list[Path] = [] skipped_paths: list[Path] = [] for p in paths_to_process: - lang_path = generate_lang_path(lang=lang, path=p) + lang_path = generate_lang_path(lang=language, path=p) if lang_path.exists(): skipped_paths.append(p) continue @@ -197,16 +204,16 @@ def translate_all(lang: str) -> None: print(f"Total paths to process: {len(missing_paths)}") for p in missing_paths: print(f"Translating: {p}") - translate_page(lang="es", path=p) + translate_page(language="es", en_path=p) print(f"Done translating: {p}") @app.command() -def list_removable(lang: str) -> list[Path]: +def list_removable(language: str) -> list[Path]: removable_paths: list[Path] = [] - lang_paths = Path(f"docs/{lang}").rglob("*.md") + lang_paths = Path(f"docs/{language}").rglob("*.md") for path in lang_paths: - en_path = generate_en_path(lang=lang, path=path) + en_path = generate_en_path(lang=language, path=path) if not en_path.exists(): removable_paths.append(path) print(removable_paths) @@ -227,8 +234,8 @@ def list_all_removable() -> list[Path]: @app.command() -def remove_removable(lang: str) -> None: - removable_paths = list_removable(lang) +def remove_removable(language: str) -> None: + removable_paths = list_removable(language) for path in removable_paths: path.unlink() print(f"Removed: {path}") @@ -245,16 +252,27 @@ def remove_all_removable() -> None: @app.command() -def list_outdated(lang: str) -> list[Path]: +def list_missing(language: str) -> list[Path]: + missing_paths: list[Path] = [] + en_lang_paths = list(iter_en_paths_to_translate()) + for path in en_lang_paths: + lang_path = generate_lang_path(lang=language, path=path) + if not lang_path.exists(): + missing_paths.append(path) + print(missing_paths) + return missing_paths + + +@app.command() +def list_outdated(language: str) -> list[Path]: dir_path = Path(__file__).absolute().parent.parent repo = git.Repo(dir_path) outdated_paths: list[Path] = [] en_lang_paths = list(iter_en_paths_to_translate()) for path in en_lang_paths: - lang_path = generate_lang_path(lang=lang, path=path) + lang_path = generate_lang_path(lang=language, path=path) if not lang_path.exists(): - outdated_paths.append(path) continue en_commit_datetime = list(repo.iter_commits(paths=path, max_count=1))[ 0 @@ -269,14 +287,75 @@ def list_outdated(lang: str) -> list[Path]: @app.command() -def update_outdated(lang: str) -> None: - outdated_paths = list_outdated(lang) +def update_outdated(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None: + outdated_paths = list_outdated(language) for path in outdated_paths: - print(f"Updating lang: {lang} path: {path}") - translate_page(lang=lang, path=path) + print(f"Updating lang: {language} path: {path}") + translate_page(language=language, en_path=path) print(f"Done updating: {path}") print("Done updating all outdated paths") +@app.command() +def add_missing(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None: + missing_paths = list_missing(language) + for path in missing_paths: + print(f"Adding lang: {language} path: {path}") + translate_page(language=language, en_path=path) + print(f"Done adding: {path}") + print("Done adding all missing paths") + + +@app.command() +def update_and_add(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None: + print(f"Updating outdated translations for {language}") + update_outdated(language=language) + print(f"Adding missing translations for {language}") + add_missing(language=language) + print(f"Done updating and adding for {language}") + + +@app.command() +def make_pr( + *, + language: Annotated[str | None, typer.Option(envvar="LANGUAGE")] = None, + github_token: Annotated[str, typer.Option(envvar="GITHUB_TOKEN")], + github_repository: Annotated[str, typer.Option(envvar="GITHUB_REPOSITORY")], +) -> None: + print("Setting up GitHub Actions git user") + repo = git.Repo(Path(__file__).absolute().parent.parent) + if not repo.is_dirty(untracked_files=True): + print("Repository is clean, no changes to commit") + return + subprocess.run(["git", "config", "user.name", "github-actions"], check=True) + subprocess.run( + ["git", "config", "user.email", "github-actions@github.com"], check=True + ) + branch_name = "translate" + if language: + branch_name += f"-{language}" + branch_name += f"-{secrets.token_hex(4)}" + print(f"Creating a new branch {branch_name}") + subprocess.run(["git", "checkout", "-b", branch_name], check=True) + print("Adding updated files") + git_path = Path("docs") + subprocess.run(["git", "add", str(git_path)], check=True) + print("Committing updated file") + message = "🌐 Update translations" + if language: + message += f" for {language}" + subprocess.run(["git", "commit", "-m", message], check=True) + print("Pushing branch") + subprocess.run(["git", "push", "origin", branch_name], check=True) + print("Creating PR") + g = Github(github_token) + gh_repo = g.get_repo(github_repository) + pr = gh_repo.create_pull( + title=message, body=message, base="master", head=branch_name + ) + print(f"Created PR: {pr.number}") + print("Finished") + + if __name__ == "__main__": app()