mirror
/
tiangolo.fastapi
mirror of https://github.com/tiangolo/fastapi
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

⚒️ Tweak translate script and CI (#13939)

pull/13940/head
Sebastián Ramírez 11 months ago
committed by GitHub
parent
c74e7d1ce4
commit
9aab3d9e22
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
2 changed files with 44 additions and 39 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      .github/workflows/translate.yml
  2. 79
      scripts/translate.py

4
.github/workflows/translate.yml
View File

@ -17,7 +17,7 @@ on:
- add-missing - add-missing
- update-and-add - update-and-add
- remove-all-removable - remove-all-removable
lang: language:
description: Language to translate to as a letter code (e.g. "es" for Spanish) description: Language to translate to as a letter code (e.g. "es" for Spanish)
type: string type: string
required: false required: false
@ -73,5 +73,5 @@ jobs:
env: env:
GITHUB_TOKEN: ${{ secrets.FASTAPI_TRANSLATIONS }} GITHUB_TOKEN: ${{ secrets.FASTAPI_TRANSLATIONS }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
LANG: ${{ github.event.inputs.lang }} LANGUAGE: ${{ github.event.inputs.language }}
EN_PATH: ${{ github.event.inputs.en_path }} EN_PATH: ${{ github.event.inputs.en_path }}

79
scripts/translate.py
View File

@ -94,12 +94,12 @@ def generate_en_path(*, lang: str, path: Path) -> Path:
@app.command() @app.command()
def translate_page( def translate_page(
*, *,
lang: Annotated[str, typer.Option(envvar="LANG")], language: Annotated[str, typer.Option(envvar="LANGUAGE")],
en_path: Annotated[Path, typer.Option(envvar="EN_PATH")], en_path: Annotated[Path, typer.Option(envvar="EN_PATH")],
) -> None: ) -> None:
langs = get_langs() langs = get_langs()
language = langs[lang] language_name = langs[language]
lang_path = Path(f"docs/{lang}") lang_path = Path(f"docs/{language}")
lang_path.mkdir(exist_ok=True) lang_path.mkdir(exist_ok=True)
lang_prompt_path = lang_path / "llm-prompt.md" lang_prompt_path = lang_path / "llm-prompt.md"
assert lang_prompt_path.exists(), f"Prompt file not found: {lang_prompt_path}" assert lang_prompt_path.exists(), f"Prompt file not found: {lang_prompt_path}"
@ -109,14 +109,14 @@ def translate_page(
assert str(en_path).startswith(str(en_docs_path)), ( assert str(en_path).startswith(str(en_docs_path)), (
f"Path must be inside {en_docs_path}" f"Path must be inside {en_docs_path}"
) )
out_path = generate_lang_path(lang=lang, path=en_path) out_path = generate_lang_path(lang=language, path=en_path)
out_path.parent.mkdir(parents=True, exist_ok=True) out_path.parent.mkdir(parents=True, exist_ok=True)
original_content = en_path.read_text() original_content = en_path.read_text()
old_translation: str | None = None old_translation: str | None = None
if out_path.exists(): if out_path.exists():
print(f"Found existing translation: {out_path}") print(f"Found existing translation: {out_path}")
old_translation = out_path.read_text() old_translation = out_path.read_text()
print(f"Translating {en_path} to {lang} ({language})") print(f"Translating {en_path} to {language} ({language_name})")
agent = Agent("openai:gpt-4o") agent = Agent("openai:gpt-4o")
prompt_segments = [ prompt_segments = [
@ -135,7 +135,7 @@ def translate_page(
) )
prompt_segments.extend( prompt_segments.extend(
[ [
f"Translate to {language} ({lang}).", f"Translate to {language} ({language_name}).",
"Original content:", "Original content:",
f"%%%\n{original_content}%%%", f"%%%\n{original_content}%%%",
] ]
@ -180,7 +180,7 @@ def iter_en_paths_to_translate() -> Iterable[Path]:
@app.command() @app.command()
def translate_lang(lang: Annotated[str, typer.Option(envvar="LANG")]) -> None: def translate_lang(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None:
paths_to_process = list(iter_en_paths_to_translate()) paths_to_process = list(iter_en_paths_to_translate())
print("Original paths:") print("Original paths:")
for p in paths_to_process: for p in paths_to_process:
@ -189,7 +189,7 @@ def translate_lang(lang: Annotated[str, typer.Option(envvar="LANG")]) -> None:
missing_paths: list[Path] = [] missing_paths: list[Path] = []
skipped_paths: list[Path] = [] skipped_paths: list[Path] = []
for p in paths_to_process: for p in paths_to_process:
lang_path = generate_lang_path(lang=lang, path=p) lang_path = generate_lang_path(lang=language, path=p)
if lang_path.exists(): if lang_path.exists():
skipped_paths.append(p) skipped_paths.append(p)
continue continue
@ -204,16 +204,16 @@ def translate_lang(lang: Annotated[str, typer.Option(envvar="LANG")]) -> None:
print(f"Total paths to process: {len(missing_paths)}") print(f"Total paths to process: {len(missing_paths)}")
for p in missing_paths: for p in missing_paths:
print(f"Translating: {p}") print(f"Translating: {p}")
translate_page(lang="es", en_path=p) translate_page(language="es", en_path=p)
print(f"Done translating: {p}") print(f"Done translating: {p}")
@app.command() @app.command()
def list_removable(lang: str) -> list[Path]: def list_removable(language: str) -> list[Path]:
removable_paths: list[Path] = [] removable_paths: list[Path] = []
lang_paths = Path(f"docs/{lang}").rglob("*.md") lang_paths = Path(f"docs/{language}").rglob("*.md")
for path in lang_paths: for path in lang_paths:
en_path = generate_en_path(lang=lang, path=path) en_path = generate_en_path(lang=language, path=path)
if not en_path.exists(): if not en_path.exists():
removable_paths.append(path) removable_paths.append(path)
print(removable_paths) print(removable_paths)
@ -234,8 +234,8 @@ def list_all_removable() -> list[Path]:
@app.command() @app.command()
def remove_removable(lang: str) -> None: def remove_removable(language: str) -> None:
removable_paths = list_removable(lang) removable_paths = list_removable(language)
for path in removable_paths: for path in removable_paths:
path.unlink() path.unlink()
print(f"Removed: {path}") print(f"Removed: {path}")
@ -252,11 +252,11 @@ def remove_all_removable() -> None:
@app.command() @app.command()
def list_missing(lang: str) -> list[Path]: def list_missing(language: str) -> list[Path]:
missing_paths: list[Path] = [] missing_paths: list[Path] = []
en_lang_paths = list(iter_en_paths_to_translate()) en_lang_paths = list(iter_en_paths_to_translate())
for path in en_lang_paths: for path in en_lang_paths:
lang_path = generate_lang_path(lang=lang, path=path) lang_path = generate_lang_path(lang=language, path=path)
if not lang_path.exists(): if not lang_path.exists():
missing_paths.append(path) missing_paths.append(path)
print(missing_paths) print(missing_paths)
@ -264,14 +264,14 @@ def list_missing(lang: str) -> list[Path]:
@app.command() @app.command()
def list_outdated(lang: str) -> list[Path]: def list_outdated(language: str) -> list[Path]:
dir_path = Path(__file__).absolute().parent.parent dir_path = Path(__file__).absolute().parent.parent
repo = git.Repo(dir_path) repo = git.Repo(dir_path)
outdated_paths: list[Path] = [] outdated_paths: list[Path] = []
en_lang_paths = list(iter_en_paths_to_translate()) en_lang_paths = list(iter_en_paths_to_translate())
for path in en_lang_paths: for path in en_lang_paths:
lang_path = generate_lang_path(lang=lang, path=path) lang_path = generate_lang_path(lang=language, path=path)
if not lang_path.exists(): if not lang_path.exists():
continue continue
en_commit_datetime = list(repo.iter_commits(paths=path, max_count=1))[ en_commit_datetime = list(repo.iter_commits(paths=path, max_count=1))[
@ -287,38 +287,38 @@ def list_outdated(lang: str) -> list[Path]:
@app.command() @app.command()
def update_outdated(lang: Annotated[str, typer.Option(envvar="LANG")]) -> None: def update_outdated(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None:
outdated_paths = list_outdated(lang) outdated_paths = list_outdated(language)
for path in outdated_paths: for path in outdated_paths:
print(f"Updating lang: {lang} path: {path}") print(f"Updating lang: {language} path: {path}")
translate_page(lang=lang, en_path=path) translate_page(language=language, en_path=path)
print(f"Done updating: {path}") print(f"Done updating: {path}")
print("Done updating all outdated paths") print("Done updating all outdated paths")
@app.command() @app.command()
def add_missing(lang: Annotated[str, typer.Option(envvar="LANG")]) -> None: def add_missing(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None:
missing_paths = list_missing(lang) missing_paths = list_missing(language)
for path in missing_paths: for path in missing_paths:
print(f"Adding lang: {lang} path: {path}") print(f"Adding lang: {language} path: {path}")
translate_page(lang=lang, en_path=path) translate_page(language=language, en_path=path)
print(f"Done adding: {path}") print(f"Done adding: {path}")
print("Done adding all missing paths") print("Done adding all missing paths")
@app.command() @app.command()
def update_and_add(lang: Annotated[str, typer.Option(envvar="LANG")]) -> None: def update_and_add(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) -> None:
print(f"Updating outdated translations for {lang}") print(f"Updating outdated translations for {language}")
update_outdated(lang=lang) update_outdated(language=language)
print(f"Adding missing translations for {lang}") print(f"Adding missing translations for {language}")
add_missing(lang=lang) add_missing(language=language)
print(f"Done updating and adding for {lang}") print(f"Done updating and adding for {language}")
@app.command() @app.command()
def make_pr( def make_pr(
*, *,
lang: Annotated[str, typer.Option(envvar="LANG")], language: Annotated[str | None, typer.Option(envvar="LANGUAGE")] = None,
github_token: Annotated[str, typer.Option(envvar="GITHUB_TOKEN")], github_token: Annotated[str, typer.Option(envvar="GITHUB_TOKEN")],
github_repository: Annotated[str, typer.Option(envvar="GITHUB_REPOSITORY")], github_repository: Annotated[str, typer.Option(envvar="GITHUB_REPOSITORY")],
) -> None: ) -> None:
@ -331,14 +331,19 @@ def make_pr(
subprocess.run( subprocess.run(
["git", "config", "user.email", "[email protected]"], check=True ["git", "config", "user.email", "[email protected]"], check=True
) )
branch_name = f"translate-{lang}-{secrets.token_hex(4)}" branch_name = "translate"
if language:
branch_name += f"-{language}"
branch_name += f"-{secrets.token_hex(4)}"
print(f"Creating a new branch {branch_name}") print(f"Creating a new branch {branch_name}")
subprocess.run(["git", "checkout", "-b", branch_name], check=True) subprocess.run(["git", "checkout", "-b", branch_name], check=True)
print("Adding updated files") print("Adding updated files")
lang_path = Path(f"docs/{lang}") git_path = Path("docs")
subprocess.run(["git", "add", str(lang_path)], check=True) subprocess.run(["git", "add", str(git_path)], check=True)
print("Committing updated file") print("Committing updated file")
message = f"🌐 Update translations - {lang}" message = "🌐 Update translations"
if language:
message += f" for {language}"
subprocess.run(["git", "commit", "-m", message], check=True) subprocess.run(["git", "commit", "-m", message], check=True)
print("Pushing branch") print("Pushing branch")
subprocess.run(["git", "push", "origin", branch_name], check=True) subprocess.run(["git", "push", "origin", branch_name], check=True)

Write Preview
Loading…
Cancel
Save