✅ Enable tests in CI for scripts (#14684)

scripts/doc_parsing_utils.py

@@ -1,5 +1,5 @@
 import re
-from typing import TypedDict
+from typing import TypedDict, Union
 
 CODE_INCLUDE_RE = re.compile(r"^\{\*\s*(\S+)\s*(.*)\*\}$")
 CODE_INCLUDE_PLACEHOLDER = "<CODE_INCLUDE>"
@@ -50,8 +50,8 @@ class MarkdownLinkInfo(TypedDict):
     line_no: int
     url: str
     text: str
-    title: str | None
-    attributes: str | None
+    title: Union[str, None]
+    attributes: Union[str, None]
     full_match: str
 
 
@@ -285,7 +285,11 @@ def _add_lang_code_to_url(url: str, lang_code: str) -> str:
 
 
 def _construct_markdown_link(
-    url: str, text: str, title: str | None, attributes: str | None, lang_code: str
+    url: str,
+    text: str,
+    title: Union[str, None],
+    attributes: Union[str, None],
+    lang_code: str,
 ) -> str:
     """
     Construct a markdown link, adjusting the URL for the given language code if needed.
@@ -545,7 +549,7 @@ def extract_multiline_code_blocks(text: list[str]) -> list[MultilineCodeBlockInf
     return blocks
 
 
-def _split_hash_comment(line: str) -> tuple[str, str | None]:
+def _split_hash_comment(line: str) -> tuple[str, Union[str, None]]:
     match = HASH_COMMENT_RE.match(line)
     if match:
         code = match.group("code").rstrip()
@@ -554,7 +558,7 @@ def _split_hash_comment(line: str) -> tuple[str, str | None]:
     return line.rstrip(), None
 
 
-def _split_slashes_comment(line: str) -> tuple[str, str | None]:
+def _split_slashes_comment(line: str) -> tuple[str, Union[str, None]]:
     match = SLASHES_COMMENT_RE.match(line)
     if match:
         code = match.group("code").rstrip()
@@ -600,8 +604,8 @@ def replace_multiline_code_block(
 
     code_block: list[str] = []
     for line_a, line_b in zip(block_a["content"], block_b["content"]):
-        line_a_comment: str | None = None
-        line_b_comment: str | None = None
+        line_a_comment: Union[str, None] = None
+        line_b_comment: Union[str, None] = None
 
         # Handle comments based on language
         if block_language in {

scripts/docs.py

@@ -239,7 +239,7 @@ def generate_readme() -> None:
     Generate README.md content from main index.md
     """
     readme_path = Path("README.md")
-    old_content = readme_path.read_text()
+    old_content = readme_path.read_text("utf-8")
     new_content = generate_readme_content()
     if new_content != old_content:
         print("README.md outdated from the latest index.md")

scripts/notify_translations.py

@@ -316,7 +316,7 @@ def main() -> None:
         raise RuntimeError(
             f"No github event file available at: {settings.github_event_path}"
         )
-    contents = settings.github_event_path.read_text()
+    contents = settings.github_event_path.read_text("utf-8")
     github_event = PartialGitHubEvent.model_validate_json(contents)
     logging.info(f"Using GitHub event: {github_event}")
     number = (

scripts/test.sh

@@ -4,4 +4,4 @@ set -e
 set -x
 
 export PYTHONPATH=./docs_src
-coverage run -m pytest tests ${@}
+coverage run -m pytest tests scripts/tests/ ${@}

scripts/tests/test_translation_fixer/conftest.py

@@ -1,9 +1,19 @@
 import shutil
+import sys
 from pathlib import Path
 
 import pytest
 from typer.testing import CliRunner
 
+skip_on_windows = pytest.mark.skipif(
+    sys.platform == "win32", reason="Skipping on Windows"
+)
+
+
+def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
+    for item in items:
+        item.add_marker(skip_on_windows)
+
 
 @pytest.fixture(name="runner")
 def get_runner():

scripts/tests/test_translation_fixer/test_code_blocks/test_code_blocks_lines_number_mismatch.py

@@ -22,10 +22,10 @@ def test_gt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(
-        f"{data_path}/translated_doc_lines_number_gt.md"
-    ).read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_lines_number_gt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output
@@ -46,10 +46,10 @@ def test_lt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     # assert result.exit_code == 1, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(
-        f"{data_path}/translated_doc_lines_number_lt.md"
-    ).read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_lines_number_lt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output

scripts/tests/test_translation_fixer/test_code_blocks/test_code_blocks_mermaid.py

@@ -22,10 +22,10 @@ def test_translated(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 0, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
     expected_content = Path(
         f"{data_path}/translated_doc_mermaid_translated.md"
-    ).read_text()
+    ).read_text("utf-8")
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert (
@@ -50,10 +50,10 @@ def test_not_translated(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 0, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
     expected_content = Path(
         f"{data_path}/translated_doc_mermaid_not_translated.md"
-    ).read_text()
+    ).read_text("utf-8")
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert ("Skipping mermaid code block replacement") not in result.output

scripts/tests/test_translation_fixer/test_code_blocks/test_code_blocks_number_mismatch.py

@@ -22,8 +22,10 @@ def test_gt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(f"{data_path}/translated_doc_number_gt.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_number_gt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output
@@ -45,8 +47,10 @@ def test_lt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     # assert result.exit_code == 1, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(f"{data_path}/translated_doc_number_lt.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_number_lt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output

scripts/tests/test_translation_fixer/test_code_blocks/test_code_blocks_wrong_lang_code.py

@@ -22,10 +22,10 @@ def test_wrong_lang_code_1(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(
-        f"{data_path}/translated_doc_wrong_lang_code.md"
-    ).read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_wrong_lang_code.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output
@@ -46,10 +46,10 @@ def test_wrong_lang_code_2(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
     expected_content = Path(
         f"{data_path}/translated_doc_wrong_lang_code_2.md"
-    ).read_text()
+    ).read_text("utf-8")
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output

scripts/tests/test_translation_fixer/test_code_includes/test_number_mismatch.py

@@ -22,8 +22,10 @@ def test_gt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(f"{data_path}/translated_doc_number_gt.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_number_gt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output
@@ -45,8 +47,10 @@ def test_lt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(f"{data_path}/translated_doc_number_lt.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_number_lt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output

scripts/tests/test_translation_fixer/test_complex_doc/test_compex_doc.py

@@ -22,8 +22,8 @@ def test_fix(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 0, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = (data_path / "translated_doc_expected.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = (data_path / "translated_doc_expected.md").read_text("utf-8")
     assert fixed_content == expected_content
 
     assert "Fixing multiline code blocks in" in result.output

scripts/tests/test_translation_fixer/test_header_permalinks/test_header_level_mismatch.py

@@ -22,10 +22,10 @@ def test_level_mismatch_1(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
     expected_content = Path(
         f"{data_path}/translated_doc_level_mismatch_1.md"
-    ).read_text()
+    ).read_text("utf-8")
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output
@@ -47,10 +47,10 @@ def test_level_mismatch_2(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
     expected_content = Path(
         f"{data_path}/translated_doc_level_mismatch_2.md"
-    ).read_text()
+    ).read_text("utf-8")
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output

scripts/tests/test_translation_fixer/test_header_permalinks/test_header_number_mismatch.py

@@ -22,8 +22,10 @@ def test_gt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(f"{data_path}/translated_doc_number_gt.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_number_gt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output
@@ -45,8 +47,10 @@ def test_lt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(f"{data_path}/translated_doc_number_lt.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_number_lt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output

scripts/tests/test_translation_fixer/test_html_links/test_html_links_number_mismatch.py

@@ -20,8 +20,10 @@ def test_gt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(f"{data_path}/translated_doc_number_gt.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_number_gt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output
@@ -43,8 +45,10 @@ def test_lt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     # assert result.exit_code == 1, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(f"{data_path}/translated_doc_number_lt.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_number_lt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output

scripts/tests/test_translation_fixer/test_markdown_links/test_mkd_links_number_mismatch.py

@@ -22,8 +22,10 @@ def test_gt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     assert result.exit_code == 1, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(f"{data_path}/translated_doc_number_gt.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_number_gt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output
@@ -45,8 +47,10 @@ def test_lt(runner: CliRunner, root_dir: Path, copy_test_files):
     )
     # assert result.exit_code == 1, result.output
 
-    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text()
-    expected_content = Path(f"{data_path}/translated_doc_number_lt.md").read_text()
+    fixed_content = (root_dir / "docs" / "lang" / "docs" / "doc.md").read_text("utf-8")
+    expected_content = Path(f"{data_path}/translated_doc_number_lt.md").read_text(
+        "utf-8"
+    )
 
     assert fixed_content == expected_content  # Translated doc remains unchanged
     assert "Error processing docs/lang/docs/doc.md" in result.output