Yurii Motov
6 months ago
2 changed files with 530 additions and 0 deletions
@ -0,0 +1,444 @@ |
|||||
|
import re |
||||
|
from typing import TypedDict |
||||
|
|
||||
|
CODE_INCLUDE_RE = re.compile(r"^\{\*\s*(\S+)\s*(.*)\*\}$") |
||||
|
CODE_INCLUDE_PLACEHOLDER = "<CODE_INCLUDE>" |
||||
|
|
||||
|
HEADER_WITH_PERMALINK_RE = re.compile(r"^(#{1,6}) (.+?)(\s*\{\s*#.*\s*\})?\s*$") |
||||
|
HEADER_LINE_RE = re.compile(r"^(#{1,6}) (.+?)(?:\s*\{\s*(#.*)\s*\})?\s*$") |
||||
|
|
||||
|
TIANGOLO_COM = "https://fastapi.tiangolo.com" |
||||
|
|
||||
|
MARKDOWN_LINK_RE = re.compile( |
||||
|
r"(?<!\\)(?<!\!)" # not an image ![...] and not escaped \[...] |
||||
|
r"\[(?P<text>.*?)\]" # link text (non-greedy) |
||||
|
r"\(" |
||||
|
r"(?P<url>[^)\s]+)" # url (no spaces and `)`) |
||||
|
r'(?:\s+["\'](?P<title>.*?)["\'])?' # optional title in "" or '' |
||||
|
r"\)" |
||||
|
r"(?:\s*\{(?P<attrs>[^}]*)\})?" # optional attributes in {} |
||||
|
) |
||||
|
|
||||
|
HTML_LINK_RE = re.compile(r"<a\s+[^>]*>.*?</a>") |
||||
|
HTML_LINK_TEXT = re.compile(r"<a\b([^>]*)>(.*?)</a>") |
||||
|
HTML_LINK_OPEN_TAG_RE = re.compile(r"<a\b([^>]*)>") |
||||
|
HTML_ATTR_RE = re.compile(r'(\w+)\s*=\s*([\'"])(.*?)\2') |
||||
|
|
||||
|
|
||||
|
class CodeIncludeInfo(TypedDict): |
||||
|
line_no: int |
||||
|
line: str |
||||
|
|
||||
|
|
||||
|
class HeaderPermalinkInfo(TypedDict): |
||||
|
line_no: int |
||||
|
hashes: str |
||||
|
permalink: str |
||||
|
|
||||
|
|
||||
|
class MarkdownLinkInfo(TypedDict): |
||||
|
line_no: int |
||||
|
url: str |
||||
|
text: str |
||||
|
title: str | None |
||||
|
attributes: str | None |
||||
|
|
||||
|
|
||||
|
class HTMLLinkAttribute(TypedDict): |
||||
|
name: str |
||||
|
quote: str |
||||
|
value: str |
||||
|
|
||||
|
|
||||
|
class HtmlLinkInfo(TypedDict): |
||||
|
line_no: int |
||||
|
full_tag: str |
||||
|
attributes: list[HTMLLinkAttribute] |
||||
|
text: str |
||||
|
|
||||
|
|
||||
|
# Code includes |
||||
|
# ----------------------------------------------------------------------------------------- |
||||
|
|
||||
|
|
||||
|
def extract_code_includes(lines: list[str]) -> list[CodeIncludeInfo]: |
||||
|
""" |
||||
|
Exctract lines that contain code includes. |
||||
|
|
||||
|
Return list of CodeIncludeInfo namedtuples, where each tuple contains: |
||||
|
- `line_no` - line number (1-based) |
||||
|
- `line` - text of the line |
||||
|
""" |
||||
|
|
||||
|
includes: list[CodeIncludeInfo] = [] |
||||
|
for line_no, line in enumerate(lines, start=1): |
||||
|
if CODE_INCLUDE_RE.match(line): |
||||
|
includes.append(CodeIncludeInfo(line_no=line_no, line=line)) |
||||
|
return includes |
||||
|
|
||||
|
|
||||
|
def replace_code_includes_with_placeholders(text: list[str]) -> list[str]: |
||||
|
""" |
||||
|
Replace code includes with placeholders. |
||||
|
""" |
||||
|
|
||||
|
includes = extract_code_includes(text) |
||||
|
for include in includes: |
||||
|
text[include["line_no"] - 1] = CODE_INCLUDE_PLACEHOLDER |
||||
|
return text |
||||
|
|
||||
|
|
||||
|
def replace_placeholders_with_code_includes( |
||||
|
text: list[str], original_includes: list[CodeIncludeInfo] |
||||
|
) -> list[str]: |
||||
|
""" |
||||
|
Replace code includes placeholders with actual code includes from the original (English) document. |
||||
|
Fail if the number of placeholders does not match the number of original includes. |
||||
|
""" |
||||
|
|
||||
|
modified_text: list[str] = [] |
||||
|
include_index = 0 |
||||
|
for line in text: |
||||
|
if line.strip() == CODE_INCLUDE_PLACEHOLDER: |
||||
|
if include_index >= len(original_includes): |
||||
|
raise ValueError( |
||||
|
"Number of placeholders exceeds number of code includes in the original document" |
||||
|
) |
||||
|
modified_text.append(original_includes[include_index]["line"]) |
||||
|
include_index += 1 |
||||
|
else: |
||||
|
modified_text.append(line) |
||||
|
|
||||
|
if include_index < len(original_includes): |
||||
|
raise ValueError( |
||||
|
"Number of placeholders is less than number of code includes in the original document" |
||||
|
) |
||||
|
|
||||
|
return modified_text |
||||
|
|
||||
|
|
||||
|
# Header permalinks |
||||
|
# ----------------------------------------------------------------------------------------- |
||||
|
|
||||
|
|
||||
|
def extract_header_permalinks(lines: list[str]) -> list[HeaderPermalinkInfo]: |
||||
|
""" |
||||
|
Extract list of header permalinks from the given lines. |
||||
|
|
||||
|
Return list of HeaderPermalinkInfo namedtuples, where each tuple contains: |
||||
|
- `line_no` - line number (1-based) |
||||
|
- `hashes` - string of hashes representing header level (e.g., "###") |
||||
|
- `permalink` - permalink string (e.g., "{#permalink}") |
||||
|
""" |
||||
|
|
||||
|
headers: list[HeaderPermalinkInfo] = [] |
||||
|
in_code_block3 = False |
||||
|
in_code_block4 = False |
||||
|
|
||||
|
for line_no, line in enumerate(lines, start=1): |
||||
|
if not (in_code_block3 or in_code_block4): |
||||
|
if line.startswith("```"): |
||||
|
count = len(line) - len(line.lstrip("`")) |
||||
|
if count == 3: |
||||
|
in_code_block3 = True |
||||
|
continue |
||||
|
elif count >= 4: |
||||
|
in_code_block4 = True |
||||
|
continue |
||||
|
|
||||
|
header_match = HEADER_WITH_PERMALINK_RE.match(line) |
||||
|
if header_match: |
||||
|
hashes, _title, permalink = header_match.groups() |
||||
|
headers.append( |
||||
|
HeaderPermalinkInfo( |
||||
|
hashes=hashes, line_no=line_no, permalink=permalink |
||||
|
) |
||||
|
) |
||||
|
|
||||
|
elif in_code_block3: |
||||
|
if line.startswith("```"): |
||||
|
count = len(line) - len(line.lstrip("`")) |
||||
|
if count == 3: |
||||
|
in_code_block3 = False |
||||
|
continue |
||||
|
|
||||
|
elif in_code_block4: |
||||
|
if line.startswith("````"): |
||||
|
count = len(line) - len(line.lstrip("`")) |
||||
|
if count >= 4: |
||||
|
in_code_block4 = False |
||||
|
continue |
||||
|
|
||||
|
return headers |
||||
|
|
||||
|
|
||||
|
def remove_header_permalinks(lines: list[str]) -> list[str]: |
||||
|
""" |
||||
|
Remove permalinks from headers in the given lines. |
||||
|
""" |
||||
|
|
||||
|
modified_lines: list[str] = [] |
||||
|
for line in lines: |
||||
|
header_match = HEADER_WITH_PERMALINK_RE.match(line) |
||||
|
if header_match: |
||||
|
hashes, title, _permalink = header_match.groups() |
||||
|
modified_line = f"{hashes} {title}" |
||||
|
modified_lines.append(modified_line) |
||||
|
else: |
||||
|
modified_lines.append(line) |
||||
|
return modified_lines |
||||
|
|
||||
|
|
||||
|
def replace_header_permalinks( |
||||
|
text: list[str], original_permalinks: list[HeaderPermalinkInfo] |
||||
|
) -> list[str]: |
||||
|
""" |
||||
|
Replace permalinks in the given text with the permalinks from the original document. |
||||
|
|
||||
|
Fail if the number or order of headers does not match the original. |
||||
|
""" |
||||
|
|
||||
|
modified_text: list[str] = [] |
||||
|
permalink_index = 0 |
||||
|
for line in text: |
||||
|
header_match = HEADER_LINE_RE.match(line) |
||||
|
if header_match: |
||||
|
if permalink_index >= len(original_permalinks): |
||||
|
raise ValueError( |
||||
|
"Number of headers exceeds number of headers in the original document" |
||||
|
) |
||||
|
hashes, title, _permalink = header_match.groups() |
||||
|
original_permalink_info = original_permalinks[permalink_index] |
||||
|
if original_permalink_info["hashes"] != hashes: |
||||
|
raise ValueError( |
||||
|
"Header levels do not match between document and original document" |
||||
|
) |
||||
|
|
||||
|
modified_line = f"{hashes} {title}{original_permalink_info['permalink']}" |
||||
|
modified_text.append(modified_line) |
||||
|
permalink_index += 1 |
||||
|
else: |
||||
|
modified_text.append(line) |
||||
|
|
||||
|
if permalink_index < len(original_permalinks): |
||||
|
raise ValueError( |
||||
|
"Number of headers is less than number of headers in the original document" |
||||
|
) |
||||
|
|
||||
|
return modified_text |
||||
|
|
||||
|
|
||||
|
# Markdown links |
||||
|
# ----------------------------------------------------------------------------------------- |
||||
|
|
||||
|
|
||||
|
def extract_markdown_links(lines: list[str]) -> list[tuple[str, int]]: |
||||
|
""" |
||||
|
Extract all markdown links from the given lines. |
||||
|
|
||||
|
Return list of MarkdownLinkInfo namedtuples, where each tuple contains: |
||||
|
- `line_no` - line number (1-based) |
||||
|
- `url` - link URL |
||||
|
- `text` - link text |
||||
|
- `title` - link title (if any) |
||||
|
""" |
||||
|
|
||||
|
links: list[MarkdownLinkInfo] = [] |
||||
|
for line_no, line in enumerate(lines, start=1): |
||||
|
for m in MARKDOWN_LINK_RE.finditer(line): |
||||
|
links.append( |
||||
|
MarkdownLinkInfo( |
||||
|
line_no=line_no, |
||||
|
url=m.group("url"), |
||||
|
text=m.group("text"), |
||||
|
title=m.group("title"), |
||||
|
attributes=m.group("attrs"), |
||||
|
) |
||||
|
) |
||||
|
return links |
||||
|
|
||||
|
|
||||
|
def _construct_markdown_link( |
||||
|
url: str, text: str, title: str | None, attributes: str | None, lang_code: str |
||||
|
) -> str: |
||||
|
""" |
||||
|
Construct a markdown link, adjusting the URL for the given language code if needed. |
||||
|
""" |
||||
|
|
||||
|
if url.startswith(TIANGOLO_COM): |
||||
|
url = url.replace(TIANGOLO_COM, f"{TIANGOLO_COM}/{lang_code}") |
||||
|
|
||||
|
if title: |
||||
|
link = f'[{text}]({url} "{title}")' |
||||
|
else: |
||||
|
link = f"[{text}]({url})" |
||||
|
|
||||
|
if attributes: |
||||
|
link += f" {{{attributes}}}" |
||||
|
|
||||
|
return link |
||||
|
|
||||
|
|
||||
|
def replace_markdown_links( |
||||
|
text: list[str], original_links: list[MarkdownLinkInfo], lang_code: str |
||||
|
) -> list[str]: |
||||
|
""" |
||||
|
Replace markdown links in the given text with the original links. |
||||
|
|
||||
|
Fail if the number of links does not match the original. |
||||
|
""" |
||||
|
|
||||
|
modified_text: list[str] = [] |
||||
|
link_index = 0 |
||||
|
for line in text: |
||||
|
modified_line = line |
||||
|
for m in MARKDOWN_LINK_RE.finditer(line): |
||||
|
if link_index >= len(original_links): |
||||
|
raise ValueError( |
||||
|
"Number of markdown links exceeds number of markdown links in the original document" |
||||
|
) |
||||
|
link_text = m.group("text") |
||||
|
assert isinstance(link_text, str) |
||||
|
link_title = m.group("title") |
||||
|
assert link_title is None or isinstance(link_title, str) |
||||
|
|
||||
|
original_link_info = original_links[link_index] |
||||
|
|
||||
|
# Replace |
||||
|
replacement_link = _construct_markdown_link( |
||||
|
url=original_link_info["url"], |
||||
|
text=link_text, |
||||
|
title=link_title, |
||||
|
attributes=original_link_info["attributes"], |
||||
|
lang_code=lang_code, |
||||
|
) |
||||
|
modified_line = modified_line.replace(m.group(0), replacement_link, 1) |
||||
|
|
||||
|
link_index += 1 |
||||
|
modified_text.append(modified_line) |
||||
|
|
||||
|
if link_index < len(original_links): |
||||
|
raise ValueError( |
||||
|
"Number of markdown links is less than in the original document" |
||||
|
) |
||||
|
|
||||
|
return modified_text |
||||
|
|
||||
|
|
||||
|
# HTML links |
||||
|
# ----------------------------------------------------------------------------------------- |
||||
|
|
||||
|
|
||||
|
def extract_html_links(lines: list[str]) -> list[HtmlLinkInfo]: |
||||
|
""" |
||||
|
Extract all HTML links from the given lines. |
||||
|
|
||||
|
Return list of HtmlLinkInfo namedtuples, where each tuple contains: |
||||
|
- `line_no` - line number (1-based) |
||||
|
- `full_tag` - full HTML link tag |
||||
|
- `attributes` - list of HTMLLinkAttribute namedtuples (name, quote, value) |
||||
|
- `text` - link text |
||||
|
""" |
||||
|
|
||||
|
links = [] |
||||
|
for line_no, line in enumerate(lines, start=1): |
||||
|
for html_link in HTML_LINK_RE.finditer(line): |
||||
|
link_str = html_link.group(0) |
||||
|
|
||||
|
link_text_match = HTML_LINK_TEXT.match(link_str) |
||||
|
assert link_text_match is not None |
||||
|
link_text = link_text_match.group(2) |
||||
|
assert isinstance(link_text, str) |
||||
|
|
||||
|
link_open_tag_match = HTML_LINK_OPEN_TAG_RE.match(link_str) |
||||
|
assert link_open_tag_match is not None |
||||
|
link_open_tag = link_open_tag_match.group(1) |
||||
|
assert isinstance(link_open_tag, str) |
||||
|
|
||||
|
attributes: list[HTMLLinkAttribute] = [] |
||||
|
for attr_name, attr_quote, attr_value in re.findall( |
||||
|
HTML_ATTR_RE, link_open_tag |
||||
|
): |
||||
|
assert isinstance(attr_name, str) |
||||
|
assert isinstance(attr_quote, str) |
||||
|
assert isinstance(attr_value, str) |
||||
|
attributes.append( |
||||
|
HTMLLinkAttribute( |
||||
|
name=attr_name, quote=attr_quote, value=attr_value |
||||
|
) |
||||
|
) |
||||
|
links.append( |
||||
|
HtmlLinkInfo( |
||||
|
line_no=line_no, |
||||
|
full_tag=link_str, |
||||
|
attributes=attributes, |
||||
|
text=link_text, |
||||
|
) |
||||
|
) |
||||
|
return links |
||||
|
|
||||
|
|
||||
|
def _construct_html_link( |
||||
|
link_text: str, |
||||
|
attributes: list[HTMLLinkAttribute], |
||||
|
lang_code: str, |
||||
|
) -> str: |
||||
|
""" |
||||
|
Reconstruct HTML link, adjusting the URL for the given language code if needed. |
||||
|
""" |
||||
|
|
||||
|
attributes_upd: list[HTMLLinkAttribute] = [] |
||||
|
for attribute in attributes: |
||||
|
if attribute["name"] == "href": |
||||
|
original_url = attribute["value"] |
||||
|
if original_url.startswith(TIANGOLO_COM): |
||||
|
url = original_url.replace(TIANGOLO_COM, f"{TIANGOLO_COM}/{lang_code}") |
||||
|
else: |
||||
|
url = original_url |
||||
|
attributes_upd.append( |
||||
|
HTMLLinkAttribute(name="href", quote=attribute["quote"], value=url) |
||||
|
) |
||||
|
else: |
||||
|
attributes_upd.append(attribute) |
||||
|
|
||||
|
attrs_str = " ".join( |
||||
|
f"{attribute['name']}={attribute['quote']}{attribute['value']}{attribute['quote']}" |
||||
|
for attribute in attributes_upd |
||||
|
) |
||||
|
return f"<a {attrs_str}>{link_text}</a>" |
||||
|
|
||||
|
|
||||
|
def replace_html_links( |
||||
|
text: list[str], original_links: list[HtmlLinkInfo], lang_code: str |
||||
|
) -> list[str]: |
||||
|
""" |
||||
|
Replace HTML links in the given text with the links from the original document. |
||||
|
|
||||
|
Adjust URLs for the given language code. |
||||
|
Fail if the number of links does not match the original. |
||||
|
""" |
||||
|
|
||||
|
links = extract_html_links(text) |
||||
|
if len(links) > len(original_links): |
||||
|
raise ValueError( |
||||
|
"Number of HTML links exceeds number of HTML links in the original document" |
||||
|
) |
||||
|
elif len(links) < len(original_links): |
||||
|
raise ValueError("Number of HTML links is less than in the original document") |
||||
|
|
||||
|
modified_text = text.copy() |
||||
|
for link_index, link in enumerate(links): |
||||
|
original_link_info = original_links[link_index] |
||||
|
|
||||
|
# Replace in the document text |
||||
|
replacement_link = _construct_html_link( |
||||
|
link_text=link["text"], |
||||
|
attributes=original_link_info["attributes"], |
||||
|
lang_code=lang_code, |
||||
|
) |
||||
|
line_no = link["line_no"] - 1 |
||||
|
modified_text[line_no] = modified_text[line_no].replace( |
||||
|
link["full_tag"], replacement_link, 1 |
||||
|
) |
||||
|
|
||||
|
return modified_text |
||||
@ -0,0 +1,86 @@ |
|||||
|
from pathlib import Path |
||||
|
from typing import Annotated |
||||
|
|
||||
|
import typer |
||||
|
|
||||
|
from scripts.doc_parsing_utils import ( |
||||
|
extract_code_includes, |
||||
|
extract_header_permalinks, |
||||
|
extract_html_links, |
||||
|
extract_markdown_links, |
||||
|
replace_code_includes_with_placeholders, |
||||
|
replace_header_permalinks, |
||||
|
replace_html_links, |
||||
|
replace_markdown_links, |
||||
|
replace_placeholders_with_code_includes, |
||||
|
) |
||||
|
|
||||
|
cli = typer.Typer() |
||||
|
|
||||
|
|
||||
|
@cli.callback() |
||||
|
def callback(): |
||||
|
pass |
||||
|
|
||||
|
|
||||
|
@cli.command() |
||||
|
def fix_pages( |
||||
|
doc_paths: Annotated[ |
||||
|
list[Path], |
||||
|
typer.Argument(help="List of paths to documents."), |
||||
|
], |
||||
|
): |
||||
|
for path in doc_paths: |
||||
|
lang_code = path.parts[1] |
||||
|
if lang_code == "en": |
||||
|
print(f"Skipping English document: {path}") |
||||
|
continue |
||||
|
|
||||
|
en_doc_path = Path("docs") / "en" / Path(*path.parts[2:]) |
||||
|
|
||||
|
doc_lines = path.read_text(encoding="utf-8").splitlines() |
||||
|
en_doc_lines = en_doc_path.read_text(encoding="utf-8").splitlines() |
||||
|
|
||||
|
# Fix code includes |
||||
|
en_code_includes = extract_code_includes(en_doc_lines) |
||||
|
doc_lines_with_placeholders = replace_code_includes_with_placeholders(doc_lines) |
||||
|
fixed_doc_lines = replace_placeholders_with_code_includes( |
||||
|
doc_lines_with_placeholders, en_code_includes |
||||
|
) |
||||
|
if fixed_doc_lines != doc_lines: |
||||
|
print(f"Fixing code includes in: {path}") |
||||
|
doc_lines = fixed_doc_lines |
||||
|
|
||||
|
# Fix permalinks |
||||
|
en_permalinks = extract_header_permalinks(en_doc_lines) |
||||
|
fixed_doc_lines = replace_header_permalinks(doc_lines, en_permalinks) |
||||
|
if fixed_doc_lines != doc_lines: |
||||
|
print(f"Fixing header permalinks in: {path}") |
||||
|
doc_lines = fixed_doc_lines |
||||
|
|
||||
|
# Fix markdown links |
||||
|
en_markdown_links = extract_markdown_links(en_doc_lines) |
||||
|
fixed_doc_lines = replace_markdown_links( |
||||
|
doc_lines, en_markdown_links, lang_code |
||||
|
) |
||||
|
if fixed_doc_lines != doc_lines: |
||||
|
print(f"Fixing markdown links in: {path}") |
||||
|
doc_lines = fixed_doc_lines |
||||
|
|
||||
|
# Fix HTML links |
||||
|
en_html_links = extract_html_links(en_doc_lines) |
||||
|
fixed_doc_lines = replace_html_links(doc_lines, en_html_links, lang_code) |
||||
|
if fixed_doc_lines != doc_lines: |
||||
|
print(f"Fixing HTML links in: {path}") |
||||
|
doc_lines = fixed_doc_lines |
||||
|
|
||||
|
# Fix multiline code blocks |
||||
|
# TODO: Implement |
||||
|
|
||||
|
# Write back the fixed document |
||||
|
doc_lines.append("") # Ensure file ends with a newline |
||||
|
path.write_text("\n".join(doc_lines), encoding="utf-8") |
||||
|
|
||||
|
|
||||
|
if __name__ == "__main__": |
||||
|
cli() |
||||
Loading…
Reference in new issue