convert.py
raw ·
blame ·
history
#!/usr/bin/env python3
"""Convert Quake 3 tutorial HTML dumps (from archive.org) to clean Markdown."""
import os
import re
import shutil
from pathlib import Path
from bs4 import BeautifulSoup, NavigableString, Tag
BASE = Path(__file__).parent / "html_dumps_from_archive_org"
OUT = Path(__file__).parent / "leveldk.co.uk"
IMAGES = OUT / "images"
# (source html number, output stem, full title)
TUTORIALS = [
(2, "tutorial-01", "Basic Room"),
(3, "tutorial-02", "2-Point Clipping, Brush and Patch Work"),
(4, "tutorial-03", "Detail and Structural Brushes, Level Layout"),
(5, "tutorial-04", "Advanced Patch Work, Lighting, Stairs and Ramps"),
(6, "tutorial-05", "Stuff with Holes, Liquids, Rocks and Terrain"),
(7, "tutorial-06", "Lighting the Level – Advanced Techniques"),
(8, "tutorial-07", "Compiling, Creating .pk3 Files"),
(9, "tutorial-08", "Converting the Level for CTF"),
(10, "tutorial-09", "Extra Entities"),
]
# File extensions and patterns to wrap in backticks
_CODE_EXTS = r"(?:pk3|bsp|map|arena|aas|ase|shader|zip|cfg|bat|htm|txt|jpg|png|md3)"
_FILE_RE = re.compile(
r"(?<![`\w])" # not already in backtick / word char
r"([\w][\w\-\.]*\.(?:" + _CODE_EXTS + r"))" # filename.ext
r"(?![`\w])",
re.IGNORECASE,
)
_CMD_RE = re.compile(
r"(?<![`\w/])"
r"(/[a-zA-Z_][a-zA-Z0-9_]*(?:\s+[\S]+)*?)(?=[,\.!\?\s]|$)"
)
def apply_inline_code(text: str) -> str:
"""Wrap console commands and filenames in backticks."""
# Wrap slash-prefixed console commands: /sv_pure 0, /devmap, etc.
text = _CMD_RE.sub(r"`\1`", text)
# Wrap filenames with known Q3/mapping extensions
text = _FILE_RE.sub(r"`\1`", text)
return text
def fix_encoding(text: str) -> str:
"""Fix Windows-1252 curly quotes / dashes that survived as latin-1 bytes."""
replacements = {
"\u2018": "'", # left single quotation mark
"\u2019": "'", # right single quotation mark (also apostrophe)
"\u201c": '"', # left double quotation mark
"\u201d": '"', # right double quotation mark
"\u2013": "--", # en dash
"\u2014": "--", # em dash
"\u2026": "...",# ellipsis
"\u00e2\u0080\u0099": "'", # UTF-8 misread artefact
# Raw windows-1252 bytes that sometimes survive
"\x91": "'", "\x92": "'",
"\x93": '"', "\x94": '"',
"\x96": "--", "\x97": "--",
}
for bad, good in replacements.items():
text = text.replace(bad, good)
return text
def element_to_markdown(el, first_heading_seen: list) -> str | None:
"""
Convert a single BeautifulSoup element to a Markdown string.
Returns None to signal 'skip this element'.
first_heading_seen is a mutable list used as a flag.
"""
if not isinstance(el, Tag):
return None
if el.name != "p":
return None
# --- Image paragraph (check before blank-text check, img has no text) ---
img = el.find("img")
if img:
src = img.get("src", "")
# Strip the "level_tutoral_N_files/" prefix, keep just the filename
filename = Path(src).name
return f""
# --- Blank / whitespace-only paragraphs ---
raw_text = el.get_text(separator=" ").strip()
if not raw_text or raw_text == "\xa0":
return None # skip spacer
# --- Skip the archive.org "Back to index" link paragraph ---
link = el.find("a")
if link and "archive.org" in link.get("href", ""):
return None
# --- Figure caption paragraph: starts with "fig." ---
if re.match(r"fig\.", raw_text, re.IGNORECASE):
return f"*{fix_encoding(raw_text)}*"
# --- Heading paragraph: <p align="center"><b>...</b></p> ---
align = el.get("align", "") or el.get("style", "")
is_centered = "center" in align
bold = el.find("b")
if is_centered and bold and bold.get_text(strip=True):
heading_text = bold.get_text(separator=" ").strip()
heading_text = re.sub(r"\s+", " ", heading_text)
heading_text = fix_encoding(heading_text)
if not first_heading_seen:
first_heading_seen.append(True)
return f"# {heading_text}"
else:
return f"## {heading_text}"
# --- Regular paragraph: collect inline markup ---
parts = []
for child in el.children:
if isinstance(child, NavigableString):
# Normalise inline whitespace (HTML source has bare newlines)
parts.append(re.sub(r"[\r\n]+", " ", str(child)))
elif child.name == "b":
inner = child.get_text(separator=" ").strip()
if inner:
parts.append(f"**{inner}**")
elif child.name == "i":
inner = child.get_text(separator=" ").strip()
if inner:
parts.append(f"*{inner}*")
elif child.name == "a":
# Strip archive.org URLs; just keep link text
parts.append(child.get_text(separator=" "))
elif child.name == "u":
inner = child.get_text(separator=" ")
parts.append(inner)
elif child.name == "img":
pass # handled above at paragraph level
else:
parts.append(child.get_text(separator=" "))
# Ensure a space separates a closing *...* marker from the next word.
# Do this at the parts level (before joining) to avoid mistakenly adding
# space after an *opening* marker like team_CTF_*colour*flag.
normalized: list[str] = []
for part in parts:
if (normalized
and normalized[-1].endswith(("*", "**"))
and part
and part[0].isalpha()):
normalized.append(" ")
normalized.append(part)
parts = normalized
para = "".join(parts).strip()
para = fix_encoding(para)
para = apply_inline_code(para)
# Collapse excessive whitespace
para = re.sub(r"[ \t]+", " ", para)
para = re.sub(r"\n+", " ", para)
if not para:
return None
return para
def convert_tutorial(num: int, stem: str, title: str) -> None:
html_path = BASE / f"level_tutoral_{num}.html"
files_dir = BASE / f"level_tutoral_{num}_files"
# Read bytes and let BS detect encoding from meta charset
raw = html_path.read_bytes()
soup = BeautifulSoup(raw, "html.parser", from_encoding="windows-1252")
body = soup.find("body")
if not body:
print(f" WARNING: no <body> found in {html_path.name}")
return
# --- Copy images ---
if files_dir.exists():
for img_file in files_dir.glob("*.jpg"):
shutil.copy2(img_file, IMAGES / img_file.name)
# --- Build Markdown ---
lines = []
first_heading = []
for el in body.children:
md = element_to_markdown(el, first_heading)
if md is not None:
lines.append(md)
# If the page title wasn't in the HTML headings, prepend it
if not first_heading:
lines.insert(0, f"# {title}")
# --- Navigation footer ---
idx = next(i for i, t in enumerate(TUTORIALS) if t[0] == num)
prev_link = f"[← Tutorial {idx}]({TUTORIALS[idx - 1][1]}.md)" if idx > 0 else ""
next_link = f"[Tutorial {idx + 2} →]({TUTORIALS[idx + 1][1]}.md)" if idx < len(TUTORIALS) - 1 else ""
nav_parts = [p for p in [prev_link, "[Index](index.md)", next_link] if p]
nav = " | ".join(nav_parts)
lines.append("")
lines.append("---")
lines.append(nav)
# Write output
out_path = OUT / f"{stem}.md"
out_path.write_text("\n\n".join(lines) + "\n", encoding="utf-8")
print(f" Written {out_path.name} ({len(lines)} blocks)")
def build_index() -> None:
descriptions = {
2: "Creating a brush, selecting textures, changing views, selecting faces, adding a start point, saving and compiling.",
3: "2-point clipping, endcaps and bevels, making a curved hallway, texturing patches.",
4: "Detail and structural brushes, level layout, triggers and traps, adding geometry, .ase models.",
5: "Advanced patch manipulation, lighting tips, stairs and ramps including spiral stairs.",
6: "Nodraw and surfaces with holes, custom textures and shaders, alphaMod volumes, lava/liquids, rocks and terrain.",
7: "Advanced lighting techniques, light styles, clip brushes and bot optimisation, hint brushes, clusterportals.",
8: "Compiling the map, finishing the level, creating a .pk3, adding music, .arena files, levelshots and batch files.",
9: "Converting the level for CTF: team_CTF spawn, player and flag entities.",
10: "Extra entities: func_door, func_rotating, func_bobbing, func_train, func_plat, areaportals, target_speaker, triggers.",
}
lines = [
"# Quake 3 Level Design Tutorials",
"",
"A series of tutorials covering the construction of a complete Quake 3 Arena level "
"for both Deathmatch and Capture the Flag, using GTKRadiant 1.4.",
"",
"All map files and media are available in `dk_lmtut.pk3`.",
"",
"## Tutorials",
"",
]
for i, (num, stem, title) in enumerate(TUTORIALS, 1):
desc = descriptions.get(num, "")
lines.append(f"- **[Tutorial {i}: {title}]({stem}.md)** — {desc}")
out = OUT / "index.md"
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
print(f" Written index.md")
def main() -> None:
IMAGES.mkdir(parents=True, exist_ok=True)
OUT.mkdir(parents=True, exist_ok=True)
for num, stem, title in TUTORIALS:
print(f"Converting tutorial {num} → {stem}.md …")
convert_tutorial(num, stem, title)
print("Building index …")
build_index()
print("Done.")
if __name__ == "__main__":
main()