"""Generate data/*.yaml. Two sources: * ``data/termine.yaml`` comes from the six TablePress tables in the WXR export. * ``data/members.yaml`` and ``data/memoriam.yaml`` come from the *live* homepage. Those two blocks were WordPress widgets, and WXR does not export widgets at all, so scraping the rendered page is the only way to recover them. Run once during the migration. After that the YAML files are the source of truth. """ from __future__ import annotations import html import json import re import sys import urllib.request from pathlib import Path from wxr import REPO_ROOT, clean_media_name, load_items LIVE_HOME = "https://motorradclub-giebelwald.de/" USER_AGENT = "mcg-website-migration/1.0 (+https://mcg.luxick.de)" DATA = REPO_ROOT / "data" def yaml_str(value: str) -> str: """Always double-quote; the cells contain colons, umlauts and inline HTML.""" return '"' + value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + '"' # --------------------------------------------------------------------------- termine def build_termine() -> None: items = load_items() tables = [i for i in items.values() if i.post_type == "tablepress_table"] years: dict[int, list[dict[str, str]]] = {} for table in tables: match = re.search(r"(\d{4})", table.title) if not match: print(f"!! skipping table without a year in the title: {table.title}", file=sys.stderr) continue year = int(match.group(1)) rows = [] for row in json.loads(table.content): date = (row[0] if len(row) > 0 else "").strip() desc = (row[1] if len(row) > 1 else "").strip() if not date and not desc: continue # TablePress spacer rows rows.append({"date": date, "description": desc}) years[year] = rows lines = [ "# Vereinstermine, ursprünglich TablePress-Tabellen aus WordPress.", "# Neue Termine: einfach unter dem passenden Jahr ergänzen.", "", ] for year in sorted(years, reverse=True): lines.append(f'"{year}":') if not years[year]: lines[-1] += " []" continue for row in years[year]: lines.append(f" - date: {yaml_str(row['date'])}") lines.append(f" description: {yaml_str(row['description'])}") lines.append("") (DATA / "termine.yaml").write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") total = sum(len(v) for v in years.values()) print(f"termine.yaml: {len(years)} Jahre, {total} Termine") # ------------------------------------------------------------------- widget scraping def live_home() -> str: req = urllib.request.Request(LIVE_HOME, headers={"User-Agent": USER_AGENT}) with urllib.request.urlopen(req, timeout=60) as resp: return resp.read().decode("utf-8", errors="replace") GALLERY_ITEM = re.compile( r"", re.S, ) def gallery_entries(fragment: str) -> list[tuple[str, str]]: out = [] for m in GALLERY_ITEM.finditer(fragment): href = m.group("href") caption = html.unescape(re.sub(r"<[^>]+>", "", m.group("caption") or "")).strip() out.append((href, caption)) return out def image_ref(url: str) -> str: """Map an uploads URL to the committed path under assets/img/.""" return "img/" + clean_media_name(url.rsplit("/", 1)[-1]) def build_widgets() -> None: page = live_home() # Each widget renders as

NAME

followed by a WP # gallery. Keep the trailing inside the group, otherwise the last # entry of every gallery is silently dropped. members_html = re.search( r"Unsere Mitglieder(.*?)\s*", page, re.S ) memoriam_html = re.search( r"In Erinnerung(.*?)\s*", page, re.S ) if not members_html or not memoriam_html: raise SystemExit("could not locate the member/memoriam widgets on the live page") members = gallery_entries(members_html.group(1)) memoriam = gallery_entries(memoriam_html.group(1)) # The WordPress gallery widget shuffles its order on every request, so the # source order carries no meaning. Sort by name for a stable, readable list. members.sort(key=lambda entry: entry[1].casefold()) lines = [ "# Mitglieder-Galerie der Startseite.", "# Aus dem WordPress-Widget übernommen (Widgets sind nicht im XML-Export enthalten).", "", ] for url, name in members: lines.append(f"- name: {yaml_str(name)}") lines.append(f" image: {yaml_str(image_ref(url))}") (DATA / "members.yaml").write_text("\n".join(lines) + "\n", encoding="utf-8") print(f"members.yaml: {len(members)} Mitglieder") lines = [ "# \"In Erinnerung\" – ebenfalls aus einem WordPress-Widget übernommen.", "", ] for url, caption in memoriam: # Captions look like "Roland Groos ✝ 27.5.2020". name, _, died = caption.partition("✝") lines.append(f"- name: {yaml_str(name.strip())}") lines.append(f" died: {yaml_str(died.strip())}") lines.append(f" image: {yaml_str(image_ref(url))}") (DATA / "memoriam.yaml").write_text("\n".join(lines) + "\n", encoding="utf-8") print(f"memoriam.yaml: {len(memoriam)} Einträge") return [image_ref(u) for u, _ in members] + [image_ref(u) for u, _ in memoriam] def main() -> int: DATA.mkdir(parents=True, exist_ok=True) build_termine() build_widgets() return 0 if __name__ == "__main__": raise SystemExit(main())