mcg-website/tools/build_data.py

"""Generate data/*.yaml.

Two sources:

* ``data/termine.yaml`` comes from the six TablePress tables in the WXR export.
* ``data/members.yaml`` and ``data/memoriam.yaml`` come from the *live* homepage.
  Those two blocks were WordPress widgets, and WXR does not export widgets at all,
  so scraping the rendered page is the only way to recover them.

Run once during the migration. After that the YAML files are the source of truth.
"""

from __future__ import annotations

import html
import json
import re
import sys
import urllib.request
from pathlib import Path

from wxr import REPO_ROOT, clean_media_name, load_items

LIVE_HOME = "https://motorradclub-giebelwald.de/"
USER_AGENT = "mcg-website-migration/1.0 (+https://mcg.luxick.de)"
DATA = REPO_ROOT / "data"


def yaml_str(value: str) -> str:
    """Always double-quote; the cells contain colons, umlauts and inline HTML."""
    return '"' + value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + '"'


# --------------------------------------------------------------------------- termine


def build_termine() -> None:
    items = load_items()
    tables = [i for i in items.values() if i.post_type == "tablepress_table"]

    years: dict[int, list[dict[str, str]]] = {}
    for table in tables:
        match = re.search(r"(\d{4})", table.title)
        if not match:
            print(f"!! skipping table without a year in the title: {table.title}",
                  file=sys.stderr)
            continue
        year = int(match.group(1))

        rows = []
        for row in json.loads(table.content):
            date = (row[0] if len(row) > 0 else "").strip()
            desc = (row[1] if len(row) > 1 else "").strip()
            if not date and not desc:
                continue  # TablePress spacer rows
            rows.append({"date": date, "description": desc})
        years[year] = rows

    lines = [
        "# Vereinstermine, ursprünglich TablePress-Tabellen aus WordPress.",
        "# Neue Termine: einfach unter dem passenden Jahr ergänzen.",
        "",
    ]
    for year in sorted(years, reverse=True):
        lines.append(f'"{year}":')
        if not years[year]:
            lines[-1] += " []"
            continue
        for row in years[year]:
            lines.append(f"  - date: {yaml_str(row['date'])}")
            lines.append(f"    description: {yaml_str(row['description'])}")
        lines.append("")

    (DATA / "termine.yaml").write_text("\n".join(lines).rstrip() + "\n",
                                       encoding="utf-8")
    total = sum(len(v) for v in years.values())
    print(f"termine.yaml: {len(years)} Jahre, {total} Termine")


# ------------------------------------------------------------------- widget scraping


def live_home() -> str:
    req = urllib.request.Request(LIVE_HOME, headers={"User-Agent": USER_AGENT})
    with urllib.request.urlopen(req, timeout=60) as resp:
        return resp.read().decode("utf-8", errors="replace")


GALLERY_ITEM = re.compile(
    r"<figure class='gallery-item'>.*?"
    r"<a href='(?P<href>[^']+)'>.*?"
    r"(?:<figcaption[^>]*>(?P<caption>.*?)</figcaption>)?"
    r"</figure>",
    re.S,
)


def gallery_entries(fragment: str) -> list[tuple[str, str]]:
    out = []
    for m in GALLERY_ITEM.finditer(fragment):
        href = m.group("href")
        caption = html.unescape(re.sub(r"<[^>]+>", "", m.group("caption") or "")).strip()
        out.append((href, caption))
    return out


def image_ref(url: str) -> str:
    """Map an uploads URL to the committed path under assets/img/."""
    return "img/" + clean_media_name(url.rsplit("/", 1)[-1])


def build_widgets() -> None:
    page = live_home()

    # Each widget renders as <h4 class="widget-title">NAME</h4> followed by a WP
    # gallery. Keep the trailing </figure> inside the group, otherwise the last
    # entry of every gallery is silently dropped.
    members_html = re.search(
        r"Unsere Mitglieder</h4>(.*?</figure>)\s*</div>", page, re.S
    )
    memoriam_html = re.search(
        r"In Erinnerung</h4>(.*?</figure>)\s*</div>", page, re.S
    )
    if not members_html or not memoriam_html:
        raise SystemExit("could not locate the member/memoriam widgets on the live page")

    members = gallery_entries(members_html.group(1))
    memoriam = gallery_entries(memoriam_html.group(1))

    # The WordPress gallery widget shuffles its order on every request, so the
    # source order carries no meaning. Sort by name for a stable, readable list.
    members.sort(key=lambda entry: entry[1].casefold())

    lines = [
        "# Mitglieder-Galerie der Startseite.",
        "# Aus dem WordPress-Widget übernommen (Widgets sind nicht im XML-Export enthalten).",
        "",
    ]
    for url, name in members:
        lines.append(f"- name: {yaml_str(name)}")
        lines.append(f"  image: {yaml_str(image_ref(url))}")
    (DATA / "members.yaml").write_text("\n".join(lines) + "\n", encoding="utf-8")
    print(f"members.yaml: {len(members)} Mitglieder")

    lines = [
        "# \"In Erinnerung\" – ebenfalls aus einem WordPress-Widget übernommen.",
        "",
    ]
    for url, caption in memoriam:
        # Captions look like "Roland Groos ✝ 27.5.2020".
        name, _, died = caption.partition("✝")
        lines.append(f"- name: {yaml_str(name.strip())}")
        lines.append(f"  died: {yaml_str(died.strip())}")
        lines.append(f"  image: {yaml_str(image_ref(url))}")
    (DATA / "memoriam.yaml").write_text("\n".join(lines) + "\n", encoding="utf-8")
    print(f"memoriam.yaml: {len(memoriam)} Einträge")

    return [image_ref(u) for u, _ in members] + [image_ref(u) for u, _ in memoriam]


def main() -> int:
    DATA.mkdir(parents=True, exist_ok=True)
    build_termine()
    build_widgets()
    return 0


if __name__ == "__main__":
    raise SystemExit(main())