"""Generate data/*.yaml.
Two sources:
* ``data/termine.yaml`` comes from the six TablePress tables in the WXR export.
* ``data/members.yaml`` and ``data/memoriam.yaml`` come from the *live* homepage.
Those two blocks were WordPress widgets, and WXR does not export widgets at all,
so scraping the rendered page is the only way to recover them.
Run once during the migration. After that the YAML files are the source of truth.
"""
from __future__ import annotations
import html
import json
import re
import sys
import urllib.request
from pathlib import Path
from wxr import REPO_ROOT, clean_media_name, load_items
LIVE_HOME = "https://motorradclub-giebelwald.de/"
USER_AGENT = "mcg-website-migration/1.0 (+https://mcg.luxick.de)"
DATA = REPO_ROOT / "data"
def yaml_str(value: str) -> str:
"""Always double-quote; the cells contain colons, umlauts and inline HTML."""
return '"' + value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + '"'
# --------------------------------------------------------------------------- termine
def build_termine() -> None:
items = load_items()
tables = [i for i in items.values() if i.post_type == "tablepress_table"]
years: dict[int, list[dict[str, str]]] = {}
for table in tables:
match = re.search(r"(\d{4})", table.title)
if not match:
print(f"!! skipping table without a year in the title: {table.title}",
file=sys.stderr)
continue
year = int(match.group(1))
rows = []
for row in json.loads(table.content):
date = (row[0] if len(row) > 0 else "").strip()
desc = (row[1] if len(row) > 1 else "").strip()
if not date and not desc:
continue # TablePress spacer rows
rows.append({"date": date, "description": desc})
years[year] = rows
lines = [
"# Vereinstermine, ursprünglich TablePress-Tabellen aus WordPress.",
"# Neue Termine: einfach unter dem passenden Jahr ergänzen.",
"",
]
for year in sorted(years, reverse=True):
lines.append(f'"{year}":')
if not years[year]:
lines[-1] += " []"
continue
for row in years[year]:
lines.append(f" - date: {yaml_str(row['date'])}")
lines.append(f" description: {yaml_str(row['description'])}")
lines.append("")
(DATA / "termine.yaml").write_text("\n".join(lines).rstrip() + "\n",
encoding="utf-8")
total = sum(len(v) for v in years.values())
print(f"termine.yaml: {len(years)} Jahre, {total} Termine")
# ------------------------------------------------------------------- widget scraping
def live_home() -> str:
req = urllib.request.Request(LIVE_HOME, headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(req, timeout=60) as resp:
return resp.read().decode("utf-8", errors="replace")
GALLERY_ITEM = re.compile(
r"