From a58ed0e805e36e5154c40f1cf482e0df07e81f9e Mon Sep 17 00:00:00 2001 From: luxick Date: Tue, 28 Jul 2026 19:31:02 +0200 Subject: [PATCH] Remove migration tooling and documentation The import is verified and content/ is now the source of truth, so the one-shot WordPress importer under tools/ and MIGRATION.md are no longer needed. Also drops the remaining references to the old site from the README, config comments and data-file headers. Build is unchanged: 38 pages, 718 internal links, none broken. Co-Authored-By: Claude Opus 5 --- .gitignore | 6 +- MIGRATION.md | 113 ------- README.md | 35 +-- assets/css/main.css | 2 +- content/_index.md | 4 +- data/members.yaml | 2 +- data/memoriam.yaml | 2 +- data/termine.yaml | 4 +- hugo.toml | 4 +- layouts/_partials/func/featured-image.html | 4 +- tools/build_data.py | 169 ---------- tools/fetch_media.py | 109 ------- tools/html2md.py | 171 ----------- tools/verify_migration.py | 133 -------- tools/wp2hugo.py | 340 --------------------- tools/wxr.py | 126 -------- 16 files changed, 29 insertions(+), 1195 deletions(-) delete mode 100644 MIGRATION.md delete mode 100644 tools/build_data.py delete mode 100644 tools/fetch_media.py delete mode 100644 tools/html2md.py delete mode 100644 tools/verify_migration.py delete mode 100644 tools/wp2hugo.py delete mode 100644 tools/wxr.py diff --git a/.gitignore b/.gitignore index 3a8cf0e..391ef3a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,5 @@ /resources/_gen/ /.hugo_build.lock -# Migration scratch space: originals pulled off the old WordPress host. -# Only the files that get copied into content/, assets/ and static/ are committed. +# Local scratch space /.cache/ - -__pycache__/ -*.py[cod] diff --git a/MIGRATION.md b/MIGRATION.md deleted file mode 100644 index 90eac5a..0000000 --- a/MIGRATION.md +++ /dev/null @@ -1,113 +0,0 @@ -# WordPress → Hugo Migration - -Record of the one-time import from the WordPress export at -`wordpress-export/motorradclubgiebelwaldev.WordPress.2026-07-28.xml` -(WordPress 7.0.2, HitMag theme, TablePress, `widgets_on_pages`). - -## Result - -| | | -|---|---| -| Content files | 26 (9 pages + 17 posts) | -| Media placed | 127 / 165 attachments | -| Internal links checked | 718, 0 broken | -| Stale `wp-content/` URLs | 0 | - -Re-verify at any time with: - -```sh -hugo --gc --minify -python tools/verify_migration.py -``` - -## URL changes - -Old permalinks were **not** preserved (deliberate — no redirects were required). - -| Old | New | -|---|---| -| `/2026/01/13/29-o-fahrt-zum-50-jubilaum/` | `/aktuelles/29-o-fahrt-zum-50-jubilaeum/` | -| `/2025/01/03/termin/` | `/aktuelles/28-o-fahrt-2025/` | -| `/2022/06/09/o-fahrt-2019/` | `/aktuelles/25-o-fahrt-2022/` | -| `/40-jahre-mcg/` | `/archiv/40-jahre-mcg/` | -| `/familenausfahrt-2019/` | `/archiv/familienausfahrt-2019/` | -| `/bilder-familienausfahrt/` | `/archiv/bilder-familienausfahrt/` | -| `/familienfahrt-2022/` | `/archiv/familienfahrt-2022/` | -| `/vergangene-o-fahren/` | `/archiv/vergangene-o-fahrten/` | -| `/aktuelles/`, `/termine/`, `/downloads/`, `/impressum/` | unchanged | - -Two post slugs were misleading in WordPress and were renamed to match their own -titles: `o-fahrt-2019` → `25-o-fahrt-2022`, `termin` → `28-o-fahrt-2025`. -Umlaut slugs were transliterated (`jubilaum` → `jubilaeum`). - -## What was dropped, and why - -| WP id | Item | Reason | -|---|---|---| -| 3 | "Privacy Policy" | Stock English WordPress template, still a draft, references an old bare IP address. Not usable. The Impressum already carries a `Datenschutz` section, which was migrated verbatim. | -| 12 | "Home" | Body was the placeholder *"Dies ist nur eine Demo Seite"*. The real homepage came from the HitMag magazine template plus widgets. | -| 410 | untitled draft post | Empty. | - -## Things the export did not contain - -The WXR format carries neither media binaries nor widgets. Both were recovered -from the live site before it is retired: - -- **Media** — `tools/fetch_media.py` downloads all 165 uploads, caps them at - 2000 px on the long edge and re-encodes JPEG at q82. Cached in `.cache/` - (gitignored); only files actually used are committed. -- **Widgets** — the homepage *Unsere Mitglieder* grid (20 portraits) and the - sidebar *In Erinnerung* block were WordPress widgets. `tools/build_data.py` - scrapes them off the rendered homepage into `data/members.yaml` and - `data/memoriam.yaml`. - - Note: the WordPress gallery widget **shuffles member order on every request**, - so the original order carried no meaning. Members are now sorted by name. - -## Structural changes - -- **Termine** — the six TablePress tables (`Termine 2020`–`2026`) became - `data/termine.yaml`. WordPress only ever embedded table id 7 (*Termine 2026*) - on one page; the other five were orphaned. The `/termine/` page now shows the - newest year expanded with earlier years in collapsible sections. -- **Aktuelles** — was a page containing `[widgets_on_pages id="1"]`. It is now a - real Hugo section with an RSS feed. -- **Categories** — `uncategorized` and `aktuelles` were noise and were dropped. - `ofahrt` became the tag `O-Fahrt`. - -## Data defects found in the export - -- Post 81 (*Ehrungen von Mitgliedern 2018*) references `wp-image-83`, but - attachment 83 no longer exists — the media record was deleted while the upload - survived. `tools/wp2hugo.py` recovers such references by matching the `src` - filename instead. -- Almost every page ends with an empty `

` paragraph; these are stripped. -- WordPress wrote emphasis with the whitespace *inside* the markers - (`Samstag `), which is not valid CommonMark emphasis. The - converter moves the whitespace outside. - -## Unused media (38 files, downloaded but not committed) - -These exist in the WordPress media library but appear nowhere on the live site. -They are still in `.cache/uploads/` if any are wanted later. - -- **Superseded member portraits (2019)** — `Bene_2`, `Burckhardt_2`, `Christian_2`, - `Christoph_2`, `Christopher_2`, `HeikeF`, `HeikeR`, `Henning`, `JensB`, - `Katrin_2`, `Marcel_2`, `Marvin_2`, `Rainer`, `Steffi_und_Sabine`, `Sven_2`, - `Timo_1`, `Timo_2`, `Udo`, `Uwe`, `Gruppenbildanja`. Replaced by the 2022 - `*_web.jpg` set. -- **Former members** — `Jannik_web.jpg`, `Timo_web.jpg` (present in the library, - removed from the live member grid). -- **Old theme assets** — `title.jpg`, `cropped-title.jpg`, `bg.jpeg`, - `Banner_small_web.jpg` (uncropped), and six `cropped-Emblem_transparent*.png` - logo crops. The site now uses `assets/img/logo.png` and `assets/img/banner.jpg`. -- **Orphans** — `SZ-Artikel.jpg`, `Banner.jpg` (2023), `1-1.jpeg`, `6-scaled.jpg`, - `IMG_20190209_205905-3.jpg`, `jahresfahrt-2019.jpg`. - -## Open items - -- `content/_index.md` carries a **placeholder intro** marked with a `TODO`. There - was no homepage text in WordPress to migrate — please replace it. -- No separate Datenschutzerklärung page exists. The site is fully static with no - cookies, forms or third-party embeds, and the Impressum contains a Datenschutz - section. Worth a legal review before the production cutover. diff --git a/README.md b/README.md index 73e6c7c..a3ad33a 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,6 @@ Static website of the Motorradclub Giebelwald e.V., built with [Hugo](https://go - Development: - Production: -Migrated from WordPress in July 2026 — see [MIGRATION.md](MIGRATION.md). - ## Requirements Hugo **0.164.0**. The *non-extended* build is sufficient: the site uses plain CSS, @@ -46,7 +44,7 @@ it the first image in the folder is used. ### Add or change a Termin -Edit `data/termine.yaml`. Newest year is shown expanded, older years collapse: +Edit `data/termine.yaml`. The newest year is shown expanded, older years collapse: ```yaml "2027": @@ -74,6 +72,23 @@ Put the PDF in `static/dokumente/` and link it from `content/downloads.md`: | `{{< figure "bild.jpg" "Bildunterschrift" >}}` | Single image with caption | | `{{< pdf "datei.pdf" "Titel" >}}` | Download link with file size | +## Site structure + +``` +content/ + _index.md Startseite + aktuelles/ News section (RSS enabled) + archiv/ Rückblicke auf frühere Fahrten + termine.md uses layouts/termine.html + data/termine.yaml + downloads.md + impressum.md +data/ termine, members, memoriam +assets/ css/main.css, img/ (logo, banner, portraits) +static/dokumente/ PDFs +layouts/ baseof, home, list, page, termine + _partials/ _shortcodes/ _markup/ +``` + ## Deployment `.gitea/workflows/deploy.yml` builds with Hugo and rsyncs `public/` over SSH. @@ -99,17 +114,3 @@ Set these in the Gitea repository settings: rsync runs with `--delete`, so each `DEPLOY_PATH_*` must point at a directory owned solely by this site. The workflow refuses to deploy if the path is unset, too shallow, or if the build produced no `index.html`. - -## Migration tooling - -`tools/` holds the one-shot WordPress importer, kept so the import stays -reproducible and reviewable. It is not part of the build. - -```sh -python tools/fetch_media.py # download uploads from the live WP host -python tools/build_data.py # termine / members / memoriam -python tools/wp2hugo.py # content bundles (won't overwrite; --force to do so) -python tools/verify_migration.py # inventory + link checks (run after `hugo`) -``` - -Requires Python 3.11+ and Pillow. diff --git a/assets/css/main.css b/assets/css/main.css index 63dd219..fdc928d 100644 --- a/assets/css/main.css +++ b/assets/css/main.css @@ -3,7 +3,7 @@ cannot compile Sass. */ :root { - --ink: #404040; /* carried over from the old HitMag custom CSS */ + --ink: #404040; --ink-soft: #6b6b6b; --ink-faint: #8c8c8c; --accent: #9c1c1c; diff --git a/content/_index.md b/content/_index.md index f614803..8952a9c 100644 --- a/content/_index.md +++ b/content/_index.md @@ -2,9 +2,7 @@ title: "Motorradclub Giebelwald e.V." --- - + Willkommen beim Motorradclub Giebelwald e.V. — gegründet 1976 im Siegerland. diff --git a/data/members.yaml b/data/members.yaml index 74d5c92..932c41d 100644 --- a/data/members.yaml +++ b/data/members.yaml @@ -1,5 +1,5 @@ # Mitglieder-Galerie der Startseite. -# Aus dem WordPress-Widget übernommen (Widgets sind nicht im XML-Export enthalten). +# Bilder liegen in assets/img/ und werden automatisch quadratisch zugeschnitten. - name: "Benedikt" image: "img/benedikt-web.jpg" diff --git a/data/memoriam.yaml b/data/memoriam.yaml index b45cd90..0c552f1 100644 --- a/data/memoriam.yaml +++ b/data/memoriam.yaml @@ -1,4 +1,4 @@ -# "In Erinnerung" – ebenfalls aus einem WordPress-Widget übernommen. +# "In Erinnerung" – wird auf der Startseite unter den Mitgliedern angezeigt. - name: "Roland Groos" died: "27.5.2020" diff --git a/data/termine.yaml b/data/termine.yaml index 871fbe5..38fbd4c 100644 --- a/data/termine.yaml +++ b/data/termine.yaml @@ -1,5 +1,5 @@ -# Vereinstermine, ursprünglich TablePress-Tabellen aus WordPress. -# Neue Termine: einfach unter dem passenden Jahr ergänzen. +# Vereinstermine. Neue Termine einfach unter dem passenden Jahr ergänzen; +# das neueste Jahr wird ausgeklappt angezeigt, ältere Jahre eingeklappt. "2026": - date: "07.02.2026\n18:30" diff --git a/hugo.toml b/hugo.toml index e2ee013..ed7892e 100644 --- a/hugo.toml +++ b/hugo.toml @@ -18,8 +18,8 @@ buildFuture = false dateFormat = '2. January 2006' [markup.goldmark.renderer] - # The imported WordPress content contains hand-written HTML (tables, links with - # target attributes) that must survive rendering. + # Some pages contain hand-written HTML (tables, links with target attributes) + # that must survive rendering. unsafe = true [markup.tableOfContents] diff --git a/layouts/_partials/func/featured-image.html b/layouts/_partials/func/featured-image.html index 261423a..e780fa8 100644 --- a/layouts/_partials/func/featured-image.html +++ b/layouts/_partials/func/featured-image.html @@ -1,7 +1,7 @@ {{- /* Resolve a page's featured image to a Resource. - Looks at the `featured` front-matter key first (set by the WordPress import), - then falls back to the first image in the page bundle. + Looks at the `featured` front-matter key first, then falls back to the first + image in the page bundle. Returns the Resource, or nothing. */ -}} {{- $img := "" -}} diff --git a/tools/build_data.py b/tools/build_data.py deleted file mode 100644 index c352752..0000000 --- a/tools/build_data.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Generate data/*.yaml. - -Two sources: - -* ``data/termine.yaml`` comes from the six TablePress tables in the WXR export. -* ``data/members.yaml`` and ``data/memoriam.yaml`` come from the *live* homepage. - Those two blocks were WordPress widgets, and WXR does not export widgets at all, - so scraping the rendered page is the only way to recover them. - -Run once during the migration. After that the YAML files are the source of truth. -""" - -from __future__ import annotations - -import html -import json -import re -import sys -import urllib.request -from pathlib import Path - -from wxr import REPO_ROOT, clean_media_name, load_items - -LIVE_HOME = "https://motorradclub-giebelwald.de/" -USER_AGENT = "mcg-website-migration/1.0 (+https://mcg.luxick.de)" -DATA = REPO_ROOT / "data" - - -def yaml_str(value: str) -> str: - """Always double-quote; the cells contain colons, umlauts and inline HTML.""" - return '"' + value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + '"' - - -# --------------------------------------------------------------------------- termine - - -def build_termine() -> None: - items = load_items() - tables = [i for i in items.values() if i.post_type == "tablepress_table"] - - years: dict[int, list[dict[str, str]]] = {} - for table in tables: - match = re.search(r"(\d{4})", table.title) - if not match: - print(f"!! skipping table without a year in the title: {table.title}", - file=sys.stderr) - continue - year = int(match.group(1)) - - rows = [] - for row in json.loads(table.content): - date = (row[0] if len(row) > 0 else "").strip() - desc = (row[1] if len(row) > 1 else "").strip() - if not date and not desc: - continue # TablePress spacer rows - rows.append({"date": date, "description": desc}) - years[year] = rows - - lines = [ - "# Vereinstermine, ursprünglich TablePress-Tabellen aus WordPress.", - "# Neue Termine: einfach unter dem passenden Jahr ergänzen.", - "", - ] - for year in sorted(years, reverse=True): - lines.append(f'"{year}":') - if not years[year]: - lines[-1] += " []" - continue - for row in years[year]: - lines.append(f" - date: {yaml_str(row['date'])}") - lines.append(f" description: {yaml_str(row['description'])}") - lines.append("") - - (DATA / "termine.yaml").write_text("\n".join(lines).rstrip() + "\n", - encoding="utf-8") - total = sum(len(v) for v in years.values()) - print(f"termine.yaml: {len(years)} Jahre, {total} Termine") - - -# ------------------------------------------------------------------- widget scraping - - -def live_home() -> str: - req = urllib.request.Request(LIVE_HOME, headers={"User-Agent": USER_AGENT}) - with urllib.request.urlopen(req, timeout=60) as resp: - return resp.read().decode("utf-8", errors="replace") - - -GALLERY_ITEM = re.compile( - r"
.*?" - r".*?" - r"(?:]*>(?P.*?))?" - r"
", - re.S, -) - - -def gallery_entries(fragment: str) -> list[tuple[str, str]]: - out = [] - for m in GALLERY_ITEM.finditer(fragment): - href = m.group("href") - caption = html.unescape(re.sub(r"<[^>]+>", "", m.group("caption") or "")).strip() - out.append((href, caption)) - return out - - -def image_ref(url: str) -> str: - """Map an uploads URL to the committed path under assets/img/.""" - return "img/" + clean_media_name(url.rsplit("/", 1)[-1]) - - -def build_widgets() -> None: - page = live_home() - - # Each widget renders as

NAME

followed by a WP - # gallery. Keep the trailing inside the group, otherwise the last - # entry of every gallery is silently dropped. - members_html = re.search( - r"Unsere Mitglieder(.*?)\s*", page, re.S - ) - memoriam_html = re.search( - r"In Erinnerung(.*?)\s*", page, re.S - ) - if not members_html or not memoriam_html: - raise SystemExit("could not locate the member/memoriam widgets on the live page") - - members = gallery_entries(members_html.group(1)) - memoriam = gallery_entries(memoriam_html.group(1)) - - # The WordPress gallery widget shuffles its order on every request, so the - # source order carries no meaning. Sort by name for a stable, readable list. - members.sort(key=lambda entry: entry[1].casefold()) - - lines = [ - "# Mitglieder-Galerie der Startseite.", - "# Aus dem WordPress-Widget übernommen (Widgets sind nicht im XML-Export enthalten).", - "", - ] - for url, name in members: - lines.append(f"- name: {yaml_str(name)}") - lines.append(f" image: {yaml_str(image_ref(url))}") - (DATA / "members.yaml").write_text("\n".join(lines) + "\n", encoding="utf-8") - print(f"members.yaml: {len(members)} Mitglieder") - - lines = [ - "# \"In Erinnerung\" – ebenfalls aus einem WordPress-Widget übernommen.", - "", - ] - for url, caption in memoriam: - # Captions look like "Roland Groos ✝ 27.5.2020". - name, _, died = caption.partition("✝") - lines.append(f"- name: {yaml_str(name.strip())}") - lines.append(f" died: {yaml_str(died.strip())}") - lines.append(f" image: {yaml_str(image_ref(url))}") - (DATA / "memoriam.yaml").write_text("\n".join(lines) + "\n", encoding="utf-8") - print(f"memoriam.yaml: {len(memoriam)} Einträge") - - return [image_ref(u) for u, _ in members] + [image_ref(u) for u, _ in memoriam] - - -def main() -> int: - DATA.mkdir(parents=True, exist_ok=True) - build_termine() - build_widgets() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/fetch_media.py b/tools/fetch_media.py deleted file mode 100644 index 1389ef0..0000000 --- a/tools/fetch_media.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Download every attachment referenced by the WXR export into .cache/uploads/. - -The export only carries URLs, not binaries, so the files have to come off the live -WordPress host before it is retired. Images are capped at MAX_EDGE px on the long -side and re-encoded; PDFs are stored verbatim. - -Re-running is cheap: anything already in the cache is skipped. -""" - -from __future__ import annotations - -import sys -import urllib.error -import urllib.parse -import urllib.request -from io import BytesIO - -from PIL import Image - -from wxr import CACHE, attachments, cache_path, load_items - -MAX_EDGE = 2000 -JPEG_QUALITY = 82 -USER_AGENT = "mcg-website-migration/1.0 (+https://mcg.luxick.de)" - - -def fetch(url: str) -> bytes: - # A few uploads have umlauts in the filename (e.g. Hövelhof.jpg); the HTTP - # request line must be ASCII, so percent-encode the path before sending. - parts = urllib.parse.urlsplit(url) - safe = urllib.parse.urlunsplit( - parts._replace(path=urllib.parse.quote(parts.path, safe="/%")) - ) - req = urllib.request.Request(safe, headers={"User-Agent": USER_AGENT}) - with urllib.request.urlopen(req, timeout=60) as resp: - return resp.read() - - -def store_image(raw: bytes, dest) -> None: - img = Image.open(BytesIO(raw)) - img.load() - - # Honour the EXIF orientation now; the tag is dropped on save. - try: - from PIL import ImageOps - - img = ImageOps.exif_transpose(img) - except Exception: - pass - - if max(img.size) > MAX_EDGE: - img.thumbnail((MAX_EDGE, MAX_EDGE), Image.LANCZOS) - - suffix = dest.suffix.lower() - if suffix in (".jpg", ".jpeg"): - img.convert("RGB").save(dest, "JPEG", quality=JPEG_QUALITY, optimize=True, - progressive=True) - elif suffix == ".png": - img.save(dest, "PNG", optimize=True) - else: - img.save(dest) - - -def main() -> int: - items = load_items() - atts = attachments(items) - - skipped = downloaded = failed = 0 - for post_id, item in sorted(atts.items(), key=lambda kv: int(kv[0])): - url = item.attachment_url - if not url: - print(f"!! {post_id}: no attachment_url", file=sys.stderr) - failed += 1 - continue - - dest = cache_path(url) - if dest.exists() and dest.stat().st_size > 0: - skipped += 1 - continue - - dest.parent.mkdir(parents=True, exist_ok=True) - try: - raw = fetch(url) - except Exception as exc: # one bad upload must not abort the whole run - print(f"!! {post_id}: {url} -> {exc!r}", file=sys.stderr) - failed += 1 - continue - - try: - if dest.suffix.lower() == ".pdf": - dest.write_bytes(raw) - else: - store_image(raw, dest) - except Exception as exc: # corrupt/unsupported source, keep the original bytes - print(f"!! {post_id}: could not process {url} ({exc}); storing raw", - file=sys.stderr) - dest.write_bytes(raw) - - downloaded += 1 - print(f" {dest.relative_to(CACHE)} ({len(raw) // 1024} KiB -> " - f"{dest.stat().st_size // 1024} KiB)") - - print(f"\ndownloaded={downloaded} cached={skipped} failed={failed} " - f"total={len(atts)}") - return 1 if failed else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/html2md.py b/tools/html2md.py deleted file mode 100644 index cee8dab..0000000 --- a/tools/html2md.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Minimal HTML -> Markdown converter for the imported WordPress content. - -Deliberately narrow: it only handles the tags that actually occur in this export -(paragraphs, headings, lists, emphasis, links, images, breaks, rules). Anything -unrecognised is passed through as raw HTML, which Goldmark renders because -``markup.goldmark.renderer.unsafe`` is on. -""" - -from __future__ import annotations - -import re -from html.parser import HTMLParser - -INLINE = {"strong", "b", "em", "i", "a", "br", "span", "code", "u", "s"} -BLOCK = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "hr", - "blockquote", "figure", "figcaption", "div"} - -ESCAPE = re.compile(r"([\\`*_\[\]])") - - -class Converter(HTMLParser): - def __init__(self, resolve_image=None): - super().__init__(convert_charrefs=True) - self.out: list[tuple[str, str]] = [] - self.resolve_image = resolve_image or (lambda src, cls: src) - self._stack: list[str] = [] - self._list: list[str] = [] # 'ul' / 'ol' nesting - self._li_index: list[int] = [] - self._link: str | None = None - self._buf: list[str] = [] - - # -- helpers --------------------------------------------------------- - def _emit(self, text: str) -> None: - self._buf.append(text) - - def _flush_block(self, prefix: str = "", kind: str = "block") -> None: - text = "".join(self._buf).strip() - self._buf = [] - if text: - self.out.append((kind, prefix + text)) - - def _open_emphasis(self, marker: str) -> None: - # Emitted lazily: WordPress often writes " 57258 ...", - # and "** 57258**" is not valid emphasis in CommonMark. The marker is - # placed after any leading whitespace instead (see handle_data). - self._pending_emphasis = marker - - def _close_emphasis(self, marker: str) -> None: - if getattr(self, "_pending_emphasis", None) == marker: - self._pending_emphasis = None # empty - return - # Likewise move trailing whitespace out: "**Samstag **" -> "**Samstag** ". - trailing = "" - while self._buf and self._buf[-1] and self._buf[-1][-1] in " \t\n": - trailing = self._buf[-1][-1] + trailing - self._buf[-1] = self._buf[-1][:-1] - if not self._buf[-1]: - self._buf.pop() - if not self._buf: - return - self._emit(marker) - if trailing: - self._emit(trailing) - - # -- tags ------------------------------------------------------------ - def handle_starttag(self, tag, attrs): - a = dict(attrs) - if tag == "p": - self._flush_block() - elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"): - self._flush_block() - self._pending_prefix = "#" * int(tag[1]) + " " - elif tag in ("ul", "ol"): - self._flush_block() - self._list.append(tag) - self._li_index.append(0) - elif tag == "li": - self._flush_block() - if self._list: - self._li_index[-1] += 1 - indent = " " * (len(self._list) - 1) - marker = ("- " if self._list[-1] == "ul" - else f"{self._li_index[-1]}. ") - self._pending_prefix = indent + marker - elif tag == "hr": - self._flush_block() - self.out.append(("block", "---")) - elif tag == "br": - self._emit(" \n") - elif tag in ("strong", "b"): - self._open_emphasis("**") - elif tag in ("em", "i"): - self._open_emphasis("*") - elif tag == "a": - self._link = a.get("href", "") - self._emit("[") - elif tag == "img": - src = self.resolve_image(a.get("src", ""), a.get("class", "")) - alt = (a.get("alt") or "").replace("]", "") - self._emit(f"![{alt}]({src})") - self._stack.append(tag) - - def handle_endtag(self, tag): - while self._stack and self._stack.pop() != tag: - pass - if tag == "p": - self._flush_block() - elif tag in ("h1", "h2", "h3", "h4", "h5", "h6", "li"): - prefix = getattr(self, "_pending_prefix", "") - self._pending_prefix = "" - self._flush_block(prefix, kind="li" if tag == "li" else "block") - elif tag in ("ul", "ol"): - self._flush_block() - if self._list: - self._list.pop() - self._li_index.pop() - elif tag in ("strong", "b"): - self._close_emphasis("**") - elif tag in ("em", "i"): - self._close_emphasis("*") - elif tag == "a": - self._emit(f"]({self._link or ''})") - self._link = None - - def handle_data(self, data): - if not data: - return - # Collapse WordPress' aggressive whitespace, but keep explicit breaks. - text = data.replace(" ", " ") - text = re.sub(r"[ \t]*\n[ \t]*", "\n", text) - - pending = getattr(self, "_pending_emphasis", None) - if pending: - if not text.strip(): - return # whitespace only; keep waiting for real content - lead = text[:len(text) - len(text.lstrip())] - if lead: - self._emit(lead) - text = text.lstrip() - self._emit(pending) - self._pending_emphasis = None - elif not self._buf and not text.strip(): - return - - self._emit(ESCAPE.sub(r"\\\1", text) if self._link is None - else text.replace("[", r"\[").replace("]", r"\]")) - - def result(self) -> str: - self._flush_block(getattr(self, "_pending_prefix", "")) - blocks = [(kind, text.strip()) for kind, text in self.out if text.strip()] - - out = "" - for index, (kind, text) in enumerate(blocks): - if index == 0: - out = text - continue - # Keep consecutive list items tight; a blank line between them makes - # Goldmark render a "loose" list, wrapping every item in

. - sep = "\n" if kind == "li" and blocks[index - 1][0] == "li" else "\n\n" - out += sep + text - return out - - -def html_to_markdown(html_text: str, resolve_image=None) -> str: - conv = Converter(resolve_image=resolve_image) - conv.feed(html_text) - conv.close() - text = conv.result() - # An escaped em-dash entity or stray double-escape looks worse than the source. - text = text.replace("\\_\\_", "__") - return re.sub(r"\n{3,}", "\n\n", text).strip() diff --git a/tools/verify_migration.py b/tools/verify_migration.py deleted file mode 100644 index 2d95dc7..0000000 --- a/tools/verify_migration.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Post-migration checks. Run after `hugo --gc --minify`. - -1. Inventory - every published WordPress page/post has a content file. -2. Media - every attachment is either placed in the site or explicitly - accounted for as an unused media-library leftover. -3. Links - no internal href/src in public/ points at a missing file, and no - absolute wp-content URL survived the import. - -Exits non-zero if any check fails. -""" - -from __future__ import annotations - -import re -import sys -from pathlib import Path -from urllib.parse import unquote, urlsplit - -from wp2hugo import PAGE_MAP, SKIP, SLUG_FIXES -from wxr import REPO_ROOT, clean_media_name, load_items, slugify - -PUBLIC = REPO_ROOT / "public" -CONTENT = REPO_ROOT / "content" - -failures: list[str] = [] -notes: list[str] = [] - - -def check_inventory(items) -> None: - expected = 0 - for item in items.values(): - if item.post_id in SKIP or item.status != "publish": - continue - if item.post_type == "page": - mapping = PAGE_MAP.get(item.post_id) - if not mapping: - failures.append(f"page {item.post_id} ({item.slug}) is unmapped") - continue - path = CONTENT / mapping[0] - elif item.post_type == "post": - path = CONTENT / "aktuelles" / SLUG_FIXES.get( - item.slug, slugify(item.slug)) / "index.md" - else: - continue - expected += 1 - if not path.exists(): - failures.append(f"missing content file for {item.post_type} " - f"{item.post_id} ({item.slug}): {path}") - notes.append(f"inventory: {expected} published pages/posts accounted for") - - for post_id, reason in SKIP.items(): - notes.append(f" skipped {post_id}: {reason}") - - -def check_media(items) -> None: - placed = {p.name for p in CONTENT.rglob("*") if p.is_file() - and p.suffix.lower() != ".md"} - placed |= {p.name for p in (REPO_ROOT / "assets" / "img").rglob("*") - if p.is_file()} - placed |= {p.name for p in (REPO_ROOT / "static").rglob("*") if p.is_file()} - # assets/img/logo.png and banner.jpg are renamed copies. - placed |= {"Emblem_transparent.png", "cropped-Banner_small_web.jpg"} - - unused = [] - for item in items.values(): - if item.post_type != "attachment" or not item.attachment_url: - continue - original = item.attachment_url.rsplit("/", 1)[-1] - if original in placed or clean_media_name(original) in placed: - continue - unused.append(item.attachment_url) - - total = sum(1 for i in items.values() if i.post_type == "attachment") - notes.append(f"media: {total - len(unused)}/{total} attachments placed") - if unused: - notes.append(f" {len(unused)} media-library leftovers not shown on the " - f"live site (see MIGRATION.md)") - - -HREF = re.compile(r'(?:href|src)=["\']?([^"\'\s>]+)', re.I) - - -def check_links() -> None: - if not PUBLIC.exists(): - failures.append("public/ does not exist - run `hugo` first") - return - - checked = broken = 0 - for html in PUBLIC.rglob("*.html"): - text = html.read_text(encoding="utf-8", errors="replace") - - for stale in re.findall(r'https?://motorradclub-giebelwald\.de/wp-content/\S*', - text): - failures.append(f"{html.relative_to(PUBLIC)}: stale WordPress URL {stale}") - - for raw in HREF.findall(text): - url = urlsplit(raw) - if url.scheme or url.netloc or not url.path: - continue # external, mailto:, protocol-relative - if raw.startswith("#"): - continue - checked += 1 - path = unquote(url.path) - target = (PUBLIC / path.lstrip("/") if path.startswith("/") - else html.parent / path) - if target.is_dir(): - target = target / "index.html" - if not target.exists(): - broken += 1 - failures.append(f"{html.relative_to(PUBLIC)}: broken link -> {raw}") - - notes.append(f"links: {checked} internal references checked, {broken} broken") - - -def main() -> int: - items = load_items() - check_inventory(items) - check_media(items) - check_links() - - for note in notes: - print(note) - if failures: - print(f"\n{len(failures)} FAILURE(S):", file=sys.stderr) - for f in failures: - print(f" - {f}", file=sys.stderr) - return 1 - print("\nall checks passed") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/wp2hugo.py b/tools/wp2hugo.py deleted file mode 100644 index aef4442..0000000 --- a/tools/wp2hugo.py +++ /dev/null @@ -1,340 +0,0 @@ -"""One-shot WordPress WXR -> Hugo importer. - -Run once during the migration: - - python tools/fetch_media.py # pull the binaries off the live host - python tools/build_data.py # termine / members / memoriam - python tools/wp2hugo.py # content bundles + assets - -Afterwards the generated Markdown in content/ is the source of truth and this -script is kept only so the import is reproducible and reviewable. -""" - -from __future__ import annotations - -import json -import re -import shutil -import sys -from dataclasses import dataclass, field -from pathlib import Path - -from html2md import html_to_markdown -from wxr import (REPO_ROOT, cache_path, clean_media_name, load_items, slugify) - -CONTENT = REPO_ROOT / "content" -ASSETS_IMG = REPO_ROOT / "assets" / "img" -STATIC_PDF = REPO_ROOT / "static" / "dokumente" - -# WordPress page id -> destination. Anything not listed is skipped. -PAGE_MAP = { - "13": ("aktuelles/_index.md", None), - "72": ("termine.md", "termine"), - "128": ("downloads.md", None), - "138": ("impressum.md", None), - "98": ("archiv/vergangene-o-fahrten.md", None), - "194": ("archiv/40-jahre-mcg/index.md", None), - "217": ("archiv/familienausfahrt-2019/index.md", None), - "287": ("archiv/bilder-familienausfahrt/index.md", None), - "473": ("archiv/familienfahrt-2022/index.md", None), -} - -# Deliberately dropped, with the reason recorded for the migration report. -SKIP = { - "3": "stock WordPress privacy-policy draft (English, references an old IP)", - "12": "WP home page body was the placeholder 'Dies ist nur eine Demo Seite'", - "410": "empty draft post", -} - -# Site chrome pulled out of the media library into assets/img/. -SITE_IMAGES = { - "Emblem_transparent.png": "logo.png", - "cropped-Banner_small_web.jpg": "banner.jpg", -} - -SLUG_FIXES = { - "29-o-fahrt-zum-50-jubilaum": "29-o-fahrt-zum-50-jubilaeum", - "regelmasger-stammtisch": "regelmaessiger-stammtisch", - "vorankundigung-mcg-wochenende-2024": "vorankuendigung-mcg-wochenende-2024", - "o-fahrt-2019": "25-o-fahrt-2022", # title says 25. O-Fahrt 2022; slug was stale - "termin": "28-o-fahrt-2025", # title says 28. O-Fahrt 2025 -} - - -@dataclass -class Target: - item: object - path: Path - bundle: Path | None - layout: str | None = None - images: dict[str, str] = field(default_factory=dict) # attachment id -> filename - - -# --------------------------------------------------------------------- helpers - - -def yaml_quote(value: str) -> str: - return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' - - -def iso(date_gmt: str) -> str: - if not date_gmt or date_gmt.startswith("0000"): - return "" - return date_gmt.replace(" ", "T") + "Z" - - -def referenced_ids(item, by_filename: dict[str, str] | None = None) -> list[str]: - """Every attachment id an item points at, in document order. - - Some posts carry stale ``wp-image-NNN`` ids whose attachment record was - deleted from WordPress even though the upload itself survived (post 81 points - at id 83, which no longer exists). When ``by_filename`` is supplied, such - references are recovered from the ``src`` URL instead. - """ - ids: list[str] = [] - - def add(value): - if value and value not in ids: - ids.append(value) - - thumb = item.meta.get("_thumbnail_id") - add(thumb) - for m in re.finditer(r']*>', item.content): - tag = m.group(0) - cls = re.search(r'wp-image-(\d+)', tag) - if cls: - add(cls.group(1)) - if by_filename is not None: - src = re.search(r'src="([^"]+)"', tag) - if src: - add(by_filename.get(clean_media_name(src.group(1).rsplit("/", 1)[-1]))) - for m in re.finditer(r'', item.content, re.S): - try: - for gid in json.loads(m.group(1)).get("ids", []): - add(str(gid)) - except json.JSONDecodeError: - pass - for m in re.finditer(r'', item.content, re.S): - try: - add(str(json.loads(m.group(1)).get("id", ""))) - except json.JSONDecodeError: - pass - return [i for i in ids if i] - - -# ------------------------------------------------------------------ conversion - - -def convert_body(item, target: Target, attachments: dict, pdf_names: dict) -> str: - body = item.content - - # Galleries: replace the whole block with a shortcode listing bundle files. - def gallery_repl(match: re.Match) -> str: - try: - ids = [str(i) for i in json.loads(match.group(1)).get("ids", [])] - except json.JSONDecodeError: - return "" - files = [target.images[i] for i in ids if i in target.images] - if not files: - return "" - args = " ".join(f'"{f}"' for f in files) - return f"\n\n{{{{< gallery {args} >}}}}\n\n" - - body = re.sub( - r'.*?', - gallery_repl, body, flags=re.S) - - # File blocks: PDFs live in static/dokumente/, linked via the pdf shortcode. - def file_repl(match: re.Match) -> str: - block = match.group(0) - try: - attrs = json.loads(match.group(1)) - except json.JSONDecodeError: - return "" - att_id = str(attrs.get("id", "")) - href = attrs.get("href", "") - name = pdf_names.get(att_id) or clean_media_name(href.rsplit("/", 1)[-1]) - label = re.search(r'>([^<]+)', block) - label = label.group(1).strip() if label else name - return f'\n\n{{{{< pdf "{name}" "{label}" >}}}}\n\n' - - body = re.sub( - r'.*?', - file_repl, body, flags=re.S) - - # Shortcodes that have no Hugo equivalent. - body = re.sub(r'.*?', "", - body, flags=re.S) - body = re.sub(r'\[table id=\d+\s*/?\]', "", body) - body = re.sub(r'\[widgets_on_pages[^\]]*\]', "", body) - - # Remaining Gutenberg comments carry no information once converted. - body = re.sub(r'', "", body) - - # WordPress' editor leaves these around every translated paragraph. - body = re.sub(r'\s*', "", body) - body = re.sub(r'', "", body) - - def resolve_image(src: str, cls: str) -> str: - m = re.search(r'wp-image-(\d+)', cls or "") - if m and m.group(1) in target.images: - return target.images[m.group(1)] - # Fall back to matching on the original upload filename. - base = clean_media_name(src.rsplit("/", 1)[-1]) - for name in target.images.values(): - if name == base: - return name - return src - - return html_to_markdown(body, resolve_image=resolve_image) - - -def front_matter(item, target: Target, extra_tags: list[str]) -> str: - lines = ["---", f"title: {yaml_quote(item.title)}"] - if iso(item.date_gmt): - lines.append(f"date: {iso(item.date_gmt)}") - if iso(item.modified_gmt) and item.modified_gmt != item.date_gmt: - lines.append(f"lastmod: {iso(item.modified_gmt)}") - if extra_tags: - lines.append("tags: [" + ", ".join(yaml_quote(t) for t in extra_tags) + "]") - thumb = item.meta.get("_thumbnail_id") - if thumb and thumb in target.images: - lines.append(f"featured: {yaml_quote(target.images[thumb])}") - if target.layout: - lines.append(f"layout: {yaml_quote(target.layout)}") - lines.append("---") - return "\n".join(lines) - - -# ------------------------------------------------------------------------ main - - -def main() -> int: - items = load_items() - attachments = {k: v for k, v in items.items() if v.post_type == "attachment"} - - # Cleaned upload filename -> attachment id, for recovering stale wp-image ids. - by_filename: dict[str, str] = {} - for att_id, att in attachments.items(): - if att.attachment_url: - by_filename.setdefault( - clean_media_name(att.attachment_url.rsplit("/", 1)[-1]), att_id) - - targets: list[Target] = [] - - for item in items.values(): - if item.post_id in SKIP or item.status != "publish": - continue - - if item.post_type == "page": - mapping = PAGE_MAP.get(item.post_id) - if not mapping: - print(f"!! unmapped page {item.post_id} ({item.slug})", file=sys.stderr) - continue - rel, layout = mapping - path = CONTENT / rel - elif item.post_type == "post": - slug = SLUG_FIXES.get(item.slug, slugify(item.slug)) - path = CONTENT / "aktuelles" / slug / "index.md" - layout = None - else: - continue - - bundle = path.parent if path.name == "index.md" else None - targets.append(Target(item=item, path=path, bundle=bundle, layout=layout)) - - # ---- assign media ------------------------------------------------- - used: set[str] = set() - pdf_names: dict[str, str] = {} - - for target in targets: - for att_id in referenced_ids(target.item, by_filename): - att = attachments.get(att_id) - if not att or not att.attachment_url: - continue - src = cache_path(att.attachment_url) - if not src.exists(): - print(f"!! missing cached media for {att_id}: {att.attachment_url}", - file=sys.stderr) - continue - name = clean_media_name(att.attachment_url.rsplit("/", 1)[-1]) - - if src.suffix.lower() == ".pdf": - STATIC_PDF.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, STATIC_PDF / name) - pdf_names[att_id] = name - target.images[att_id] = name - else: - if target.bundle is None: - # Non-bundle page (termine.md, impressum.md, ...) -> shared assets. - ASSETS_IMG.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, ASSETS_IMG / name) - target.images[att_id] = f"/img/{name}" - else: - target.bundle.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, target.bundle / name) - target.images[att_id] = name - used.add(att_id) - - # Any PDF in the library that no page links to is still worth keeping around - # for the Downloads page; copy them all so nothing is lost. - for att_id, att in attachments.items(): - if not att.attachment_url.lower().endswith(".pdf"): - continue - src = cache_path(att.attachment_url) - if src.exists(): - STATIC_PDF.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, STATIC_PDF / clean_media_name( - att.attachment_url.rsplit("/", 1)[-1])) - used.add(att_id) - - # Site chrome + widget images (members / memoriam) into assets/img/. - ASSETS_IMG.mkdir(parents=True, exist_ok=True) - widget_images = set() - for yaml_file in ("members.yaml", "memoriam.yaml"): - text = (REPO_ROOT / "data" / yaml_file).read_text(encoding="utf-8") - widget_images.update(re.findall(r'image:\s*"img/([^"]+)"', text)) - - for att_id, att in attachments.items(): - original = att.attachment_url.rsplit("/", 1)[-1] - name = clean_media_name(original) - src = cache_path(att.attachment_url) - if not src.exists(): - continue - if original in SITE_IMAGES: - shutil.copy2(src, ASSETS_IMG / SITE_IMAGES[original]) - used.add(att_id) - elif name in widget_images: - shutil.copy2(src, ASSETS_IMG / name) - used.add(att_id) - - # ---- write content ------------------------------------------------ - # content/ is the source of truth once imported, so an accidental re-run must - # not throw away hand edits. Pass --force to regenerate anyway. - force = "--force" in sys.argv - written = kept = 0 - for target in targets: - item = target.item - target.path.parent.mkdir(parents=True, exist_ok=True) - if target.path.exists() and not force: - kept += 1 - continue - tags = ["O-Fahrt"] if any(nice == "ofahrt" for _, nice in item.categories) else [] - body = convert_body(item, target, attachments, pdf_names) - target.path.write_text( - front_matter(item, target, tags) + "\n\n" + body + "\n", - encoding="utf-8") - written += 1 - - print(f"wrote {written} content files" - + (f", kept {kept} existing (use --force to overwrite)" if kept else "")) - print(f"media used: {len(used)}/{len(attachments)} attachments") - unused = sorted(set(attachments) - used, key=int) - print(f"unreferenced attachments: {len(unused)}") - for att_id in unused: - print(f" - {attachments[att_id].attachment_url}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/wxr.py b/tools/wxr.py deleted file mode 100644 index c4615b5..0000000 --- a/tools/wxr.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Shared helpers for reading the WordPress WXR export.""" - -from __future__ import annotations - -import re -import unicodedata -import xml.etree.ElementTree as ET -from dataclasses import dataclass, field -from pathlib import Path - -NS = { - "wp": "http://wordpress.org/export/1.2/", - "content": "http://purl.org/rss/1.0/modules/content/", - "excerpt": "http://wordpress.org/export/1.2/excerpt/", - "dc": "http://purl.org/dc/elements/1.1/", -} - -REPO_ROOT = Path(__file__).resolve().parent.parent -EXPORT = REPO_ROOT / "wordpress-export" / "motorradclubgiebelwaldev.WordPress.2026-07-28.xml" -CACHE = REPO_ROOT / ".cache" / "uploads" - -# German transliteration first, so "Jubiläum" becomes "jubilaeum" and not "jubilaum". -UMLAUTS = { - "ä": "ae", "ö": "oe", "ü": "ue", "ß": "ss", - "Ä": "Ae", "Ö": "Oe", "Ü": "Ue", -} - - -def slugify(value: str, *, keep_dots: bool = False) -> str: - for src, dst in UMLAUTS.items(): - value = value.replace(src, dst) - value = unicodedata.normalize("NFKD", value) - value = "".join(c for c in value if not unicodedata.combining(c)) - allowed = r"[^a-z0-9._-]+" if keep_dots else r"[^a-z0-9-]+" - value = re.sub(allowed, "-", value.lower()) - return re.sub(r"-{2,}", "-", value).strip("-") - - -def slugify_filename(name: str) -> str: - stem, _, ext = name.rpartition(".") - return f"{slugify(stem, keep_dots=False)}.{ext.lower()}" - - -def clean_media_name(name: str) -> str: - """Strip the bookkeeping suffixes WordPress bakes into upload filenames. - - ``Harry_2-e1654849333614.jpg`` -> ``harry-2.jpg`` - ``IMG_20220527_121440-small-scaled.jpg`` -> ``img-20220527-121440-small.jpg`` - """ - stem, _, ext = slugify_filename(name).rpartition(".") - stem = re.sub(r"-e\d{9,}$", "", stem) # post-crop revision id - stem = re.sub(r"-\d+x\d+$", "", stem) # generated size variant - stem = re.sub(r"-scaled$", "", stem) # WP "big image" downscale - return f"{stem or 'bild'}.{ext}" - - -def text(node: ET.Element | None, path: str, default: str = "") -> str: - if node is None: - return default - found = node.find(path, NS) - if found is None or found.text is None: - return default - return found.text - - -@dataclass -class Item: - element: ET.Element - post_id: str - post_type: str - status: str - title: str - slug: str - link: str - parent: str - date_gmt: str - modified_gmt: str - content: str - excerpt: str - attachment_url: str - meta: dict[str, str] = field(default_factory=dict) - categories: list[tuple[str, str]] = field(default_factory=list) - - -def load_items() -> dict[str, Item]: - channel = ET.parse(EXPORT).getroot().find("channel") - items: dict[str, Item] = {} - for el in channel.findall("item"): - meta = {} - for m in el.findall("wp:postmeta", NS): - key = text(m, "wp:meta_key") - meta[key] = text(m, "wp:meta_value") - cats = [ - (c.get("domain", ""), c.get("nicename", "")) - for c in el.findall("category") - ] - item = Item( - element=el, - post_id=text(el, "wp:post_id"), - post_type=text(el, "wp:post_type"), - status=text(el, "wp:status"), - title=(el.findtext("title") or "").strip(), - slug=text(el, "wp:post_name"), - link=(el.findtext("link") or "").strip(), - parent=text(el, "wp:post_parent"), - date_gmt=text(el, "wp:post_date_gmt"), - modified_gmt=text(el, "wp:post_modified_gmt"), - content=text(el, "content:encoded"), - excerpt=text(el, "excerpt:encoded"), - attachment_url=text(el, "wp:attachment_url"), - meta=meta, - categories=cats, - ) - items[item.post_id] = item - return items - - -def attachments(items: dict[str, Item]) -> dict[str, Item]: - return {k: v for k, v in items.items() if v.post_type == "attachment"} - - -def cache_path(url: str) -> Path: - """Local cache location mirroring the wp-content/uploads/YYYY/MM layout.""" - rel = url.split("/wp-content/uploads/", 1)[-1] - parts = rel.split("/") - return CACHE.joinpath(*parts[:-1], slugify_filename(parts[-1]))