luxick
bef5182545
Replaces the WordPress 7.0.2 / HitMag site with a static Hugo build,
deployed by Gitea Actions over rsync/SSH.
Content: 26 files (9 pages + 17 posts) as Markdown page bundles, plus
127 of 165 media attachments. The remaining 38 are media-library
leftovers that appear nowhere on the live site; they are listed in
MIGRATION.md.
The WXR export contains neither media binaries nor widgets, so both were
recovered from the live host before it is retired:
- tools/fetch_media.py downloads all uploads, capping them at 2000px
- tools/build_data.py scrapes the "Unsere Mitglieder" and
"In Erinnerung" widgets into data/*.yaml
TablePress' six Termine tables became data/termine.yaml, rendered with
the newest year expanded and earlier years collapsed. Old permalinks are
not preserved (clean slugs, no redirects, as agreed).
Custom layouts, no third-party theme. Plain CSS, since the pinned Hugo
0.164.0 is the non-extended build and cannot compile Sass.
Verified: clean build with zero warnings, 718 internal links checked and
none broken, no surviving wp-content URLs.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
341 lines
13 KiB
Python
341 lines
13 KiB
Python
"""One-shot WordPress WXR -> Hugo importer.
|
|
|
|
Run once during the migration:
|
|
|
|
python tools/fetch_media.py # pull the binaries off the live host
|
|
python tools/build_data.py # termine / members / memoriam
|
|
python tools/wp2hugo.py # content bundles + assets
|
|
|
|
Afterwards the generated Markdown in content/ is the source of truth and this
|
|
script is kept only so the import is reproducible and reviewable.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import shutil
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
from html2md import html_to_markdown
|
|
from wxr import (REPO_ROOT, cache_path, clean_media_name, load_items, slugify)
|
|
|
|
CONTENT = REPO_ROOT / "content"
|
|
ASSETS_IMG = REPO_ROOT / "assets" / "img"
|
|
STATIC_PDF = REPO_ROOT / "static" / "dokumente"
|
|
|
|
# WordPress page id -> destination. Anything not listed is skipped.
|
|
PAGE_MAP = {
|
|
"13": ("aktuelles/_index.md", None),
|
|
"72": ("termine.md", "termine"),
|
|
"128": ("downloads.md", None),
|
|
"138": ("impressum.md", None),
|
|
"98": ("archiv/vergangene-o-fahrten.md", None),
|
|
"194": ("archiv/40-jahre-mcg/index.md", None),
|
|
"217": ("archiv/familienausfahrt-2019/index.md", None),
|
|
"287": ("archiv/bilder-familienausfahrt/index.md", None),
|
|
"473": ("archiv/familienfahrt-2022/index.md", None),
|
|
}
|
|
|
|
# Deliberately dropped, with the reason recorded for the migration report.
|
|
SKIP = {
|
|
"3": "stock WordPress privacy-policy draft (English, references an old IP)",
|
|
"12": "WP home page body was the placeholder 'Dies ist nur eine Demo Seite'",
|
|
"410": "empty draft post",
|
|
}
|
|
|
|
# Site chrome pulled out of the media library into assets/img/.
|
|
SITE_IMAGES = {
|
|
"Emblem_transparent.png": "logo.png",
|
|
"cropped-Banner_small_web.jpg": "banner.jpg",
|
|
}
|
|
|
|
SLUG_FIXES = {
|
|
"29-o-fahrt-zum-50-jubilaum": "29-o-fahrt-zum-50-jubilaeum",
|
|
"regelmasger-stammtisch": "regelmaessiger-stammtisch",
|
|
"vorankundigung-mcg-wochenende-2024": "vorankuendigung-mcg-wochenende-2024",
|
|
"o-fahrt-2019": "25-o-fahrt-2022", # title says 25. O-Fahrt 2022; slug was stale
|
|
"termin": "28-o-fahrt-2025", # title says 28. O-Fahrt 2025
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class Target:
|
|
item: object
|
|
path: Path
|
|
bundle: Path | None
|
|
layout: str | None = None
|
|
images: dict[str, str] = field(default_factory=dict) # attachment id -> filename
|
|
|
|
|
|
# --------------------------------------------------------------------- helpers
|
|
|
|
|
|
def yaml_quote(value: str) -> str:
|
|
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
|
|
|
|
def iso(date_gmt: str) -> str:
|
|
if not date_gmt or date_gmt.startswith("0000"):
|
|
return ""
|
|
return date_gmt.replace(" ", "T") + "Z"
|
|
|
|
|
|
def referenced_ids(item, by_filename: dict[str, str] | None = None) -> list[str]:
|
|
"""Every attachment id an item points at, in document order.
|
|
|
|
Some posts carry stale ``wp-image-NNN`` ids whose attachment record was
|
|
deleted from WordPress even though the upload itself survived (post 81 points
|
|
at id 83, which no longer exists). When ``by_filename`` is supplied, such
|
|
references are recovered from the ``src`` URL instead.
|
|
"""
|
|
ids: list[str] = []
|
|
|
|
def add(value):
|
|
if value and value not in ids:
|
|
ids.append(value)
|
|
|
|
thumb = item.meta.get("_thumbnail_id")
|
|
add(thumb)
|
|
for m in re.finditer(r'<img[^>]*>', item.content):
|
|
tag = m.group(0)
|
|
cls = re.search(r'wp-image-(\d+)', tag)
|
|
if cls:
|
|
add(cls.group(1))
|
|
if by_filename is not None:
|
|
src = re.search(r'src="([^"]+)"', tag)
|
|
if src:
|
|
add(by_filename.get(clean_media_name(src.group(1).rsplit("/", 1)[-1])))
|
|
for m in re.finditer(r'<!--\s*wp:gallery\s*(\{.*?\})\s*-->', item.content, re.S):
|
|
try:
|
|
for gid in json.loads(m.group(1)).get("ids", []):
|
|
add(str(gid))
|
|
except json.JSONDecodeError:
|
|
pass
|
|
for m in re.finditer(r'<!--\s*wp:file\s*(\{.*?\})\s*-->', item.content, re.S):
|
|
try:
|
|
add(str(json.loads(m.group(1)).get("id", "")))
|
|
except json.JSONDecodeError:
|
|
pass
|
|
return [i for i in ids if i]
|
|
|
|
|
|
# ------------------------------------------------------------------ conversion
|
|
|
|
|
|
def convert_body(item, target: Target, attachments: dict, pdf_names: dict) -> str:
|
|
body = item.content
|
|
|
|
# Galleries: replace the whole block with a shortcode listing bundle files.
|
|
def gallery_repl(match: re.Match) -> str:
|
|
try:
|
|
ids = [str(i) for i in json.loads(match.group(1)).get("ids", [])]
|
|
except json.JSONDecodeError:
|
|
return ""
|
|
files = [target.images[i] for i in ids if i in target.images]
|
|
if not files:
|
|
return ""
|
|
args = " ".join(f'"{f}"' for f in files)
|
|
return f"\n\n{{{{< gallery {args} >}}}}\n\n"
|
|
|
|
body = re.sub(
|
|
r'<!--\s*wp:gallery\s*(\{.*?\})\s*-->.*?<!--\s*/wp:gallery\s*-->',
|
|
gallery_repl, body, flags=re.S)
|
|
|
|
# File blocks: PDFs live in static/dokumente/, linked via the pdf shortcode.
|
|
def file_repl(match: re.Match) -> str:
|
|
block = match.group(0)
|
|
try:
|
|
attrs = json.loads(match.group(1))
|
|
except json.JSONDecodeError:
|
|
return ""
|
|
att_id = str(attrs.get("id", ""))
|
|
href = attrs.get("href", "")
|
|
name = pdf_names.get(att_id) or clean_media_name(href.rsplit("/", 1)[-1])
|
|
label = re.search(r'>([^<]+)</a>', block)
|
|
label = label.group(1).strip() if label else name
|
|
return f'\n\n{{{{< pdf "{name}" "{label}" >}}}}\n\n'
|
|
|
|
body = re.sub(
|
|
r'<!--\s*wp:file\s*(\{.*?\})\s*-->.*?<!--\s*/wp:file\s*-->',
|
|
file_repl, body, flags=re.S)
|
|
|
|
# Shortcodes that have no Hugo equivalent.
|
|
body = re.sub(r'<!--\s*wp:shortcode\s*-->.*?<!--\s*/wp:shortcode\s*-->', "",
|
|
body, flags=re.S)
|
|
body = re.sub(r'\[table id=\d+\s*/?\]', "", body)
|
|
body = re.sub(r'\[widgets_on_pages[^\]]*\]', "", body)
|
|
|
|
# Remaining Gutenberg comments carry no information once converted.
|
|
body = re.sub(r'<!--\s*/?wp:[^>]*?-->', "", body)
|
|
|
|
# WordPress' editor leaves these around every translated paragraph.
|
|
body = re.sub(r'<span style="vertical-align: inherit;">\s*', "", body)
|
|
body = re.sub(r'</span>', "", body)
|
|
|
|
def resolve_image(src: str, cls: str) -> str:
|
|
m = re.search(r'wp-image-(\d+)', cls or "")
|
|
if m and m.group(1) in target.images:
|
|
return target.images[m.group(1)]
|
|
# Fall back to matching on the original upload filename.
|
|
base = clean_media_name(src.rsplit("/", 1)[-1])
|
|
for name in target.images.values():
|
|
if name == base:
|
|
return name
|
|
return src
|
|
|
|
return html_to_markdown(body, resolve_image=resolve_image)
|
|
|
|
|
|
def front_matter(item, target: Target, extra_tags: list[str]) -> str:
|
|
lines = ["---", f"title: {yaml_quote(item.title)}"]
|
|
if iso(item.date_gmt):
|
|
lines.append(f"date: {iso(item.date_gmt)}")
|
|
if iso(item.modified_gmt) and item.modified_gmt != item.date_gmt:
|
|
lines.append(f"lastmod: {iso(item.modified_gmt)}")
|
|
if extra_tags:
|
|
lines.append("tags: [" + ", ".join(yaml_quote(t) for t in extra_tags) + "]")
|
|
thumb = item.meta.get("_thumbnail_id")
|
|
if thumb and thumb in target.images:
|
|
lines.append(f"featured: {yaml_quote(target.images[thumb])}")
|
|
if target.layout:
|
|
lines.append(f"layout: {yaml_quote(target.layout)}")
|
|
lines.append("---")
|
|
return "\n".join(lines)
|
|
|
|
|
|
# ------------------------------------------------------------------------ main
|
|
|
|
|
|
def main() -> int:
|
|
items = load_items()
|
|
attachments = {k: v for k, v in items.items() if v.post_type == "attachment"}
|
|
|
|
# Cleaned upload filename -> attachment id, for recovering stale wp-image ids.
|
|
by_filename: dict[str, str] = {}
|
|
for att_id, att in attachments.items():
|
|
if att.attachment_url:
|
|
by_filename.setdefault(
|
|
clean_media_name(att.attachment_url.rsplit("/", 1)[-1]), att_id)
|
|
|
|
targets: list[Target] = []
|
|
|
|
for item in items.values():
|
|
if item.post_id in SKIP or item.status != "publish":
|
|
continue
|
|
|
|
if item.post_type == "page":
|
|
mapping = PAGE_MAP.get(item.post_id)
|
|
if not mapping:
|
|
print(f"!! unmapped page {item.post_id} ({item.slug})", file=sys.stderr)
|
|
continue
|
|
rel, layout = mapping
|
|
path = CONTENT / rel
|
|
elif item.post_type == "post":
|
|
slug = SLUG_FIXES.get(item.slug, slugify(item.slug))
|
|
path = CONTENT / "aktuelles" / slug / "index.md"
|
|
layout = None
|
|
else:
|
|
continue
|
|
|
|
bundle = path.parent if path.name == "index.md" else None
|
|
targets.append(Target(item=item, path=path, bundle=bundle, layout=layout))
|
|
|
|
# ---- assign media -------------------------------------------------
|
|
used: set[str] = set()
|
|
pdf_names: dict[str, str] = {}
|
|
|
|
for target in targets:
|
|
for att_id in referenced_ids(target.item, by_filename):
|
|
att = attachments.get(att_id)
|
|
if not att or not att.attachment_url:
|
|
continue
|
|
src = cache_path(att.attachment_url)
|
|
if not src.exists():
|
|
print(f"!! missing cached media for {att_id}: {att.attachment_url}",
|
|
file=sys.stderr)
|
|
continue
|
|
name = clean_media_name(att.attachment_url.rsplit("/", 1)[-1])
|
|
|
|
if src.suffix.lower() == ".pdf":
|
|
STATIC_PDF.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src, STATIC_PDF / name)
|
|
pdf_names[att_id] = name
|
|
target.images[att_id] = name
|
|
else:
|
|
if target.bundle is None:
|
|
# Non-bundle page (termine.md, impressum.md, ...) -> shared assets.
|
|
ASSETS_IMG.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src, ASSETS_IMG / name)
|
|
target.images[att_id] = f"/img/{name}"
|
|
else:
|
|
target.bundle.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src, target.bundle / name)
|
|
target.images[att_id] = name
|
|
used.add(att_id)
|
|
|
|
# Any PDF in the library that no page links to is still worth keeping around
|
|
# for the Downloads page; copy them all so nothing is lost.
|
|
for att_id, att in attachments.items():
|
|
if not att.attachment_url.lower().endswith(".pdf"):
|
|
continue
|
|
src = cache_path(att.attachment_url)
|
|
if src.exists():
|
|
STATIC_PDF.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(src, STATIC_PDF / clean_media_name(
|
|
att.attachment_url.rsplit("/", 1)[-1]))
|
|
used.add(att_id)
|
|
|
|
# Site chrome + widget images (members / memoriam) into assets/img/.
|
|
ASSETS_IMG.mkdir(parents=True, exist_ok=True)
|
|
widget_images = set()
|
|
for yaml_file in ("members.yaml", "memoriam.yaml"):
|
|
text = (REPO_ROOT / "data" / yaml_file).read_text(encoding="utf-8")
|
|
widget_images.update(re.findall(r'image:\s*"img/([^"]+)"', text))
|
|
|
|
for att_id, att in attachments.items():
|
|
original = att.attachment_url.rsplit("/", 1)[-1]
|
|
name = clean_media_name(original)
|
|
src = cache_path(att.attachment_url)
|
|
if not src.exists():
|
|
continue
|
|
if original in SITE_IMAGES:
|
|
shutil.copy2(src, ASSETS_IMG / SITE_IMAGES[original])
|
|
used.add(att_id)
|
|
elif name in widget_images:
|
|
shutil.copy2(src, ASSETS_IMG / name)
|
|
used.add(att_id)
|
|
|
|
# ---- write content ------------------------------------------------
|
|
# content/ is the source of truth once imported, so an accidental re-run must
|
|
# not throw away hand edits. Pass --force to regenerate anyway.
|
|
force = "--force" in sys.argv
|
|
written = kept = 0
|
|
for target in targets:
|
|
item = target.item
|
|
target.path.parent.mkdir(parents=True, exist_ok=True)
|
|
if target.path.exists() and not force:
|
|
kept += 1
|
|
continue
|
|
tags = ["O-Fahrt"] if any(nice == "ofahrt" for _, nice in item.categories) else []
|
|
body = convert_body(item, target, attachments, pdf_names)
|
|
target.path.write_text(
|
|
front_matter(item, target, tags) + "\n\n" + body + "\n",
|
|
encoding="utf-8")
|
|
written += 1
|
|
|
|
print(f"wrote {written} content files"
|
|
+ (f", kept {kept} existing (use --force to overwrite)" if kept else ""))
|
|
print(f"media used: {len(used)}/{len(attachments)} attachments")
|
|
unused = sorted(set(attachments) - used, key=int)
|
|
print(f"unreferenced attachments: {len(unused)}")
|
|
for att_id in unused:
|
|
print(f" - {attachments[att_id].attachment_url}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|