"""Download every attachment referenced by the WXR export into .cache/uploads/. The export only carries URLs, not binaries, so the files have to come off the live WordPress host before it is retired. Images are capped at MAX_EDGE px on the long side and re-encoded; PDFs are stored verbatim. Re-running is cheap: anything already in the cache is skipped. """ from __future__ import annotations import sys import urllib.error import urllib.parse import urllib.request from io import BytesIO from PIL import Image from wxr import CACHE, attachments, cache_path, load_items MAX_EDGE = 2000 JPEG_QUALITY = 82 USER_AGENT = "mcg-website-migration/1.0 (+https://mcg.luxick.de)" def fetch(url: str) -> bytes: # A few uploads have umlauts in the filename (e.g. Hövelhof.jpg); the HTTP # request line must be ASCII, so percent-encode the path before sending. parts = urllib.parse.urlsplit(url) safe = urllib.parse.urlunsplit( parts._replace(path=urllib.parse.quote(parts.path, safe="/%")) ) req = urllib.request.Request(safe, headers={"User-Agent": USER_AGENT}) with urllib.request.urlopen(req, timeout=60) as resp: return resp.read() def store_image(raw: bytes, dest) -> None: img = Image.open(BytesIO(raw)) img.load() # Honour the EXIF orientation now; the tag is dropped on save. try: from PIL import ImageOps img = ImageOps.exif_transpose(img) except Exception: pass if max(img.size) > MAX_EDGE: img.thumbnail((MAX_EDGE, MAX_EDGE), Image.LANCZOS) suffix = dest.suffix.lower() if suffix in (".jpg", ".jpeg"): img.convert("RGB").save(dest, "JPEG", quality=JPEG_QUALITY, optimize=True, progressive=True) elif suffix == ".png": img.save(dest, "PNG", optimize=True) else: img.save(dest) def main() -> int: items = load_items() atts = attachments(items) skipped = downloaded = failed = 0 for post_id, item in sorted(atts.items(), key=lambda kv: int(kv[0])): url = item.attachment_url if not url: print(f"!! {post_id}: no attachment_url", file=sys.stderr) failed += 1 continue dest = cache_path(url) if dest.exists() and dest.stat().st_size > 0: skipped += 1 continue dest.parent.mkdir(parents=True, exist_ok=True) try: raw = fetch(url) except Exception as exc: # one bad upload must not abort the whole run print(f"!! {post_id}: {url} -> {exc!r}", file=sys.stderr) failed += 1 continue try: if dest.suffix.lower() == ".pdf": dest.write_bytes(raw) else: store_image(raw, dest) except Exception as exc: # corrupt/unsupported source, keep the original bytes print(f"!! {post_id}: could not process {url} ({exc}); storing raw", file=sys.stderr) dest.write_bytes(raw) downloaded += 1 print(f" {dest.relative_to(CACHE)} ({len(raw) // 1024} KiB -> " f"{dest.stat().st_size // 1024} KiB)") print(f"\ndownloaded={downloaded} cached={skipped} failed={failed} " f"total={len(atts)}") return 1 if failed else 0 if __name__ == "__main__": raise SystemExit(main())