commit f7a9714e93cd81fe86018c0e548d98d1de13dc03
Author: veg <veg@silentio.us>
Date:   Sat May 30 23:30:07 2026 +0000

    Add mm3-hk-export: HyperKitty archive -> mbox exporter
    
    Scriptable equivalent of HyperKitty's web Download button for
    backup-and-destroy loops. Runs the export through Debian's mailman-web
    wrapper (HyperKitty lives in the web DB, separate from core), reproducing
    the web view's Email.as_bytes() output.
    
    - Handles the three archive states without writing partial/empty files:
      populated (writes mbox), row-but-empty, and no row at all.
    - For a missing row, best-effort core REST lookup distinguishes
      archive_policy=never from 'nothing archived yet'.
    - -z/--gzip matches the web .mbox.gz (deterministic header, mtime=0).
    - Privilege-safe: web user writes a 0600 mkstemp file, root moves it.
    - Address validated before interpolation into the embedded Python.

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..5cc8d4d
--- /dev/null
+++ b/README.md
@@ -0,0 +1,86 @@
+# mm3-hk-export
+
+Export a private **HyperKitty** (Mailman 3) list archive to an `mbox` file from
+the command line: the scriptable equivalent of clicking the web UI's
+**Download** button, so it fits a backup-and-destroy loop without manual
+browser steps.
+
+## Why this exists
+
+HyperKitty has **no `hyperkitty_export` management command**; the only built-in
+export is the web "Download" view (gated by the `HYPERKITTY_MBOX_EXPORT`
+setting). This script reproduces that view's exact output by calling the same
+`Email.as_bytes()` code path on each archived message, driven through Debian's
+`mailman-web` wrapper: no web login, no direct database reconstruction.
+
+The archives live in the **mailman-web / HyperKitty** database, which is
+separate from Mailman **core**. That is why the export must go through
+`mailman-web` (not `mailman` / `mailman-wrapper`, whose `shell` only sees core).
+
+## Requirements
+
+- Debian-style Mailman 3 with the `mailman-web` wrapper on `PATH`
+  (Django site at `/usr/share/mailman3-web/`, HyperKitty installed).
+- Run as **root**: `mailman-web` drops privileges to the web user to read the
+  archive, and the script (as root) moves the finished file into place.
+- Optional, only to enrich the "not archived" notice: `curl` and the
+  `mailman-wrapper` core CLI (for a best-effort `archive_policy` lookup over the
+  core REST API). Missing either just falls back to a generic message.
+
+## Usage
+
+```
+mm3-hk-export [-z|--gzip] <list@domain> [output]
+```
+
+- `output` defaults to `./<localpart>.mbox` (`./<localpart>.mbox.gz` with `-z`).
+- `-z` / `--gzip` gzip-compresses the stream, matching the web download's
+  `.mbox.gz` byte-for-byte (deterministic header, `mtime=0`); `.gz` is appended
+  to the output name if absent.
+
+Examples:
+
+```sh
+mm3-hk-export announce@lists.example.com /root/mm3/announce.mbox
+mm3-hk-export --gzip announce@lists.example.com /root/mm3/announce.mbox.gz
+```
+
+## Outcomes
+
+The script never writes a partial or empty file, and exits `0` on all three:
+
+| Situation | HyperKitty state | Result |
+|---|---|---|
+| Archive has messages | row with N emails | writes the mbox, prints `wrote N messages to …` |
+| Archived but empty | row, 0 emails | no file, prints an "empty archive" notice |
+| No archive at all | no `MailingList` row | no file; consults core `archive_policy` to say whether the list is `never`-archived or simply has nothing archived yet |
+
+A genuine error (e.g. `mailman-web` failing) exits non-zero with the underlying
+output on stderr.
+
+## In a backup-and-destroy loop
+
+```sh
+mkdir -p /root/mm3
+for list in list1 list2; do
+    addr="$list@domain.tld"
+    mm3-hk-export "$addr" "/root/mm3/$list.mbox"   # skips cleanly if no archive
+    mailman-wrapper members "$addr" > "/root/mm3/$list"
+    mailman-wrapper remove "$addr"
+done
+```
+
+Export before `remove`. (HyperKitty's archive lives in a separate database and
+is not deleted by core's `remove`, but exporting first keeps the intent clear.)
+
+## Caveats
+
+- **Reconstruction, not the raw original.** HyperKitty rebuilds each message
+  from parsed fields: `From`, `To`, `Subject`, `Date`, `Message-ID`,
+  `In-Reply-To`, body, and attachments. Headers like `Cc`, `References`,
+  `List-*`, and custom `X-*` are not preserved. This is inherent to
+  HyperKitty; the manual web download has the identical limitation.
+- **Memory.** Messages are streamed with a DB iterator and written
+  incrementally, avoiding the web export's known blow-up on large lists.
+- **Re-import.** The output is valid mbox (proper `From ` separators,
+  `From`-mangled bodies) and can be re-imported with `hyperkitty_import`.
diff --git a/mm3-hk-export b/mm3-hk-export
new file mode 100755
index 0000000..1e5be38
--- /dev/null
+++ b/mm3-hk-export
@@ -0,0 +1,142 @@
+#!/bin/sh
+# mm3-hk-export: export a private HyperKitty (Mailman 3) list archive to an mbox.
+#
+# Usage:  mm3-hk-export [-z|--gzip] <list@domain> [output]
+#
+# Output is byte-for-byte what the web "Download" button produces (same
+# Email.as_bytes() code path). With -z/--gzip the stream is gzip-compressed,
+# matching the web download's .mbox.gz exactly (and ".gz" is appended to the
+# output name if absent). If the list has no archived messages — including the
+# case where HyperKitty never created a MailingList row for it — nothing is
+# written and the script exits 0 with a notice.
+#
+# Runs the export through Debian's `mailman-web` wrapper, which executes the
+# Django site as the correct web user. The mbox is created by that (privilege-
+# dropped) user via tempfile.mkstemp (0600, race-free) and then moved into
+# place by this script, which is expected to run as root.
+
+set -eu
+
+usage() {
+    echo "usage: mm3-hk-export [-z|--gzip] <list@domain> [output]" >&2
+}
+
+GZIP=0
+while [ $# -gt 0 ]; do
+    case "$1" in
+        -z|--gzip) GZIP=1; shift ;;
+        -h|--help) usage; exit 0 ;;
+        --)        shift; break ;;
+        -*)        echo "mm3-hk-export: unknown option: $1" >&2; usage; exit 2 ;;
+        *)         break ;;
+    esac
+done
+
+if [ $# -lt 1 ]; then usage; exit 2; fi
+addr="$1"
+out="${2:-./${addr%@*}.mbox}"
+
+if [ "$GZIP" -eq 1 ]; then
+    gzip_py=True
+    case "$out" in *.gz) ;; *) out="$out.gz" ;; esac
+else
+    gzip_py=False
+fi
+
+# Validate the address: this value is interpolated into a Python string
+# literal below, so reject anything that isn't a plain localpart@domain.
+if ! printf '%s' "$addr" | grep -Eq '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+$'; then
+    echo "mm3-hk-export: invalid list address: $addr" >&2
+    exit 2
+fi
+
+# Best-effort: ask Mailman *core* (via its REST API) for a list's archive_policy.
+# Used only to explain why a list has no HyperKitty archive. Echoes the policy
+# string (never|private|public) on success, nothing on any failure — it must
+# never abort the export. REST root + credentials are read live from
+# `mailman info` so no secrets are baked into this script.
+core_archive_policy() {
+    command -v curl            >/dev/null 2>&1 || return 0
+    command -v mailman-wrapper >/dev/null 2>&1 || return 0
+    info=$(mailman-wrapper info 2>/dev/null) || return 0
+    root=$(printf '%s\n'  "$info" | sed -n 's/^REST root url: *//p')
+    creds=$(printf '%s\n' "$info" | sed -n 's/^REST credentials: *//p')
+    [ -n "$root" ] && [ -n "$creds" ] || return 0
+    lid=$(printf '%s' "$1" | sed 's/@/./')
+    curl -sf -u "$creds" "${root}lists/${lid}/config/archive_policy" 2>/dev/null \
+        | sed -n 's/.*"archive_policy": *"\([^"]*\)".*/\1/p'
+}
+
+result=$(mailman-web shell <<EOF
+import sys, os, tempfile, gzip
+from hyperkitty.models import MailingList
+addr = "$addr"
+GZIP = $gzip_py
+try:
+    ml = MailingList.objects.get(name=addr)
+except MailingList.DoesNotExist:
+    # No HyperKitty row at all: list is not archived (archive_policy=never)
+    # or no message has ever been archived for it.
+    print("MM3HK|NONE|0")
+    sys.exit(0)
+n = ml.emails.count()
+if n == 0:
+    # Row exists but holds no messages.
+    print("MM3HK|EMPTY|0")
+    sys.exit(0)
+fd, path = tempfile.mkstemp(prefix="mm3hk-", suffix=".mbox", dir="/var/tmp")
+raw = os.fdopen(fd, "wb")
+# mtime=0 keeps the gzip header deterministic (no timestamp), matching the
+# web export, which uses a bare zlib gzip stream.
+fh = gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) if GZIP else raw
+try:
+    for e in ml.emails.order_by("archived_date").iterator():
+        fh.write(e.as_bytes())
+        fh.write(b"\n\n")
+finally:
+    if GZIP:
+        fh.close()
+    raw.close()
+print("MM3HK|FILE|%d|%s" % (n, path))
+EOF
+)
+
+line=$(printf '%s\n' "$result" | grep '^MM3HK|' | tail -n1 || true)
+if [ -z "$line" ]; then
+    echo "mm3-hk-export: unexpected output from mailman-web shell:" >&2
+    printf '%s\n' "$result" >&2
+    exit 1
+fi
+
+status=$(printf '%s' "$line" | cut -d'|' -f2)
+count=$(printf '%s' "$line" | cut -d'|' -f3)
+
+case "$status" in
+    NONE)
+        policy=$(core_archive_policy "$addr")
+        case "$policy" in
+            never)
+                echo "mm3-hk-export: $addr is set not to archive (archive_policy=never); nothing to export." >&2
+                ;;
+            private|public)
+                echo "mm3-hk-export: $addr archives (archive_policy=$policy) but nothing has been archived yet; nothing written." >&2
+                ;;
+            *)
+                echo "mm3-hk-export: $addr has no HyperKitty archive (archiving disabled, or nothing ever archived); nothing written." >&2
+                ;;
+        esac
+        ;;
+    EMPTY)
+        echo "mm3-hk-export: $addr has a HyperKitty archive but 0 messages — nothing written." >&2
+        ;;
+    FILE)
+        path=$(printf '%s' "$line" | cut -d'|' -f4)
+        mkdir -p "$(dirname "$out")"
+        mv "$path" "$out"
+        echo "mm3-hk-export: wrote $count messages to $out" >&2
+        ;;
+    *)
+        echo "mm3-hk-export: unexpected status '$status'" >&2
+        exit 1
+        ;;
+esac