Westreiche: Vault, Werkzeuge und Website
Neuinitialisierung ohne den PDF-Ballast (Regelwerke/Abenteuer/Magazine liegen auf dem Server, siehe tools/pdf_sync.py und tools/pdf_manifest.json). Vorherige Historie bewusst verworfen.
This commit is contained in:
commit
2dd6295caf
611 changed files with 100926 additions and 0 deletions
351
tools/pdf_sync.py
Executable file
351
tools/pdf_sync.py
Executable file
|
|
@ -0,0 +1,351 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Hält die ausgelagerten Regelwerk-/Abenteuer-PDFs zwischen Vault und Webserver synchron.
|
||||
|
||||
Warum: Fremdmaterial (Regelwerke, Abenteuer, Magazine) bläht das Git-Repo auf rund
|
||||
1 GB auf. Es liegt deshalb nicht mehr in Git, sondern auf dem eigenen Server –
|
||||
**außerhalb von `public_html`**, also für Crawler und Direkt-URLs unerreichbar:
|
||||
|
||||
<FTP-Wurzel>/pdf_archiv/blobs/<sha256>.pdf
|
||||
|
||||
Der kleine, versionierte Manifest `tools/pdf_manifest.json` (Pfad → SHA-256 + Größe)
|
||||
sagt einem frischen Checkout, welche Datei wohin gehört. Ablage ist
|
||||
inhaltsadressiert: Duplikate und Umbenennungen kosten keinen zweiten Upload.
|
||||
|
||||
Weil der Blob-Name nur der Hash ist, liegt auf dem Server zusätzlich
|
||||
`pdf_archiv/index.json` – dieselbe Pfad→Hash-Tabelle. Damit bleibt das Archiv
|
||||
auch ohne Repo lesbar (`--pull` holt den Index, wenn das Manifest fehlt).
|
||||
|
||||
Befehle
|
||||
-------
|
||||
tools/pdf_sync.py --scan # Manifest aus den lokalen PDFs neu schreiben
|
||||
tools/pdf_sync.py --push # fehlende Blobs hochladen
|
||||
tools/pdf_sync.py --pull # fehlende Blobs herunterladen (frischer Checkout)
|
||||
tools/pdf_sync.py --check # Abgleich lokal/Remote gegen Manifest
|
||||
tools/pdf_sync.py --index # nur index.json hochladen
|
||||
tools/pdf_sync.py --prune --yes # Remote-Blobs löschen, die im Manifest fehlen
|
||||
|
||||
Zugang: `credentials.txt` im Repo-Wurzel (FTP_URL, FTP_USER, FTP_PASSWORD), FTPS
|
||||
explizit (`--ssl-reqd`, wie in `tools/deploy_web.sh`).
|
||||
|
||||
Aufgenommen werden alle PDFs über 1 MiB aus `RPG/` und `Druckmaterial/` – **außer**
|
||||
den selbst erzeugten Handouts (Summarien, SL-Tabellen, SL-Schirm, pdf_export).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
CREDENTIALS = ROOT / "credentials.txt"
|
||||
MANIFEST = ROOT / "tools" / "pdf_manifest.json"
|
||||
|
||||
SCAN_DIRS = ["RPG", "Druckmaterial"]
|
||||
MIN_SIZE = 1024 * 1024 # 1 MiB: alles Kleinere bleibt in Git
|
||||
SKIP_PATH_PARTS = {"pdf_export", "SL Schirm"}
|
||||
SKIP_NAME_PARTS = ("Summarium", "SL-Tabellen", "SL-Schirm")
|
||||
|
||||
REMOTE_DIR = os.environ.get("PDF_REMOTE_DIR", "pdf_archiv/blobs")
|
||||
REMOTE_INDEX = os.environ.get("PDF_REMOTE_INDEX", "pdf_archiv/index.json")
|
||||
MANIFEST_VERSION = 1
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ Hilfen
|
||||
def die(msg: str) -> None:
|
||||
raise SystemExit(f"FEHLER: {msg}")
|
||||
|
||||
|
||||
def credentials() -> tuple[str, str, str]:
|
||||
if not CREDENTIALS.exists():
|
||||
die(f"{CREDENTIALS} fehlt (FTP_URL, FTP_USER, FTP_PASSWORD)")
|
||||
werte: dict[str, str] = {}
|
||||
for line in CREDENTIALS.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
werte[k.strip()] = v.strip().strip("'\"")
|
||||
try:
|
||||
return werte["FTP_URL"], werte["FTP_USER"], werte["FTP_PASSWORD"]
|
||||
except KeyError as fehlt:
|
||||
die(f"{fehlt} fehlt in credentials.txt")
|
||||
|
||||
|
||||
def curl(args: list[str], userpass: str | None = None, verbose: bool = False) -> str:
|
||||
"""curl mit FTPS; Zugangsdaten laufen über stdin-Config (nicht über argv)."""
|
||||
cmd = ["curl", "-sS", "--ssl-reqd", "--retry", "3", "--retry-delay", "2",
|
||||
"--connect-timeout", "20", "--ftp-create-dirs"]
|
||||
if not verbose:
|
||||
cmd.append("--no-progress-meter")
|
||||
cmd += args
|
||||
if userpass:
|
||||
cmd += ["--config", "-"]
|
||||
proc = subprocess.run(cmd, input=f'user = "{userpass}"\n', text=True,
|
||||
capture_output=True)
|
||||
else:
|
||||
proc = subprocess.run(cmd, text=True, capture_output=True)
|
||||
if proc.returncode != 0:
|
||||
die(f"curl: {proc.stderr.strip() or proc.stdout.strip()}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as fh:
|
||||
for block in iter(lambda: fh.read(1024 * 1024), b""):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def is_archived_pdf(path: Path) -> bool:
|
||||
if path.suffix.lower() != ".pdf" or not path.is_file():
|
||||
return False
|
||||
if path.stat().st_size <= MIN_SIZE:
|
||||
return False
|
||||
rel = path.relative_to(ROOT)
|
||||
if any(part in SKIP_PATH_PARTS for part in rel.parts):
|
||||
return False
|
||||
if any(teil in path.name for teil in SKIP_NAME_PARTS):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def discover() -> list[Path]:
|
||||
treffer: list[Path] = []
|
||||
for wurzel in SCAN_DIRS:
|
||||
basis = ROOT / wurzel
|
||||
if basis.exists():
|
||||
treffer += [p for p in basis.rglob("*.pdf") if is_archived_pdf(p)]
|
||||
return sorted(treffer)
|
||||
|
||||
|
||||
def load_manifest() -> dict:
|
||||
if not MANIFEST.exists():
|
||||
return {"version": MANIFEST_VERSION, "remote_dir": REMOTE_DIR, "files": {}}
|
||||
return json.loads(MANIFEST.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def remote_listing(url: str, userpass: str) -> set[str]:
|
||||
ziel = f"ftp://{url}/{REMOTE_DIR}/"
|
||||
try:
|
||||
ausgabe = curl(["--list-only", ziel], userpass)
|
||||
except SystemExit:
|
||||
return set() # Verzeichnis existiert noch nicht
|
||||
return {z.strip() for z in ausgabe.splitlines() if z.strip() not in ("", ".", "..")}
|
||||
|
||||
|
||||
def index_json(files: dict[str, dict]) -> str:
|
||||
"""Serverfassung des Manifests – macht das Archiv ohne Repo lesbar."""
|
||||
return json.dumps({"version": MANIFEST_VERSION, "remote_dir": REMOTE_DIR,
|
||||
"hinweis": "Pfad -> SHA-256 + Groesse; Blobs liegen unter pdf_archiv/blobs/",
|
||||
"files": dict(sorted(files.items()))},
|
||||
indent=2, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def upload_index(url: str, userpass: str, files: dict[str, dict]) -> None:
|
||||
tmp = ROOT / "tools" / ".pdf_index.tmp"
|
||||
tmp.write_text(index_json(files), encoding="utf-8")
|
||||
try:
|
||||
curl(["-T", str(tmp), f"ftp://{url}/{REMOTE_INDEX}"], userpass)
|
||||
print(f"Index hochgeladen: {REMOTE_INDEX}")
|
||||
finally:
|
||||
tmp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def fetch_index(url: str, userpass: str) -> dict:
|
||||
tmp = ROOT / "tools" / ".pdf_index.tmp"
|
||||
curl(["-o", str(tmp), f"ftp://{url}/{REMOTE_INDEX}"], userpass)
|
||||
try:
|
||||
return json.loads(tmp.read_text(encoding="utf-8"))
|
||||
finally:
|
||||
tmp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def mb(n: int) -> str:
|
||||
return f"{n / 1048576:.1f} MB"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ Befehle
|
||||
def cmd_scan(verbose: bool) -> None:
|
||||
alt = load_manifest()["files"]
|
||||
neu: dict[str, dict] = {}
|
||||
for p in discover():
|
||||
neu[p.relative_to(ROOT).as_posix()] = {"sha256": sha256(p), "size": p.stat().st_size}
|
||||
geaendert = [k for k, v in neu.items() if alt.get(k, {}).get("sha256") not in (None, v["sha256"])]
|
||||
dazu = [k for k in neu if k not in alt]
|
||||
weg = [k for k in alt if k not in neu]
|
||||
MANIFEST.write_text(json.dumps({"version": MANIFEST_VERSION, "remote_dir": REMOTE_DIR,
|
||||
"files": dict(sorted(neu.items()))},
|
||||
indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
blobs = {v["sha256"] for v in neu.values()}
|
||||
gesamt = sum(v["size"] for v in neu.values())
|
||||
print(f"Manifest: {len(neu)} Dateien, {len(blobs)} Blobs, {mb(gesamt)} → {MANIFEST.name}")
|
||||
for titel, menge in (("neu", dazu), ("geändert", geaendert), ("entfernt", weg)):
|
||||
if menge:
|
||||
print(f" {titel}: {len(menge)}")
|
||||
if verbose:
|
||||
for k in menge:
|
||||
print(f" {k}")
|
||||
|
||||
|
||||
def cmd_push(verbose: bool, force: bool) -> None:
|
||||
url, user, pw = credentials()
|
||||
userpass = f"{user}:{pw}"
|
||||
files = load_manifest()["files"]
|
||||
if not files:
|
||||
die("Manifest ist leer – zuerst --scan laufen lassen")
|
||||
vorhanden = remote_listing(url, userpass)
|
||||
offen = [(k, v) for k, v in files.items()
|
||||
if force or f"{v['sha256']}.pdf" not in vorhanden]
|
||||
blobs: dict[str, int] = {}
|
||||
for k, v in offen:
|
||||
blobs.setdefault(v["sha256"], v["size"])
|
||||
print(f"Remote: {len(vorhanden)} Blobs · zu laden: {len(blobs)} Blobs "
|
||||
f"({mb(sum(blobs.values()))}) für {len(offen)} Dateien")
|
||||
for n, (digest, size) in enumerate(sorted(blobs.items()), 1):
|
||||
quelle = next((ROOT / k for k, v in files.items() if v["sha256"] == digest), None)
|
||||
if quelle is None or not quelle.exists():
|
||||
print(f" [{n}/{len(blobs)}] übersprungen (lokal nicht vorhanden): {digest[:12]}")
|
||||
continue
|
||||
if verbose or n == 1 or n == len(blobs) or n % 10 == 0:
|
||||
print(f" [{n}/{len(blobs)}] {quelle.name} ({mb(size)})")
|
||||
curl(["-T", str(quelle), f"ftp://{url}/{REMOTE_DIR}/{digest}.pdf"], userpass)
|
||||
upload_index(url, userpass, files)
|
||||
print("Upload fertig.")
|
||||
|
||||
|
||||
def cmd_pull(verbose: bool, force: bool) -> None:
|
||||
url, user, pw = credentials()
|
||||
userpass = f"{user}:{pw}"
|
||||
files = load_manifest()["files"]
|
||||
if not files:
|
||||
print("Manifest lokal leer – hole pdf_archiv/index.json vom Server …")
|
||||
files = fetch_index(url, userpass).get("files", {})
|
||||
if files:
|
||||
MANIFEST.write_text(index_json(files), encoding="utf-8")
|
||||
print(f"Manifest aus Server-Index rekonstruiert ({len(files)} Dateien).")
|
||||
if not files:
|
||||
die("Manifest ist leer – zuerst --scan laufen lassen")
|
||||
vorhanden = remote_listing(url, userpass)
|
||||
fehlend, konflikt, geladen = [], [], 0
|
||||
for rel, meta in files.items():
|
||||
ziel = ROOT / rel
|
||||
digest = meta["sha256"]
|
||||
if ziel.exists():
|
||||
if sha256(ziel) != digest:
|
||||
konflikt.append(rel)
|
||||
if not force:
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
if f"{digest}.pdf" not in vorhanden:
|
||||
fehlend.append(rel)
|
||||
continue
|
||||
ziel.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = ziel.with_suffix(ziel.suffix + ".part")
|
||||
if verbose:
|
||||
print(f" ↓ {rel}")
|
||||
curl(["-o", str(tmp), f"ftp://{url}/{REMOTE_DIR}/{digest}.pdf"], userpass)
|
||||
if sha256(tmp) != digest:
|
||||
tmp.unlink(missing_ok=True)
|
||||
die(f"Prüfsumme falsch nach Download: {rel}")
|
||||
tmp.replace(ziel)
|
||||
geladen += 1
|
||||
print(f"Download fertig: {geladen} Dateien geladen, {len(konflikt)} Konflikte, "
|
||||
f"{len(fehlend)} fehlen auf dem Server")
|
||||
for rel in konflikt:
|
||||
print(f" KONFLIKT (lokal anders, nicht überschrieben): {rel}")
|
||||
for rel in fehlend:
|
||||
print(f" FEHLT REMOTE: {rel}")
|
||||
|
||||
|
||||
def cmd_check() -> None:
|
||||
url, user, pw = credentials()
|
||||
userpass = f"{user}:{pw}"
|
||||
files = load_manifest()["files"]
|
||||
vorhanden = remote_listing(url, userpass)
|
||||
lokal_fehlt, lokal_anders, remote_fehlt = [], [], []
|
||||
for rel, meta in files.items():
|
||||
ziel = ROOT / rel
|
||||
if not ziel.exists():
|
||||
lokal_fehlt.append(rel)
|
||||
elif sha256(ziel) != meta["sha256"]:
|
||||
lokal_anders.append(rel)
|
||||
if f"{meta['sha256']}.pdf" not in vorhanden:
|
||||
remote_fehlt.append(rel)
|
||||
ungelistet = [p.relative_to(ROOT).as_posix() for p in discover()
|
||||
if p.relative_to(ROOT).as_posix() not in files]
|
||||
print(f"Manifest: {len(files)} Dateien · Remote-Blobs: {len(vorhanden)}")
|
||||
print(f" lokal fehlt: {len(lokal_fehlt)}")
|
||||
print(f" lokal abweichend: {len(lokal_anders)}")
|
||||
print(f" remote fehlt: {len(remote_fehlt)}")
|
||||
print(f" nicht im Manifest: {len(ungelistet)}")
|
||||
for titel, menge in (("lokal fehlt", lokal_fehlt), ("lokal abweichend", lokal_anders),
|
||||
("remote fehlt", remote_fehlt),):
|
||||
for rel in menge[:20]:
|
||||
print(f" {titel}: {rel}")
|
||||
for rel in ungelistet:
|
||||
print(f" nicht im Manifest: {rel}")
|
||||
|
||||
|
||||
def cmd_prune(yes: bool, verbose: bool) -> None:
|
||||
url, user, pw = credentials()
|
||||
userpass = f"{user}:{pw}"
|
||||
gebraucht = {f"{v['sha256']}.pdf" for v in load_manifest()["files"].values()}
|
||||
vorhanden = remote_listing(url, userpass)
|
||||
ueberzaehlig = sorted(vorhanden - gebraucht)
|
||||
if not ueberzaehlig:
|
||||
print("Nichts zu löschen.")
|
||||
return
|
||||
print(f"{len(ueberzaehlig)} Blobs ohne Manifest-Eintrag:")
|
||||
for name in ueberzaehlig:
|
||||
print(f" {name}")
|
||||
if not yes:
|
||||
print("Zum Löschen mit --yes erneut aufrufen.")
|
||||
return
|
||||
for name in ueberzaehlig:
|
||||
if verbose:
|
||||
print(f" DELE {name}")
|
||||
curl(["-Q", f"DELE /{REMOTE_DIR}/{name}", f"ftp://{url}/"], userpass)
|
||||
print("Gelöscht.")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="PDF-Archiv zwischen Vault und Server syncen")
|
||||
g = ap.add_mutually_exclusive_group()
|
||||
g.add_argument("--scan", action="store_true", help="Manifest aus lokalen PDFs schreiben")
|
||||
g.add_argument("--push", action="store_true", help="fehlende Blobs hochladen")
|
||||
g.add_argument("--pull", action="store_true", help="fehlende Blobs herunterladen")
|
||||
g.add_argument("--check", action="store_true", help="Abgleich lokal/Remote")
|
||||
g.add_argument("--index", action="store_true", help="nur index.json hochladen")
|
||||
g.add_argument("--prune", action="store_true", help="Remote-Blobs ohne Manifest löschen")
|
||||
ap.add_argument("--force", action="store_true", help="vorhandene/gleichnamige Dateien überschreiben")
|
||||
ap.add_argument("--yes", action="store_true", help="Löschen bestätigen (--prune)")
|
||||
ap.add_argument("-v", "--verbose", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.scan or not any([args.push, args.pull, args.check, args.prune, args.index]):
|
||||
cmd_scan(args.verbose)
|
||||
if args.push:
|
||||
cmd_push(args.verbose, args.force)
|
||||
if args.pull:
|
||||
cmd_pull(args.verbose, args.force)
|
||||
if args.check:
|
||||
cmd_check()
|
||||
if args.index:
|
||||
url, user, pw = credentials()
|
||||
files = load_manifest()["files"]
|
||||
if not files:
|
||||
die("Manifest ist leer – zuerst --scan laufen lassen")
|
||||
upload_index(url, f"{user}:{pw}", files)
|
||||
if args.prune:
|
||||
cmd_prune(args.yes, args.verbose)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue