352 lines
14 KiB
Python
352 lines
14 KiB
Python
|
|
#!/usr/bin/env python3
|
|||
|
|
"""
|
|||
|
|
Hält die ausgelagerten Regelwerk-/Abenteuer-PDFs zwischen Vault und Webserver synchron.
|
|||
|
|
|
|||
|
|
Warum: Fremdmaterial (Regelwerke, Abenteuer, Magazine) bläht das Git-Repo auf rund
|
|||
|
|
1 GB auf. Es liegt deshalb nicht mehr in Git, sondern auf dem eigenen Server –
|
|||
|
|
**außerhalb von `public_html`**, also für Crawler und Direkt-URLs unerreichbar:
|
|||
|
|
|
|||
|
|
<FTP-Wurzel>/pdf_archiv/blobs/<sha256>.pdf
|
|||
|
|
|
|||
|
|
Der kleine, versionierte Manifest `tools/pdf_manifest.json` (Pfad → SHA-256 + Größe)
|
|||
|
|
sagt einem frischen Checkout, welche Datei wohin gehört. Ablage ist
|
|||
|
|
inhaltsadressiert: Duplikate und Umbenennungen kosten keinen zweiten Upload.
|
|||
|
|
|
|||
|
|
Weil der Blob-Name nur der Hash ist, liegt auf dem Server zusätzlich
|
|||
|
|
`pdf_archiv/index.json` – dieselbe Pfad→Hash-Tabelle. Damit bleibt das Archiv
|
|||
|
|
auch ohne Repo lesbar (`--pull` holt den Index, wenn das Manifest fehlt).
|
|||
|
|
|
|||
|
|
Befehle
|
|||
|
|
-------
|
|||
|
|
tools/pdf_sync.py --scan # Manifest aus den lokalen PDFs neu schreiben
|
|||
|
|
tools/pdf_sync.py --push # fehlende Blobs hochladen
|
|||
|
|
tools/pdf_sync.py --pull # fehlende Blobs herunterladen (frischer Checkout)
|
|||
|
|
tools/pdf_sync.py --check # Abgleich lokal/Remote gegen Manifest
|
|||
|
|
tools/pdf_sync.py --index # nur index.json hochladen
|
|||
|
|
tools/pdf_sync.py --prune --yes # Remote-Blobs löschen, die im Manifest fehlen
|
|||
|
|
|
|||
|
|
Zugang: `credentials.txt` im Repo-Wurzel (FTP_URL, FTP_USER, FTP_PASSWORD), FTPS
|
|||
|
|
explizit (`--ssl-reqd`, wie in `tools/deploy_web.sh`).
|
|||
|
|
|
|||
|
|
Aufgenommen werden alle PDFs über 1 MiB aus `RPG/` und `Druckmaterial/` – **außer**
|
|||
|
|
den selbst erzeugten Handouts (Summarien, SL-Tabellen, SL-Schirm, pdf_export).
|
|||
|
|
"""
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import argparse
|
|||
|
|
import hashlib
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
import subprocess
|
|||
|
|
import sys
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|||
|
|
CREDENTIALS = ROOT / "credentials.txt"
|
|||
|
|
MANIFEST = ROOT / "tools" / "pdf_manifest.json"
|
|||
|
|
|
|||
|
|
SCAN_DIRS = ["RPG", "Druckmaterial"]
|
|||
|
|
MIN_SIZE = 1024 * 1024 # 1 MiB: alles Kleinere bleibt in Git
|
|||
|
|
SKIP_PATH_PARTS = {"pdf_export", "SL Schirm"}
|
|||
|
|
SKIP_NAME_PARTS = ("Summarium", "SL-Tabellen", "SL-Schirm")
|
|||
|
|
|
|||
|
|
REMOTE_DIR = os.environ.get("PDF_REMOTE_DIR", "pdf_archiv/blobs")
|
|||
|
|
REMOTE_INDEX = os.environ.get("PDF_REMOTE_INDEX", "pdf_archiv/index.json")
|
|||
|
|
MANIFEST_VERSION = 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ------------------------------------------------------------------ Hilfen
|
|||
|
|
def die(msg: str) -> None:
|
|||
|
|
raise SystemExit(f"FEHLER: {msg}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def credentials() -> tuple[str, str, str]:
|
|||
|
|
if not CREDENTIALS.exists():
|
|||
|
|
die(f"{CREDENTIALS} fehlt (FTP_URL, FTP_USER, FTP_PASSWORD)")
|
|||
|
|
werte: dict[str, str] = {}
|
|||
|
|
for line in CREDENTIALS.read_text(encoding="utf-8").splitlines():
|
|||
|
|
line = line.strip()
|
|||
|
|
if not line or line.startswith("#") or "=" not in line:
|
|||
|
|
continue
|
|||
|
|
k, v = line.split("=", 1)
|
|||
|
|
werte[k.strip()] = v.strip().strip("'\"")
|
|||
|
|
try:
|
|||
|
|
return werte["FTP_URL"], werte["FTP_USER"], werte["FTP_PASSWORD"]
|
|||
|
|
except KeyError as fehlt:
|
|||
|
|
die(f"{fehlt} fehlt in credentials.txt")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def curl(args: list[str], userpass: str | None = None, verbose: bool = False) -> str:
|
|||
|
|
"""curl mit FTPS; Zugangsdaten laufen über stdin-Config (nicht über argv)."""
|
|||
|
|
cmd = ["curl", "-sS", "--ssl-reqd", "--retry", "3", "--retry-delay", "2",
|
|||
|
|
"--connect-timeout", "20", "--ftp-create-dirs"]
|
|||
|
|
if not verbose:
|
|||
|
|
cmd.append("--no-progress-meter")
|
|||
|
|
cmd += args
|
|||
|
|
if userpass:
|
|||
|
|
cmd += ["--config", "-"]
|
|||
|
|
proc = subprocess.run(cmd, input=f'user = "{userpass}"\n', text=True,
|
|||
|
|
capture_output=True)
|
|||
|
|
else:
|
|||
|
|
proc = subprocess.run(cmd, text=True, capture_output=True)
|
|||
|
|
if proc.returncode != 0:
|
|||
|
|
die(f"curl: {proc.stderr.strip() or proc.stdout.strip()}")
|
|||
|
|
return proc.stdout
|
|||
|
|
|
|||
|
|
|
|||
|
|
def sha256(path: Path) -> str:
|
|||
|
|
h = hashlib.sha256()
|
|||
|
|
with path.open("rb") as fh:
|
|||
|
|
for block in iter(lambda: fh.read(1024 * 1024), b""):
|
|||
|
|
h.update(block)
|
|||
|
|
return h.hexdigest()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def is_archived_pdf(path: Path) -> bool:
|
|||
|
|
if path.suffix.lower() != ".pdf" or not path.is_file():
|
|||
|
|
return False
|
|||
|
|
if path.stat().st_size <= MIN_SIZE:
|
|||
|
|
return False
|
|||
|
|
rel = path.relative_to(ROOT)
|
|||
|
|
if any(part in SKIP_PATH_PARTS for part in rel.parts):
|
|||
|
|
return False
|
|||
|
|
if any(teil in path.name for teil in SKIP_NAME_PARTS):
|
|||
|
|
return False
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
|
|||
|
|
def discover() -> list[Path]:
|
|||
|
|
treffer: list[Path] = []
|
|||
|
|
for wurzel in SCAN_DIRS:
|
|||
|
|
basis = ROOT / wurzel
|
|||
|
|
if basis.exists():
|
|||
|
|
treffer += [p for p in basis.rglob("*.pdf") if is_archived_pdf(p)]
|
|||
|
|
return sorted(treffer)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_manifest() -> dict:
|
|||
|
|
if not MANIFEST.exists():
|
|||
|
|
return {"version": MANIFEST_VERSION, "remote_dir": REMOTE_DIR, "files": {}}
|
|||
|
|
return json.loads(MANIFEST.read_text(encoding="utf-8"))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def remote_listing(url: str, userpass: str) -> set[str]:
|
|||
|
|
ziel = f"ftp://{url}/{REMOTE_DIR}/"
|
|||
|
|
try:
|
|||
|
|
ausgabe = curl(["--list-only", ziel], userpass)
|
|||
|
|
except SystemExit:
|
|||
|
|
return set() # Verzeichnis existiert noch nicht
|
|||
|
|
return {z.strip() for z in ausgabe.splitlines() if z.strip() not in ("", ".", "..")}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def index_json(files: dict[str, dict]) -> str:
|
|||
|
|
"""Serverfassung des Manifests – macht das Archiv ohne Repo lesbar."""
|
|||
|
|
return json.dumps({"version": MANIFEST_VERSION, "remote_dir": REMOTE_DIR,
|
|||
|
|
"hinweis": "Pfad -> SHA-256 + Groesse; Blobs liegen unter pdf_archiv/blobs/",
|
|||
|
|
"files": dict(sorted(files.items()))},
|
|||
|
|
indent=2, ensure_ascii=False) + "\n"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def upload_index(url: str, userpass: str, files: dict[str, dict]) -> None:
|
|||
|
|
tmp = ROOT / "tools" / ".pdf_index.tmp"
|
|||
|
|
tmp.write_text(index_json(files), encoding="utf-8")
|
|||
|
|
try:
|
|||
|
|
curl(["-T", str(tmp), f"ftp://{url}/{REMOTE_INDEX}"], userpass)
|
|||
|
|
print(f"Index hochgeladen: {REMOTE_INDEX}")
|
|||
|
|
finally:
|
|||
|
|
tmp.unlink(missing_ok=True)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def fetch_index(url: str, userpass: str) -> dict:
|
|||
|
|
tmp = ROOT / "tools" / ".pdf_index.tmp"
|
|||
|
|
curl(["-o", str(tmp), f"ftp://{url}/{REMOTE_INDEX}"], userpass)
|
|||
|
|
try:
|
|||
|
|
return json.loads(tmp.read_text(encoding="utf-8"))
|
|||
|
|
finally:
|
|||
|
|
tmp.unlink(missing_ok=True)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def mb(n: int) -> str:
|
|||
|
|
return f"{n / 1048576:.1f} MB"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ------------------------------------------------------------------ Befehle
|
|||
|
|
def cmd_scan(verbose: bool) -> None:
|
|||
|
|
alt = load_manifest()["files"]
|
|||
|
|
neu: dict[str, dict] = {}
|
|||
|
|
for p in discover():
|
|||
|
|
neu[p.relative_to(ROOT).as_posix()] = {"sha256": sha256(p), "size": p.stat().st_size}
|
|||
|
|
geaendert = [k for k, v in neu.items() if alt.get(k, {}).get("sha256") not in (None, v["sha256"])]
|
|||
|
|
dazu = [k for k in neu if k not in alt]
|
|||
|
|
weg = [k for k in alt if k not in neu]
|
|||
|
|
MANIFEST.write_text(json.dumps({"version": MANIFEST_VERSION, "remote_dir": REMOTE_DIR,
|
|||
|
|
"files": dict(sorted(neu.items()))},
|
|||
|
|
indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|||
|
|
blobs = {v["sha256"] for v in neu.values()}
|
|||
|
|
gesamt = sum(v["size"] for v in neu.values())
|
|||
|
|
print(f"Manifest: {len(neu)} Dateien, {len(blobs)} Blobs, {mb(gesamt)} → {MANIFEST.name}")
|
|||
|
|
for titel, menge in (("neu", dazu), ("geändert", geaendert), ("entfernt", weg)):
|
|||
|
|
if menge:
|
|||
|
|
print(f" {titel}: {len(menge)}")
|
|||
|
|
if verbose:
|
|||
|
|
for k in menge:
|
|||
|
|
print(f" {k}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def cmd_push(verbose: bool, force: bool) -> None:
|
|||
|
|
url, user, pw = credentials()
|
|||
|
|
userpass = f"{user}:{pw}"
|
|||
|
|
files = load_manifest()["files"]
|
|||
|
|
if not files:
|
|||
|
|
die("Manifest ist leer – zuerst --scan laufen lassen")
|
|||
|
|
vorhanden = remote_listing(url, userpass)
|
|||
|
|
offen = [(k, v) for k, v in files.items()
|
|||
|
|
if force or f"{v['sha256']}.pdf" not in vorhanden]
|
|||
|
|
blobs: dict[str, int] = {}
|
|||
|
|
for k, v in offen:
|
|||
|
|
blobs.setdefault(v["sha256"], v["size"])
|
|||
|
|
print(f"Remote: {len(vorhanden)} Blobs · zu laden: {len(blobs)} Blobs "
|
|||
|
|
f"({mb(sum(blobs.values()))}) für {len(offen)} Dateien")
|
|||
|
|
for n, (digest, size) in enumerate(sorted(blobs.items()), 1):
|
|||
|
|
quelle = next((ROOT / k for k, v in files.items() if v["sha256"] == digest), None)
|
|||
|
|
if quelle is None or not quelle.exists():
|
|||
|
|
print(f" [{n}/{len(blobs)}] übersprungen (lokal nicht vorhanden): {digest[:12]}")
|
|||
|
|
continue
|
|||
|
|
if verbose or n == 1 or n == len(blobs) or n % 10 == 0:
|
|||
|
|
print(f" [{n}/{len(blobs)}] {quelle.name} ({mb(size)})")
|
|||
|
|
curl(["-T", str(quelle), f"ftp://{url}/{REMOTE_DIR}/{digest}.pdf"], userpass)
|
|||
|
|
upload_index(url, userpass, files)
|
|||
|
|
print("Upload fertig.")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def cmd_pull(verbose: bool, force: bool) -> None:
|
|||
|
|
url, user, pw = credentials()
|
|||
|
|
userpass = f"{user}:{pw}"
|
|||
|
|
files = load_manifest()["files"]
|
|||
|
|
if not files:
|
|||
|
|
print("Manifest lokal leer – hole pdf_archiv/index.json vom Server …")
|
|||
|
|
files = fetch_index(url, userpass).get("files", {})
|
|||
|
|
if files:
|
|||
|
|
MANIFEST.write_text(index_json(files), encoding="utf-8")
|
|||
|
|
print(f"Manifest aus Server-Index rekonstruiert ({len(files)} Dateien).")
|
|||
|
|
if not files:
|
|||
|
|
die("Manifest ist leer – zuerst --scan laufen lassen")
|
|||
|
|
vorhanden = remote_listing(url, userpass)
|
|||
|
|
fehlend, konflikt, geladen = [], [], 0
|
|||
|
|
for rel, meta in files.items():
|
|||
|
|
ziel = ROOT / rel
|
|||
|
|
digest = meta["sha256"]
|
|||
|
|
if ziel.exists():
|
|||
|
|
if sha256(ziel) != digest:
|
|||
|
|
konflikt.append(rel)
|
|||
|
|
if not force:
|
|||
|
|
continue
|
|||
|
|
else:
|
|||
|
|
continue
|
|||
|
|
if f"{digest}.pdf" not in vorhanden:
|
|||
|
|
fehlend.append(rel)
|
|||
|
|
continue
|
|||
|
|
ziel.parent.mkdir(parents=True, exist_ok=True)
|
|||
|
|
tmp = ziel.with_suffix(ziel.suffix + ".part")
|
|||
|
|
if verbose:
|
|||
|
|
print(f" ↓ {rel}")
|
|||
|
|
curl(["-o", str(tmp), f"ftp://{url}/{REMOTE_DIR}/{digest}.pdf"], userpass)
|
|||
|
|
if sha256(tmp) != digest:
|
|||
|
|
tmp.unlink(missing_ok=True)
|
|||
|
|
die(f"Prüfsumme falsch nach Download: {rel}")
|
|||
|
|
tmp.replace(ziel)
|
|||
|
|
geladen += 1
|
|||
|
|
print(f"Download fertig: {geladen} Dateien geladen, {len(konflikt)} Konflikte, "
|
|||
|
|
f"{len(fehlend)} fehlen auf dem Server")
|
|||
|
|
for rel in konflikt:
|
|||
|
|
print(f" KONFLIKT (lokal anders, nicht überschrieben): {rel}")
|
|||
|
|
for rel in fehlend:
|
|||
|
|
print(f" FEHLT REMOTE: {rel}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def cmd_check() -> None:
|
|||
|
|
url, user, pw = credentials()
|
|||
|
|
userpass = f"{user}:{pw}"
|
|||
|
|
files = load_manifest()["files"]
|
|||
|
|
vorhanden = remote_listing(url, userpass)
|
|||
|
|
lokal_fehlt, lokal_anders, remote_fehlt = [], [], []
|
|||
|
|
for rel, meta in files.items():
|
|||
|
|
ziel = ROOT / rel
|
|||
|
|
if not ziel.exists():
|
|||
|
|
lokal_fehlt.append(rel)
|
|||
|
|
elif sha256(ziel) != meta["sha256"]:
|
|||
|
|
lokal_anders.append(rel)
|
|||
|
|
if f"{meta['sha256']}.pdf" not in vorhanden:
|
|||
|
|
remote_fehlt.append(rel)
|
|||
|
|
ungelistet = [p.relative_to(ROOT).as_posix() for p in discover()
|
|||
|
|
if p.relative_to(ROOT).as_posix() not in files]
|
|||
|
|
print(f"Manifest: {len(files)} Dateien · Remote-Blobs: {len(vorhanden)}")
|
|||
|
|
print(f" lokal fehlt: {len(lokal_fehlt)}")
|
|||
|
|
print(f" lokal abweichend: {len(lokal_anders)}")
|
|||
|
|
print(f" remote fehlt: {len(remote_fehlt)}")
|
|||
|
|
print(f" nicht im Manifest: {len(ungelistet)}")
|
|||
|
|
for titel, menge in (("lokal fehlt", lokal_fehlt), ("lokal abweichend", lokal_anders),
|
|||
|
|
("remote fehlt", remote_fehlt),):
|
|||
|
|
for rel in menge[:20]:
|
|||
|
|
print(f" {titel}: {rel}")
|
|||
|
|
for rel in ungelistet:
|
|||
|
|
print(f" nicht im Manifest: {rel}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def cmd_prune(yes: bool, verbose: bool) -> None:
|
|||
|
|
url, user, pw = credentials()
|
|||
|
|
userpass = f"{user}:{pw}"
|
|||
|
|
gebraucht = {f"{v['sha256']}.pdf" for v in load_manifest()["files"].values()}
|
|||
|
|
vorhanden = remote_listing(url, userpass)
|
|||
|
|
ueberzaehlig = sorted(vorhanden - gebraucht)
|
|||
|
|
if not ueberzaehlig:
|
|||
|
|
print("Nichts zu löschen.")
|
|||
|
|
return
|
|||
|
|
print(f"{len(ueberzaehlig)} Blobs ohne Manifest-Eintrag:")
|
|||
|
|
for name in ueberzaehlig:
|
|||
|
|
print(f" {name}")
|
|||
|
|
if not yes:
|
|||
|
|
print("Zum Löschen mit --yes erneut aufrufen.")
|
|||
|
|
return
|
|||
|
|
for name in ueberzaehlig:
|
|||
|
|
if verbose:
|
|||
|
|
print(f" DELE {name}")
|
|||
|
|
curl(["-Q", f"DELE /{REMOTE_DIR}/{name}", f"ftp://{url}/"], userpass)
|
|||
|
|
print("Gelöscht.")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main() -> None:
|
|||
|
|
ap = argparse.ArgumentParser(description="PDF-Archiv zwischen Vault und Server syncen")
|
|||
|
|
g = ap.add_mutually_exclusive_group()
|
|||
|
|
g.add_argument("--scan", action="store_true", help="Manifest aus lokalen PDFs schreiben")
|
|||
|
|
g.add_argument("--push", action="store_true", help="fehlende Blobs hochladen")
|
|||
|
|
g.add_argument("--pull", action="store_true", help="fehlende Blobs herunterladen")
|
|||
|
|
g.add_argument("--check", action="store_true", help="Abgleich lokal/Remote")
|
|||
|
|
g.add_argument("--index", action="store_true", help="nur index.json hochladen")
|
|||
|
|
g.add_argument("--prune", action="store_true", help="Remote-Blobs ohne Manifest löschen")
|
|||
|
|
ap.add_argument("--force", action="store_true", help="vorhandene/gleichnamige Dateien überschreiben")
|
|||
|
|
ap.add_argument("--yes", action="store_true", help="Löschen bestätigen (--prune)")
|
|||
|
|
ap.add_argument("-v", "--verbose", action="store_true")
|
|||
|
|
args = ap.parse_args()
|
|||
|
|
|
|||
|
|
if args.scan or not any([args.push, args.pull, args.check, args.prune, args.index]):
|
|||
|
|
cmd_scan(args.verbose)
|
|||
|
|
if args.push:
|
|||
|
|
cmd_push(args.verbose, args.force)
|
|||
|
|
if args.pull:
|
|||
|
|
cmd_pull(args.verbose, args.force)
|
|||
|
|
if args.check:
|
|||
|
|
cmd_check()
|
|||
|
|
if args.index:
|
|||
|
|
url, user, pw = credentials()
|
|||
|
|
files = load_manifest()["files"]
|
|||
|
|
if not files:
|
|||
|
|
die("Manifest ist leer – zuerst --scan laufen lassen")
|
|||
|
|
upload_index(url, f"{user}:{pw}", files)
|
|||
|
|
if args.prune:
|
|||
|
|
cmd_prune(args.yes, args.verbose)
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|