feat(iserv): shorter output, finished tasks only with ?alle=1

Attachment text is tidied and capped at 3000 characters per file.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

iserv-filter/taskfilter.py

@@ -1,91 +1,101 @@
-"""filter_task: the one place that decides what leaves the server. Explicit whitelist, free text always filtered."""
-import io
-import zipfile
-from collections import Counter
-
-from iserv import html_text
-from nerfilter import filter_filename, filter_text, split_ext
-
-MAX_TEXT = 20_000  # chars per attachment; bounds CPU time on huge PDFs
-NOT_READABLE = "Inhalt nicht maschinell lesbar"
-_MEDIA = {"jpg": "bild", "jpeg": "bild", "png": "bild", "gif": "bild", "webp": "bild", "bmp": "bild", "heic": "bild",
-          "svg": "bild", "mp3": "audio", "wav": "audio", "m4a": "audio", "ogg": "audio", "mp4": "video",
-          "mov": "video", "avi": "video", "mkv": "video", "webm": "video"}
-
-
-def _decode(data):
-    try:
-        return data.decode("utf-8")
-    except UnicodeDecodeError:
-        return data.decode("cp1252", errors="replace")
-
-
-def _pdf(data):
-    from pypdf import PdfReader
-
-    reader = PdfReader(io.BytesIO(data))
-    if reader.is_encrypted:
-        return ""
-    text = ""
-    for page in reader.pages:
-        text += (page.extract_text() or "") + "\n"
-        if len(text) > MAX_TEXT:
-            break
-    return text
-
-
-def _docx(data):
-    with zipfile.ZipFile(io.BytesIO(data)) as z:
-        if sum(i.file_size for i in z.infolist()) > 50 * 1024 * 1024:  # zip bomb
-            return ""
-    from docx import Document
-
-    doc = Document(io.BytesIO(data))
-    return "\n".join([p.text for p in doc.paragraphs] + [c.text for t in doc.tables for r in t.rows for c in r.cells])
-
-
-def _html(data):
-    return html_text(_decode(data))
-
-
-_EXTRACT = {"pdf": _pdf, "docx": _docx, "txt": _decode, "md": _decode, "html": _html, "htm": _html}
-
-
-def describe_attachment(att, stats=None):
-    """{name, typ, groesse, text | hinweis}. The file bytes never leave this function."""
-    if att.get("skipped"):
-        return {"hinweis": "weiterer Anhang nicht abgerufen"}
-    name = att.get("name") or ""
-    ext = split_ext(name)[1]
-    out = {"name": filter_filename(name, stats), "typ": _MEDIA.get(ext) or ext or "unbekannt", "groesse": att.get("size")}
-    if att.get("failed"):
-        if stats is not None:
-            stats["attachments_failed"] += 1
-        return {**out, "hinweis": "Datei nicht abrufbar"}
-    if att.get("data") is None:
-        return {**out, "hinweis": "Datei größer als 10 MB, nicht abgerufen"}
-    if ext not in _EXTRACT:
-        return {**out, "hinweis": NOT_READABLE}
-    try:
-        text = _EXTRACT[ext](att["data"]).strip()
-    except Exception:  # corrupt/encrypted file; the message may quote its content
-        text = ""
-    if not text:
-        return {**out, "hinweis": NOT_READABLE}
-    if len(text) > MAX_TEXT:  # cut at a space so no name is split in half
-        text, out["gekuerzt"] = text[:MAX_TEXT].rsplit(None, 1)[0], True
-    return {**out, "text": filter_text(text, stats)}
-
-
-def filter_task(task, stats: Counter | None = None):
-    """Whitelist: anything not named here (creator, groups, feedback, ...) cannot appear in the output."""
-    return {
-        "id": int(task["id"]),
-        "fach": filter_text(task.get("tags", ""), stats),
-        "titel": filter_text(task.get("title", ""), stats),
-        "beschreibung": filter_text(task.get("description", ""), stats),
-        "start": task.get("start"),
-        "abgabe": task.get("due"),
-        "status": None if "done" not in task else "erledigt" if task["done"] else "offen",  # None: source cannot tell
-        "anhaenge": [describe_attachment(a, stats) for a in task.get("attachments", [])],
-    }
+"""filter_task: the one place that decides what leaves the server. Explicit whitelist, free text always filtered."""
+import io
+import zipfile
+from collections import Counter
+
+from iserv import html_text
+from nerfilter import filter_filename, filter_text, split_ext
+
+READ_LIMIT = 20_000  # chars read from a file; bounds CPU time on huge PDFs
+MAX_TEXT = 3_000  # chars of a file that go out: worksheets are long and Muse needs the gist
+NOT_READABLE = "Inhalt nicht maschinell lesbar"
+_MEDIA = {"jpg": "bild", "jpeg": "bild", "png": "bild", "gif": "bild", "webp": "bild", "bmp": "bild", "heic": "bild",
+          "svg": "bild", "mp3": "audio", "wav": "audio", "m4a": "audio", "ogg": "audio", "mp4": "video",
+          "mov": "video", "avi": "video", "mkv": "video", "webm": "video"}
+
+
+def _decode(data):
+    try:
+        return data.decode("utf-8")
+    except UnicodeDecodeError:
+        return data.decode("cp1252", errors="replace")
+
+
+def _pdf(data):
+    from pypdf import PdfReader
+
+    reader = PdfReader(io.BytesIO(data))
+    if reader.is_encrypted:
+        return ""
+    text = ""
+    for page in reader.pages:
+        text += (page.extract_text() or "") + "\n"
+        if len(text) > READ_LIMIT:
+            break
+    return text
+
+
+def _docx(data):
+    with zipfile.ZipFile(io.BytesIO(data)) as z:
+        if sum(i.file_size for i in z.infolist()) > 50 * 1024 * 1024:  # zip bomb
+            return ""
+    from docx import Document
+
+    doc = Document(io.BytesIO(data))
+    return "\n".join([p.text for p in doc.paragraphs] + [c.text for t in doc.tables for r in t.rows for c in r.cells])
+
+
+def _html(data):
+    return html_text(_decode(data))
+
+
+_EXTRACT = {"pdf": _pdf, "docx": _docx, "txt": _decode, "md": _decode, "html": _html, "htm": _html}
+
+
+def _tidy(text):
+    """Drops what says nothing: blank lines, page numbers, a line identical to the one before (running headers)."""
+    out = []
+    for line in (l.strip() for l in text.splitlines()):
+        if line and not line.isdigit() and (not out or line != out[-1]):
+            out.append(line)
+    return "\n".join(out)
+
+
+def describe_attachment(att, stats=None):
+    """{name, typ, groesse, text | hinweis}. The file bytes never leave this function."""
+    if att.get("skipped"):
+        return {"hinweis": "weiterer Anhang nicht abgerufen"}
+    name = att.get("name") or ""
+    ext = split_ext(name)[1]
+    out = {"name": filter_filename(name, stats), "typ": _MEDIA.get(ext) or ext or "unbekannt", "groesse": att.get("size")}
+    if att.get("failed"):
+        if stats is not None:
+            stats["attachments_failed"] += 1
+        return {**out, "hinweis": "Datei nicht abrufbar"}
+    if att.get("data") is None:
+        return {**out, "hinweis": "Datei größer als 10 MB, nicht abgerufen"}
+    if ext not in _EXTRACT:
+        return {**out, "hinweis": NOT_READABLE}
+    try:
+        text = _tidy(_EXTRACT[ext](att["data"]))
+    except Exception:  # corrupt/encrypted file; the message may quote its content
+        text = ""
+    if not text:
+        return {**out, "hinweis": NOT_READABLE}
+    if len(text) > MAX_TEXT:  # cut at a space so no name is split in half
+        text, out["gekuerzt"] = text[:MAX_TEXT].rsplit(None, 1)[0], True
+    return {**out, "text": filter_text(text, stats)}
+
+
+def filter_task(task, stats: Counter | None = None):
+    """Whitelist: anything not named here (creator, groups, feedback, ...) cannot appear in the output."""
+    return {
+        "id": int(task["id"]),
+        "fach": filter_text(task.get("tags", ""), stats),
+        "titel": filter_text(task.get("title", ""), stats),
+        "beschreibung": filter_text(task.get("description", ""), stats),
+        "start": task.get("start"),
+        "abgabe": task.get("due"),
+        "status": None if "done" not in task else "erledigt" if task["done"] else "offen",  # None: source cannot tell
+        "anhaenge": [describe_attachment(a, stats) for a in task.get("attachments", [])],
+    }

src/api/iserv-filtered.js

@@ -1,4 +1,4 @@
-// GET /iserv/tasks: the open IServ tasks with person, place and school names removed, for the assistant "Muse".
+// GET /iserv/tasks (?alle=1 adds finished tasks): the open IServ tasks with person, place and school names removed, for the assistant "Muse".
 // IServ credentials come from the Space secrets (never from the request). The raw tasks stay in memory: the fetcher
 // (iserv.js) reads them, the filter service (iserv-filter/, Python, 127.0.0.1 only, own unix user without the IServ
 // secrets) cleans them, only its answer leaves. Nothing is stored, cached or logged.
@@ -16,12 +16,13 @@ function tokenOk(req) {
   return offered.some((o) => o && Buffer.byteLength(o) === expected.length && timingSafeEqual(Buffer.from(o), expected));
 }
 
-async function cleanTasks() {
+async function cleanTasks(includeDone) {
   const { ISERV_BASE_URL: url, ISERV_USERNAME: user, ISERV_PASSWORD: pass } = process.env;
   if (!url || !user || !pass || !url.startsWith('https://')) return { status: 503, body: { error: 'config' } };
   let raw;
   try {
     raw = await fetchIServTasks(url, user, pass);
+    if (!includeDone) raw = raw.filter((t) => t.done !== true); // finished tasks only with ?alle=1
   } catch {
     return { status: 502, body: { error: 'unreachable' } }; // the message may quote the school domain
   }
@@ -44,7 +45,8 @@ export function registerIServFiltered(app) {
     res.set('Cache-Control', 'no-store');
     if ((process.env.API_BEARER_TOKEN || '').length < 16) return res.status(503).json({ error: 'config' }); // fail closed
     if (!tokenOk(req)) return res.status(401).json({ error: 'unauthorized' });
-    const job = running.then(cleanTasks, cleanTasks);
+    const run = () => cleanTasks(req.query.alle === '1');
+    const job = running.then(run, run);
     running = job.catch(() => {});
     const { status, body } = await job;
     res.status(status).json(body);