From 17f5ef85608aa47ea67d1c9f058f3e7bf626285a Mon Sep 17 00:00:00 2001 From: Ludwig Lehnert Date: Fri, 31 Jul 2026 20:23:52 +0000 Subject: [PATCH] webui (3) --- .env.example | 1 + Dockerfile | 1 + README.md | 16 +++++---- app/audit_collector.py | 35 ++++++------------- app/audit_policy.py | 53 ++++++++++++++++++++++++++++ app/web/app.js | 13 ++++--- app/web_ui.py | 26 +++++++++++--- dev/ad-entrypoint.sh | 3 +- dev/e2e.py | 34 +++++++++++++++--- dev/preview-client.sh | 4 ++- etc/samba/smb.conf | 12 +++---- setup | 1 + tests/test_web_ui.py | 79 ++++++++++++++++++++++++++++++++++++++++-- 13 files changed, 222 insertions(+), 56 deletions(-) create mode 100644 app/audit_policy.py diff --git a/.env.example b/.env.example index bd13d7f..c214bc2 100644 --- a/.env.example +++ b/.env.example @@ -50,3 +50,4 @@ ACME_HTTP_PORT=80 # BACKUP_STATUS_FILE=/state/backup-status.json # AUDIT_COMPRESS_AFTER_HOURS=24 # AUDIT_QUERY_MAX_DAYS=31 +# AUDIT_SKIP_USER_SUFFIXES=_svc,_ServiceAcc diff --git a/Dockerfile b/Dockerfile index 39dac2e..b6d194f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,7 @@ COPY --from=step-cli /usr/local/bin/step /usr/local/bin/step COPY app/reconcile_shares.py /app/reconcile_shares.py COPY app/backup_to_destination.py /app/backup_to_destination.py +COPY app/audit_policy.py /app/audit_policy.py COPY app/audit_collector.py /app/audit_collector.py COPY app/web_ui.py /app/web_ui.py COPY app/web /app/web diff --git a/README.md b/README.md index b871cbc..685f3a3 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ This repository provides a production-oriented Samba file server container that - Setup prompts for well-known authorization groups by SID (`DOMAIN_USERS_SID`, `DOMAIN_ADMINS_SID`) to avoid localized group names. - `FSLOGIX_GROUP_SID` controls who can access the default FSLogix share (defaults to `DOMAIN_USERS_SID`). - Startup resolves those SIDs to NSS group names via winbind, then uses those resolved groups in Samba `valid users` rules. -- Share operations are audited with Samba `full_audit` (connect, list, read, write, create, delete, rename). -- A collector persists every `full_audit` record in daily NDJSON files under `/state/audit`; closed files are gzip-compressed and are never automatically deleted. +- Samba `full_audit` is restricted to successful and failed reads, writes, renames/moves, and deletions. +- A collector normalizes those four actions and persists them in daily NDJSON files under `/state/audit`; closed files are gzip-compressed and are never automatically deleted. - A read-only HTTPS web console provides group membership trees, storage usage, searchable activity, live backup progress, and system health. - Web sign-in validates the submitted username/password with Kerberos, permits only users whose winbind group SID set contains `DOMAIN_ADMINS_SID`, and issues an expiring JWT in a Secure, HttpOnly, SameSite=Strict cookie. The browser does not use NTLM/SPNEGO or Kerberos negotiation. - HTTPS certificates are requested from a configured local Smallstep CA and renewed automatically. Pre-issued certificate files are also supported. @@ -121,7 +121,7 @@ The launcher builds the current application and starts an isolated, run-scoped n - a real Smallstep CA whose ACME provisioner issues the web UI certificate for `files.localhost`; - an authenticated rsync daemon used by the normal backup implementation; - the actual file-server image, joined to the dummy domain; -- a continuous SMB client that reads, writes, and lists files as several domain users. +- a continuous SMB client that exercises reads, writes, renames/moves, and deletions, including activity from an excluded dummy service account. The preview starts with group, Private, FSLogix, and historical audit data. The client keeps current-day audit activity moving, while a real backup runs immediately and repeats in the background. Open the URL and use the credentials printed by the launcher. Defaults are: @@ -174,7 +174,7 @@ The E2E suite verifies: - domain trust, group-to-folder mapping, nested and transitive group trees; - SMB allow/deny behavior and real file operations; - Data, Private, and FSLogix usage aggregation; -- live `full_audit` ingestion, filters, facets, and pagination; +- high-level `full_audit` ingestion for all four actions, service-account exclusion, filters, facets, and pagination; - compression and querying of a closed daily audit log; - real rsync transfer progress, completed backup status, log output, and remote snapshot marker; - overview and system-health aggregation. @@ -374,20 +374,22 @@ Useful optional settings: | `WEB_MAX_GROUP_NODES` | `10000` | Membership expansion safety limit | | `AUDIT_COMPRESS_AFTER_HOURS` | `24` | Minimum idle age before compressing a closed day | | `AUDIT_QUERY_MAX_DAYS` | `31` | Largest activity query window | +| `AUDIT_SKIP_USER_SUFFIXES` | `_svc,_ServiceAcc` | Case-insensitive account suffixes excluded from collection and queries; set empty to disable | If `WEB_ENABLED` is absent on an upgraded installation and no TLS settings/certificate exist, the web service stays disabled while Samba and audit collection continue. Set `WEB_ENABLED=true` after adding TLS configuration. ## Audit Archive -Samba already produced `full_audit` messages before this web UI was added. The collector now makes them durable: +Samba emits only the selected high-level `full_audit` operations, and the collector makes them durable: - it tails every `/var/log/samba/log.*` source, remembers inode and byte offsets in `/state/audit/collector-state.json`, and follows Samba rotation without duplicating a rotated file; -- each event records timestamp, user, client address/name, share, VFS operation, read/write/list/session category, result, and path; +- each event records timestamp, user, client address/name, share, result, path, and one of the actions `read`, `write`, `move`, or `delete`; +- directory listings, sessions, metadata access, file-open/create noise, and all other VFS operations are discarded; users ending in a configured `AUDIT_SKIP_USER_SUFFIXES` value are also discarded; - records are appended to `/state/audit/YYYY-MM-DD.jsonl`; - a closed, idle daily file becomes `.jsonl.gz`; - no audit retention deletion is performed, so capacity planning for the `state_data` volume is the operator's responsibility. -Collection starts even when the web UI is disabled. Existing Samba log content is imported when the collector first starts, but audit data that Samba rotated away before this version was deployed cannot be recovered. +Collection starts even when the web UI is disabled. Existing Samba log content is imported when the collector first starts, but the same operation and user-suffix policy is applied during import. Audit data that Samba rotated away before this version was deployed cannot be recovered. ## Backups diff --git a/app/audit_collector.py b/app/audit_collector.py index db57238..1dd0e01 100644 --- a/app/audit_collector.py +++ b/app/audit_collector.py @@ -12,6 +12,11 @@ import sys import time from typing import Dict, Iterable, Optional +try: + from .audit_policy import action_for, skip_user +except ImportError: + from audit_policy import action_for, skip_user + SAMBA_LOG_GLOB = os.getenv("AUDIT_SOURCE_GLOB", "/var/log/samba/log.*") ARCHIVE_DIR = os.getenv("AUDIT_ARCHIVE_DIR", "/state/audit") @@ -69,28 +74,6 @@ def parse_samba_timestamp(raw_line: str, fallback: dt.datetime) -> str: return fallback.isoformat(timespec="milliseconds") -def action_for(operation: str) -> str: - operation = operation.lower() - if operation in { - "read", "pread", "pread_recv", "pread_send", "recvfile", "sendfile", - "offload_read_recv", "offload_read_send", - }: - return "read" - if operation in { - "write", "pwrite", "pwrite_recv", "pwrite_send", "ftruncate", - "fallocate", "create_file", "mkdirat", "mknodat", "renameat", - "unlinkat", "symlinkat", "linkat", "offload_write_recv", - "offload_write_send", "fsetxattr", "removexattr", "fremovexattr", - "mkdir", "rmdir", "rename", "unlink", - }: - return "write" - if operation in {"opendir", "fdopendir", "readdir", "freaddir_attr", "closedir"}: - return "list" - if operation in {"connect", "disconnect"}: - return "session" - return "metadata" - - def parse_audit_line(raw_line: str, source: str) -> Optional[Dict[str, object]]: match = AUDIT_MARKER_RE.search(raw_line) if match: @@ -104,16 +87,20 @@ def parse_audit_line(raw_line: str, source: str) -> Optional[Dict[str, object]]: return None observed_at = utc_now() operation = fields[5].strip() + action = action_for(operation) + user = fields[1].strip() + if action is None or skip_user(user): + return None result = fields[6].strip() return { "timestamp": parse_samba_timestamp(raw_line, observed_at), "ingestedAt": observed_at.isoformat(timespec="milliseconds"), - "user": fields[1].strip(), + "user": user, "clientIp": fields[2].strip(), "client": fields[3].strip(), "share": fields[4].strip(), "operation": operation, - "action": action_for(operation), + "action": action, "result": result, "success": result.upper() == "OK", "path": "|".join(fields[7:]).strip(), diff --git a/app/audit_policy.py b/app/audit_policy.py new file mode 100644 index 0000000..91eacd0 --- /dev/null +++ b/app/audit_policy.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Shared policy for the small set of user-facing audit events.""" + +import os +from typing import Optional, Tuple + + +AUDIT_ACTIONS = frozenset({"read", "write", "move", "delete"}) + +OPERATION_ACTIONS = { + "read": "read", + "pread": "read", + "pread_recv": "read", + "pread_send": "read", + "sendfile": "read", + "offload_read_recv": "read", + "offload_read_send": "read", + "write": "write", + "pwrite": "write", + "pwrite_recv": "write", + "pwrite_send": "write", + "recvfile": "write", + "offload_write_recv": "write", + "offload_write_send": "write", + "renameat": "move", + "rename": "move", + "unlinkat": "delete", + "unlink": "delete", + "rmdir": "delete", +} + + +def action_for(operation: str) -> Optional[str]: + return OPERATION_ACTIONS.get(operation.strip().casefold()) + + +def skipped_user_suffixes() -> Tuple[str, ...]: + raw = os.getenv("AUDIT_SKIP_USER_SUFFIXES", "_svc,_ServiceAcc") + return tuple( + suffix.strip().casefold() + for suffix in raw.split(",") + if suffix.strip() + ) + + +def account_name(user: str) -> str: + account = user.strip().rsplit("\\", 1)[-1] + return account.split("@", 1)[0] + + +def skip_user(user: str) -> bool: + account = account_name(user).casefold() + return any(account.endswith(suffix) for suffix in skipped_user_suffixes()) diff --git a/app/web/app.js b/app/web/app.js index cb556ac..422febf 100644 --- a/app/web/app.js +++ b/app/web/app.js @@ -24,7 +24,7 @@ const utcTime = value => { const pad = number => String(number).padStart(2, "0"); return `${pad(date.getUTCDate())}.${pad(date.getUTCMonth() + 1)}.${date.getUTCFullYear()} ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())} UTC`; }; -const actionLabel = value => ({read: "Lesen", write: "Schreiben", list: "Auflisten", metadata: "Metadaten", session: "Sitzung"}[value] || value || "—"); +const actionLabel = value => ({read: "Lesen", write: "Schreiben", move: "Umbenennen/Verschieben", delete: "Löschen"}[value] || value || "—"); const nodeTypeLabel = value => ({group: "Gruppe", user: "Benutzer", computer: "Computer", unknown: "Unbekannt"}[value] || value); const backupStateLabel = value => ({starting: "Startet", running: "Läuft", completed: "Abgeschlossen", failed: "Fehlgeschlagen", waiting: "Wartet", disabled: "Deaktiviert"}[value] || value || "Unbekannt"); function backupMessage(data) { @@ -127,7 +127,7 @@ function eventRows(events) { return events.map(event => ` ${esc(utcTime(event.timestamp))} ${esc(event.user)}${esc(event.clientIp)}${esc(event.share)} - ${badge(actionLabel(event.action || event.operation))}
${esc(event.operation)} + ${badge(actionLabel(event.action || event.operation))} ${esc(event.path || "—")} ${event.success ? badge("Erfolgreich") : badge(event.result || "Fehlgeschlagen", "error")} `).join(""); @@ -230,20 +230,19 @@ async function renderStorage(type) { async function renderActivity() { const today = new Date(); const yesterday = new Date(Date.now() - 86400000); - content.innerHTML = pageHead("Aktivitätsprotokoll", "Aufgezeichnete Samba-Vorgänge nach Datum, Identität, Freigabe, Aktion, Ergebnis oder Pfad durchsuchen.") + ` + content.innerHTML = pageHead("Aktivitätsprotokoll", "Lese-, Schreib-, Umbenennungs-/Verschiebe- und Löschvorgänge nach Datum, Identität, Freigabe, Ergebnis oder Pfad durchsuchen.") + `
- + -
- +

Zeit (UTC)BenutzerClientFreigabeAktionPfadErgebnis

@@ -262,7 +261,7 @@ async function renderActivity() { const more = document.querySelector("#load-more"); cursor = result.nextCursor || 0; more.hidden = !result.nextCursor; - for (const [id, values] of [["users-list", result.facets.users], ["shares-list", result.facets.shares], ["operations-list", result.facets.operations]]) { + for (const [id, values] of [["users-list", result.facets.users], ["shares-list", result.facets.shares]]) { document.querySelector(`#${id}`).innerHTML = values.filter(Boolean).map(value => `