diff --git a/.env.example b/.env.example index d8a3efb..dbb0041 100644 --- a/.env.example +++ b/.env.example @@ -31,6 +31,7 @@ ACME_HTTP_PORT=80 # WEB_DIRECTORY_CACHE_SECONDS=300 # WEB_MAX_GROUP_NODES=10000 # WEB_LOGIN_ATTEMPTS_PER_5_MIN=10 +# TRASH_RETENTION_DAYS=7 # LDAP_URI=ldaps://example.com # LDAP_BASE_DN=DC=example,DC=com # PRIVATE_SKIP_USERS=svc_backup,svc_sql diff --git a/Dockerfile b/Dockerfile index b9be259..7864df8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM smallstep/step-cli:0.30.2 AS step-cli +FROM docker.io/smallstep/step-cli:0.30.2 AS step-cli -FROM debian:12-slim +FROM docker.io/library/debian:12-slim ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC @@ -33,6 +33,7 @@ COPY app/audit_policy.py /app/audit_policy.py COPY app/state_db.py /app/state_db.py COPY app/audit_store.py /app/audit_store.py COPY app/audit_collector.py /app/audit_collector.py +COPY app/trash.py /app/trash.py COPY app/web_ui.py /app/web_ui.py COPY app/web /app/web COPY app/init.sh /app/init.sh diff --git a/README.md b/README.md index d3090f9..b559b71 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,8 @@ This repository provides a production-oriented Samba file server container that - Startup resolves those SIDs to NSS group names via winbind, then uses those resolved groups in Samba `valid users` rules. - Samba `full_audit` records successful and failed reads, writes, renames, and deletions on all three shares; FSLogix events are retained in a separate indexed activity stream. - A collector normalizes those four actions and persists them in indexed SQLite tables; activity is never automatically deleted. -- A plain HTTPS administration console provides read-only statistics and logs plus narrowly scoped actions for manual backups and share reconciliation. It also includes a fully client-side Typst PDF report. +- Samba retains deleted files for seven days in per-user recycle repositories on the same data volumes. +- A plain HTTPS administration console provides statistics and logs plus narrowly scoped actions for trash downloads/restores, manual backups, and share reconciliation. It also includes a fully client-side Typst PDF report. - Web sign-in validates the submitted username/password with Kerberos, permits only users whose winbind group SID set contains `DOMAIN_ADMINS_SID`, and issues an expiring JWT in a Secure, HttpOnly, SameSite=Strict cookie. The browser does not use NTLM/SPNEGO or Kerberos negotiation. - HTTPS certificates are requested from a configured local Smallstep CA and renewed automatically. Pre-issued certificate files are also supported. - Optional remote backups run when `BACKUP_DESTINATION` and `BACKUP_ARCHIVE_PASSWORD` are configured; each active or archived group folder is uploaded as its own encrypted, non-solid 7z archive. @@ -54,7 +55,7 @@ The database contains: - `shares`: AD group-to-folder lifecycle and ACL reconciliation state; - `audit_events`: normalized read, write, move, and delete events; - `audit_sources`: Samba log inode/offset checkpoints; -- `audit_read_dedup`: bounded, persistent fingerprints for restart-safe read deduplication; +- `audit_event_dedup`: bounded, persistent fingerprints for restart-safe main-read and FSLogix-event deduplication; - `audit_daily_totals`, `audit_daily_counts`, and `audit_daily_facets`: compact materialized metadata for fast activity counts and filters; - `audit_paths`, `audit_paths_fts`, and `audit_path_events`: deduplicated trigram path search with an incrementally maintained event mapping; - `audit_rollup_state`: bounded legacy-event backfill progress; @@ -181,7 +182,8 @@ The E2E suite verifies: - SMB allow/deny behavior and real file operations; - Data, Private, and FSLogix usage aggregation; - high-level `full_audit` ingestion for all four actions, service-account exclusion, filters, facets, and pagination; -- shared SQLite schema, integrity, indexes, legacy-log removal, and read deduplication across interleaved events and collector polls; +- shared SQLite schema, integrity, indexes, legacy-log removal, and main-read and FSLogix-event deduplication across interleaved events and collector polls; +- real Samba recycle handling plus authenticated trash listing, streamed download, and restore; - real rsync transfer progress, completed backup status, log output, remote snapshot marker, and per-group encrypted non-solid 7z archives; - anonymous action rejection plus authenticated manual backup and reconciliation actions, terminal progress, and live reconciliation output; - overview and system-health aggregation; @@ -286,6 +288,14 @@ The runner returns non-zero on the first failed assertion, prints bounded logs f - Semantics intentionally differ from `Data`: only the share root is reconciled (`03770` + ACL defaults), while user-created profile container folders/files are not recursively normalized. - Samba masks are profile-container oriented (`create mask = 0600`, `directory mask = 0700`) so profile payload stays user-private by default. +## Seven-Day Trash + +Deletes through the `Private`, `Data`, and `FSLogix` SMB shares are intercepted by Samba's recycle VFS and moved into `.trash/` on the same source volume. Moving on the same filesystem avoids copying even large profile containers. The original directory tree is retained, repeated deletions receive versioned names, and the deletion time is stored as the recycled file's modification time. + +The repository root is owned by root, vetoed from SMB access, and not included in per-user/per-group usage rows. Every hour—and once during container startup—the cleanup job permanently removes entries older than seven days. `TRASH_RETENTION_DAYS` defaults to `7` and may be set from `1` to `365`. + +Domain Admins can use **Papierkorb** in the web console to filter and list retained files, stream-download them, or restore them to their original path. Restore is a same-filesystem link/unlink operation, so it is fast for large files and preserves file metadata. It never overwrites an existing file; a conflict is reported and the retained copy remains in the bin. + ## Web Administration Console Open `https:///` after setup. Only members of the group identified by `DOMAIN_ADMINS_SID` can sign in. The form accepts `DOMAIN\username`, `username@realm`, or an unqualified username (which is qualified with `WORKGROUP`). @@ -297,6 +307,7 @@ The console is intentionally operational and plain: - **Data usage**: cached recursive size of every top-level `/Data` group folder. - **User usage**: per-user `/Private + /FSLogix` totals with component sizes. - **Activity**: dynamic date, user, share, action, result, and path filters with pagination. +- **Trash**: list seven-day recycle entries across all shares, download a retained file, or restore it without overwriting an existing path. - **Share reconciliation**: manually force reconciliation and follow its phase, progress bar, current group, and live output. - **Backups**: manually start a backup and follow live progress, active transfer rows, snapshot name, trigger, and recent output. - **PDF report**: storage totals and every storage row, complete group/folder membership hierarchies, current backup state, system checks, and TLS certificate data. @@ -304,7 +315,7 @@ The console is intentionally operational and plain: The PDF report deliberately excludes the activity log and backup log. Its dedicated snapshot endpoint removes those fields before returning data. Typst, its WebAssembly compiler, and the report fonts are shipped with the application; Typst source and PDF bytes are created only in the authenticated browser and are never uploaded to another service. The first export downloads roughly 22 MiB of compiler/font assets, which are then cached as immutable files. The CSP grants only `'wasm-unsafe-eval'` for WebAssembly compilation and does not enable JavaScript `'unsafe-eval'`. -The only operational mutation endpoints start an immediate backup or share reconciliation, and both require the same Domain Admin JWT as every protected page. The console cannot edit files, groups, ACL rules, backup schedules, retention, or `BACKUP_AUTO_ENABLED`. Automatic backups can be enabled or disabled only through the environment and therefore require a redeployment/restart. +The operational mutation endpoints restore a retained file or start an immediate backup/share reconciliation; all require the same Domain Admin JWT as every protected page. Restore cannot overwrite a live file. The console cannot otherwise edit files, groups, ACL rules, backup schedules, retention, or `BACKUP_AUTO_ENABLED`. Automatic backups can be enabled or disabled only through the environment and therefore require a redeployment/restart. ### Authentication and sessions @@ -386,6 +397,7 @@ Useful optional settings: | `WEB_USAGE_SCAN_INTERVAL_SECONDS` | `900` | Recursive storage scan interval | | `WEB_DIRECTORY_CACHE_SECONDS` | `300` | AD membership tree cache time | | `WEB_MAX_GROUP_NODES` | `10000` | Membership expansion safety limit | +| `TRASH_RETENTION_DAYS` | `7` | Recycled-file lifetime; cleanup accepts 1 to 365 days | | `STATE_DB_PATH` | `/state/shares.db` | Shared SQLite database for shares, activity, collector offsets, and caches | | `AUDIT_QUERY_MAX_DAYS` | `31` | Largest activity query window | | `AUDIT_SKIP_USER_SUFFIXES` | `_svc,_ServiceAcc` | Case-insensitive account suffixes excluded from collection and queries; set empty to disable | @@ -400,7 +412,7 @@ Samba emits selected high-level `full_audit` operations for Private, Data, and F - each event records a UTC timestamp, user, client address/name, share, result, path, and one of `read`, `write`, `move`, or `delete`; - directory listings, sessions, metadata access, file-open/create noise, and all other VFS operations are discarded; users ending in a configured `AUDIT_SKIP_USER_SUFFIXES` value are also discarded; - FSLogix profile-container events are retained and queried through their own partial indexes, API route, and UI log instead of appearing in the main activity stream; -- identical reads within the same UTC second are collapsed into one event regardless of intervening events, log source, or collector polling cycle; the persistent fingerprint cache covers the latest 48 hours and can be tuned with `AUDIT_READ_DEDUP_WINDOW_SECONDS`; +- identical main-stream reads and identical FSLogix events within the same UTC second are collapsed regardless of intervening events, log source, or collector polling cycle; Data/Private writes remain distinct, and the 48-hour fingerprint window can be tuned with `AUDIT_DEDUP_WINDOW_SECONDS`; - activity pages read only `limit + 1` indexed rows and use a stable time/id cursor; - exact counts for date, user, share, action, and result filters and all facet lists come from daily rollups; - selective substring path searches use the deduplicated trigram index and skip an expensive exact count while more pages exist; @@ -408,7 +420,7 @@ Samba emits selected high-level `full_audit` operations for Private, Data, and F - collector inserts and rollup updates are batched in one transaction; - no activity retention deletion is performed. -Collection starts even when the web UI is disabled. Existing databases remain queryable while the collector backfills rollups, path IDs, path-event mappings, and recent read deduplication in bounded chunks after prioritizing each live append. On the first collector start after the legacy archive migration, recognized daily `.jsonl`/`.jsonl.gz` files and the old collector state file under `/state/audit` are deleted without import. Existing raw Samba log content is then processed using the current action, suffix, and deduplication policy. +Collection starts even when the web UI is disabled. Existing databases remain queryable while the collector backfills rollups, path IDs, path-event mappings, and recent main-read and FSLogix-event deduplication in bounded chunks after prioritizing each live append. On the first collector start after the legacy archive migration, recognized daily `.jsonl`/`.jsonl.gz` files and the old collector state file under `/state/audit` are deleted without import. Existing raw Samba log content is then processed using the current action, suffix, and deduplication policy. ## Backups diff --git a/app/audit_policy.py b/app/audit_policy.py index 3d373ba..ffc1a57 100644 --- a/app/audit_policy.py +++ b/app/audit_policy.py @@ -49,10 +49,12 @@ def skip_user(user: str) -> bool: account = account_name(user).casefold() return any(account.endswith(suffix) for suffix in skipped_user_suffixes()) -ReadEventKey = Tuple[str, ...] +DeduplicationKey = Tuple[str, ...] -def read_deduplication_key(event: Mapping[str, object]) -> Optional[ReadEventKey]: - if str(event.get("action", "")).casefold() != "read": +def deduplication_key(event: Mapping[str, object]) -> Optional[DeduplicationKey]: + action = str(event.get("action", "")).casefold() + share = str(event.get("share", "")) + if action != "read" and share.casefold() != "fslogix": return None try: timestamp = dt.datetime.fromisoformat( @@ -71,18 +73,19 @@ def read_deduplication_key(event: Mapping[str, object]) -> Optional[ReadEventKey success = bool(event.get("success", False)) return ( second, + action, str(event.get("user", "")), str(event.get("clientIp", "")), - str(event.get("share", "")), + share, str(event.get("path", "")), "success" if success else "failure", "" if success else str(event.get("result", "")), ) -def read_deduplication_fingerprint( +def deduplication_fingerprint( event: Mapping[str, object] ) -> Optional[bytes]: - key = read_deduplication_key(event) + key = deduplication_key(event) if key is None: return None digest = hashlib.blake2b(digest_size=16) @@ -92,8 +95,9 @@ def read_deduplication_fingerprint( digest.update(encoded) return digest.digest() -def read_deduplication_fingerprint_values( +def deduplication_fingerprint_values( timestamp: object, + action: object, user: object, client_ip: object, share: object, @@ -101,9 +105,9 @@ def read_deduplication_fingerprint_values( success: object, result: object, ) -> bytes: - """Fingerprint legacy SQLite columns with the live-ingest policy.""" - fingerprint = read_deduplication_fingerprint({ - "action": "read", + """Fingerprint SQLite columns with the live-ingest policy.""" + fingerprint = deduplication_fingerprint({ + "action": action, "timestamp": timestamp, "user": user, "clientIp": client_ip, diff --git a/app/audit_store.py b/app/audit_store.py index a736e54..0d17182 100644 --- a/app/audit_store.py +++ b/app/audit_store.py @@ -11,16 +11,16 @@ from typing import Dict, Iterable, List, Optional, Set, Tuple try: from .audit_policy import ( account_name, - read_deduplication_fingerprint, - read_deduplication_fingerprint_values, + deduplication_fingerprint, + deduplication_fingerprint_values, skipped_user_suffixes, ) from .state_db import connect_state_db except ImportError: from audit_policy import ( account_name, - read_deduplication_fingerprint, - read_deduplication_fingerprint_values, + deduplication_fingerprint, + deduplication_fingerprint_values, skipped_user_suffixes, ) from state_db import connect_state_db @@ -49,13 +49,13 @@ CREATE TABLE IF NOT EXISTS audit_sources ( inode INTEGER NOT NULL, offset INTEGER NOT NULL ) WITHOUT ROWID; -CREATE TABLE IF NOT EXISTS audit_read_dedup ( +CREATE TABLE IF NOT EXISTS audit_event_dedup ( fingerprint BLOB PRIMARY KEY, occurred_second INTEGER NOT NULL, event_id INTEGER ) WITHOUT ROWID; -CREATE INDEX IF NOT EXISTS audit_read_dedup_time - ON audit_read_dedup (occurred_second); +CREATE INDEX IF NOT EXISTS audit_event_dedup_time + ON audit_event_dedup (occurred_second); CREATE INDEX IF NOT EXISTS audit_events_main_time ON audit_events (occurred_second DESC, id DESC) WHERE share <> 'FSLogix' COLLATE NOCASE; @@ -147,7 +147,7 @@ CREATE TABLE IF NOT EXISTS audit_rollup_state ( backfill_next_id INTEGER NOT NULL, backfill_max_id INTEGER NOT NULL, ready INTEGER NOT NULL CHECK (ready IN (0, 1)), - policy_version INTEGER NOT NULL DEFAULT 4 + policy_version INTEGER NOT NULL DEFAULT 5 ); """ @@ -174,6 +174,7 @@ VALUES (?, ?, ?, ?) def ensure_audit_schema(conn: sqlite3.Connection) -> None: + conn.execute("DROP TABLE IF EXISTS audit_read_dedup") conn.executescript(AUDIT_SCHEMA) for legacy_index in ( "audit_events_time", @@ -221,7 +222,8 @@ def ensure_audit_schema(conn: sqlite3.Connection) -> None: WHERE singleton = 1 """ ).fetchone() - if state is not None and int(state["policy_version"]) < 4: + if state is not None and int(state["policy_version"]) < 5: + conn.execute("DELETE FROM audit_event_dedup") for table in ( "audit_daily_totals", "audit_daily_counts", @@ -232,7 +234,7 @@ def ensure_audit_schema(conn: sqlite3.Connection) -> None: """ UPDATE audit_rollup_state SET backfill_next_id = 1, backfill_max_id = ?, ready = ?, - policy_version = 4 + policy_version = 5 WHERE singleton = 1 """, (max_id, int(max_id == 0)), @@ -240,17 +242,21 @@ def ensure_audit_schema(conn: sqlite3.Connection) -> None: conn.commit() -READ_DEDUP_DEFAULT_WINDOW_SECONDS = 2 * 86400 +DEDUP_DEFAULT_WINDOW_SECONDS = 2 * 86400 -def read_deduplication_cutoff() -> int: - try: - window = int(os.getenv( +def deduplication_cutoff() -> int: + configured = os.getenv( + "AUDIT_DEDUP_WINDOW_SECONDS", + os.getenv( "AUDIT_READ_DEDUP_WINDOW_SECONDS", - str(READ_DEDUP_DEFAULT_WINDOW_SECONDS), - )) + str(DEDUP_DEFAULT_WINDOW_SECONDS), + ), + ) + try: + window = int(configured) except ValueError: - window = READ_DEDUP_DEFAULT_WINDOW_SECONDS + window = DEDUP_DEFAULT_WINDOW_SECONDS now = int(dt.datetime.now(dt.timezone.utc).timestamp()) return now - max(1, window) @@ -285,8 +291,8 @@ class AuditStore: def __init__(self, database_path: Optional[str] = None): self.conn = connect_state_db(database_path) self.conn.create_function( - "audit_read_fingerprint", 7, - read_deduplication_fingerprint_values, deterministic=True, + "audit_event_fingerprint", 8, + deduplication_fingerprint_values, deterministic=True, ) ensure_audit_schema(self.conn) @@ -314,12 +320,12 @@ class AuditStore: totals = Counter() counts = Counter() facets = set() - seen_read_fingerprints = set() + seen_fingerprints = set() for event in events: - read_fingerprint = read_deduplication_fingerprint(event) + fingerprint = deduplication_fingerprint(event) if ( - read_fingerprint is not None - and read_fingerprint in seen_read_fingerprints + fingerprint is not None + and fingerprint in seen_fingerprints ): continue occurred_second = parse_event_second(event["timestamp"]) @@ -344,11 +350,11 @@ class AuditStore: success, str(event["path"]), str(event["source"]), - read_fingerprint, + fingerprint, ) ) - if read_fingerprint is not None: - seen_read_fingerprints.add(read_fingerprint) + if fingerprint is not None: + seen_fingerprints.add(fingerprint) self.conn.execute("BEGIN IMMEDIATE") try: @@ -364,7 +370,7 @@ class AuditStore: for row in self.conn.execute( f""" SELECT fingerprint - FROM audit_read_dedup + FROM audit_event_dedup WHERE fingerprint IN ({placeholders}) """, chunk, @@ -453,7 +459,7 @@ class AuditStore: first_inserted_id = last_inserted_id - len(event_rows) + 1 self.conn.executemany( """ - INSERT INTO audit_read_dedup ( + INSERT INTO audit_event_dedup ( fingerprint, occurred_second, event_id ) VALUES (?, ?, ?) """, @@ -474,8 +480,8 @@ class AuditStore: self.conn.executemany(ROLLUP_FACET_SQL, facets) self.conn.execute( - "DELETE FROM audit_read_dedup WHERE occurred_second < ?", - (read_deduplication_cutoff(),), + "DELETE FROM audit_event_dedup WHERE occurred_second < ?", + (deduplication_cutoff(),), ) if source_updates: @@ -544,53 +550,53 @@ class AuditStore: end_id = int(chunk[1]) self.conn.execute("BEGIN IMMEDIATE") try: - dedup_cutoff = read_deduplication_cutoff() + dedup_cutoff = deduplication_cutoff() self.conn.execute( - "DELETE FROM audit_read_dedup WHERE occurred_second < ?", + "DELETE FROM audit_event_dedup WHERE occurred_second < ?", (dedup_cutoff,), ) self.conn.execute( """ - INSERT INTO audit_read_dedup ( + INSERT INTO audit_event_dedup ( fingerprint, occurred_second, event_id ) - SELECT audit_read_fingerprint( - e.occurred_at, e.user, e.client_ip, e.share, e.path, + SELECT audit_event_fingerprint( + e.occurred_at, e.action, e.user, e.client_ip, e.share, e.path, e.success, e.result ), e.occurred_second, e.id FROM audit_events AS e WHERE e.id BETWEEN ? AND ? - AND e.action = 'read' + AND (e.action = 'read' OR e.share = 'FSLogix' COLLATE NOCASE) AND e.occurred_second >= ? ORDER BY e.id ON CONFLICT (fingerprint) DO UPDATE SET event_id = coalesce( - audit_read_dedup.event_id, excluded.event_id + audit_event_dedup.event_id, excluded.event_id ) """, (start_id, end_id, dedup_cutoff), ) - duplicate_read_sql = """ + duplicate_event_sql = """ SELECT e.id FROM audit_events AS e - JOIN audit_read_dedup AS d - ON d.fingerprint = audit_read_fingerprint( - e.occurred_at, e.user, e.client_ip, e.share, e.path, + JOIN audit_event_dedup AS d + ON d.fingerprint = audit_event_fingerprint( + e.occurred_at, e.action, e.user, e.client_ip, e.share, e.path, e.success, e.result ) WHERE e.id BETWEEN ? AND ? - AND e.action = 'read' + AND (e.action = 'read' OR e.share = 'FSLogix' COLLATE NOCASE) AND e.occurred_second >= ? AND (d.event_id IS NULL OR d.event_id <> e.id) """ self.conn.execute( f"DELETE FROM audit_path_events " - f"WHERE event_id IN ({duplicate_read_sql})", + f"WHERE event_id IN ({duplicate_event_sql})", (start_id, end_id, dedup_cutoff), ) self.conn.execute( f"DELETE FROM audit_events " - f"WHERE id IN ({duplicate_read_sql})", + f"WHERE id IN ({duplicate_event_sql})", (start_id, end_id, dedup_cutoff), ) self.conn.execute( diff --git a/app/init.sh b/app/init.sh index 206570a..85ab0f9 100755 --- a/app/init.sh +++ b/app/init.sh @@ -32,7 +32,7 @@ require_vfs_modules() { local missing=0 local module - for module in acl_xattr full_audit; do + for module in acl_xattr recycle full_audit; do if [[ ! -f "$modules_dir/vfs/${module}.so" ]]; then printf '[init] ERROR: missing VFS module %s at %s/vfs/%s.so\n' "$module" "$modules_dir" "$module" >&2 missing=1 @@ -251,6 +251,10 @@ write_runtime_env_file() { if [[ -n "${LDAP_BASE_DN:-}" ]]; then printf 'export LDAP_BASE_DN=%q\n' "$LDAP_BASE_DN" fi + printf 'export TRASH_RETENTION_DAYS=%q\n' "${TRASH_RETENTION_DAYS:-7}" + if [[ -n "${GROUP_ROOT:-}" ]]; then printf 'export GROUP_ROOT=%q\n' "$GROUP_ROOT"; fi + if [[ -n "${PRIVATE_ROOT:-}" ]]; then printf 'export PRIVATE_ROOT=%q\n' "$PRIVATE_ROOT"; fi + if [[ -n "${FSLOGIX_ROOT:-}" ]]; then printf 'export FSLOGIX_ROOT=%q\n' "$FSLOGIX_ROOT"; fi } > /app/runtime.env chmod 600 /app/runtime.env } @@ -567,6 +571,7 @@ install_cron_job() { SHELL=/bin/bash PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin */5 * * * * root source /app/runtime.env && RECONCILE_TRIGGER=automatic /usr/bin/python3 /app/reconcile_shares.py +17 * * * * root source /app/runtime.env && /usr/bin/python3 /app/trash.py --cleanup EOF if [[ -n "${BACKUP_DESTINATION:-}" ]] && env_is_true "${BACKUP_AUTO_ENABLED:-true}"; then @@ -622,6 +627,9 @@ write_runtime_env_file log 'Running startup reconciliation' RECONCILE_TRIGGER=startup python3 /app/reconcile_shares.py +log 'Preparing seven-day trash repositories' +python3 /app/trash.py --cleanup + start_observability_services if [[ -n "${BACKUP_DESTINATION:-}" ]] && env_is_true "${BACKUP_AUTO_ENABLED:-true}"; then diff --git a/app/trash.py b/app/trash.py new file mode 100644 index 0000000..7866ed9 --- /dev/null +++ b/app/trash.py @@ -0,0 +1,416 @@ +#!/usr/bin/env python3 +"""Safe listing, expiry, download, and restoration for Samba recycle bins.""" + +import argparse +import base64 +import datetime as dt +import os +import re +import stat +from typing import BinaryIO, Dict, List, Optional, Tuple + + +TRASH_DIRECTORY = ".trash" +DEFAULT_RETENTION_DAYS = 7 +VERSION_PREFIX_RE = re.compile(r"^Copy #\d+ of ", re.IGNORECASE) + + +def retention_days() -> int: + try: + value = int(os.getenv("TRASH_RETENTION_DAYS", str(DEFAULT_RETENTION_DAYS))) + except ValueError: + value = DEFAULT_RETENTION_DAYS + return max(1, min(365, value)) + + +def share_roots() -> Dict[str, str]: + return { + "Data": os.path.abspath(os.getenv("GROUP_ROOT", "/data/groups/data")), + "Private": os.path.abspath(os.getenv("PRIVATE_ROOT", "/data/private")), + "FSLogix": os.path.abspath(os.getenv("FSLOGIX_ROOT", "/data/fslogix")), + } + + +def trash_root(share_root: str) -> str: + return os.path.join(share_root, TRASH_DIRECTORY) + + +def ensure_trash_roots() -> None: + """Create non-listable sticky repositories users can write through Samba.""" + for root in share_roots().values(): + os.makedirs(root, exist_ok=True) + repository = trash_root(root) + os.makedirs(repository, exist_ok=True) + try: + os.chown(repository, 0, 0) + except PermissionError: + if os.geteuid() == 0: + raise + os.chmod(repository, 0o1733) + + +def _share_name(value: str) -> str: + for name in share_roots(): + if name.casefold() == value.casefold(): + return name + raise ValueError("Unbekannte Freigabe") + + +def _relative_parts(value: str) -> List[str]: + if not value or value.startswith(("/", "\\")) or "\0" in value: + raise ValueError("Ungültiger Papierkorbpfad") + parts = value.split("/") + if any(part in {"", ".", ".."} for part in parts): + raise ValueError("Ungültiger Papierkorbpfad") + return parts + + +def encode_item_id(share: str, relative_path: str) -> str: + canonical_share = _share_name(share) + _relative_parts(relative_path) + payload = f"{canonical_share}\0{relative_path}".encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def decode_item_id(item_id: str) -> Tuple[str, str]: + if not item_id or len(item_id) > 8192: + raise ValueError("Ungültige Papierkorb-ID") + try: + padding = "=" * (-len(item_id) % 4) + payload = base64.b64decode( + item_id + padding, + altchars=b"-_", + validate=True, + ).decode("utf-8") + share, relative_path = payload.split("\0", 1) + except (ValueError, UnicodeDecodeError) as exc: + raise ValueError("Ungültige Papierkorb-ID") from exc + canonical_share = _share_name(share) + _relative_parts(relative_path) + return canonical_share, relative_path + + +def _original_relative(relative_path: str) -> str: + parts = _relative_parts(relative_path) + if len(parts) < 2: + raise ValueError("Papierkorbeintrag enthält keinen Originalpfad") + original = parts[1:] + original[-1] = VERSION_PREFIX_RE.sub("", original[-1], count=1) + if not original[-1]: + raise ValueError("Papierkorbeintrag enthält keinen Dateinamen") + return "/".join(original) + + +def _open_directory_chain( + root: str, + parts: List[str], + *, + create: bool = False, + uid: int = 0, + gid: int = 0, + mode: int = 0o700, +) -> int: + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + current_fd = os.open(root, flags) + try: + for part in parts: + try: + next_fd = os.open(part, flags, dir_fd=current_fd) + except FileNotFoundError: + if not create: + raise + os.mkdir(part, mode=mode, dir_fd=current_fd) + os.chown( + part, + uid, + gid, + dir_fd=current_fd, + follow_symlinks=False, + ) + os.chmod( + part, + mode, + dir_fd=current_fd, + follow_symlinks=False, + ) + next_fd = os.open(part, flags, dir_fd=current_fd) + os.close(current_fd) + current_fd = next_fd + return current_fd + except Exception: + os.close(current_fd) + raise + + +def _resolved_item(item_id: str) -> Tuple[str, str, str, List[str], os.stat_result]: + share, relative_path = decode_item_id(item_id) + share_root = share_roots()[share] + repository = trash_root(share_root) + parts = _relative_parts(relative_path) + parent_fd = _open_directory_chain(repository, parts[:-1]) + try: + info = os.stat(parts[-1], dir_fd=parent_fd, follow_symlinks=False) + finally: + os.close(parent_fd) + if not stat.S_ISREG(info.st_mode): + raise ValueError("Nur reguläre Dateien können verarbeitet werden") + cutoff = dt.datetime.now(dt.timezone.utc).timestamp() - retention_days() * 86400 + if info.st_mtime < cutoff: + raise FileNotFoundError("Papierkorbeintrag ist abgelaufen") + return share, share_root, relative_path, parts, info + + +def _iso_timestamp(seconds: float) -> str: + return dt.datetime.fromtimestamp(seconds, dt.timezone.utc).isoformat( + timespec="seconds" + ) + + +def _item_payload( + share: str, + relative_path: str, + info: os.stat_result, + days: int, +) -> Dict[str, object]: + original = _original_relative(relative_path) + deleted_at = float(info.st_mtime) + return { + "id": encode_item_id(share, relative_path), + "share": share, + "path": original, + "name": original.rsplit("/", 1)[-1], + "deletedBy": relative_path.split("/", 1)[0], + "deletedAt": _iso_timestamp(deleted_at), + "expiresAt": _iso_timestamp(deleted_at + days * 86400), + "size": int(info.st_size), + } + + +def list_items( + *, + share: str = "", + path: str = "", + limit: int = 200, + now: Optional[dt.datetime] = None, +) -> Dict[str, object]: + days = retention_days() + current = now or dt.datetime.now(dt.timezone.utc) + cutoff = current.timestamp() - days * 86400 + selected_share = _share_name(share) if share else "" + path_filter = path.strip().casefold() + items: List[Dict[str, object]] = [] + + for share_name, root in share_roots().items(): + if selected_share and share_name != selected_share: + continue + repository = trash_root(root) + try: + walker = os.walk(repository, topdown=True, followlinks=False) + for directory, subdirectories, filenames in walker: + safe_subdirectories = [] + for name in subdirectories: + candidate = os.path.join(directory, name) + try: + if not stat.S_ISLNK(os.lstat(candidate).st_mode): + safe_subdirectories.append(name) + except OSError: + continue + subdirectories[:] = safe_subdirectories + for filename in filenames: + candidate = os.path.join(directory, filename) + try: + info = os.lstat(candidate) + except OSError: + continue + if not stat.S_ISREG(info.st_mode) or info.st_mtime < cutoff: + continue + relative_path = os.path.relpath(candidate, repository).replace( + os.sep, "/" + ) + try: + payload = _item_payload( + share_name, relative_path, info, days + ) + except ValueError: + continue + if path_filter and path_filter not in str(payload["path"]).casefold(): + continue + items.append(payload) + except OSError: + continue + + items.sort( + key=lambda item: (str(item["deletedAt"]), str(item["id"])), + reverse=True, + ) + maximum = max(1, min(500, int(limit))) + return { + "items": items[:maximum], + "matched": len(items), + "truncated": len(items) > maximum, + "retentionDays": days, + "scannedAt": current.isoformat(timespec="seconds"), + } + + +def open_download(item_id: str) -> Tuple[BinaryIO, Dict[str, object]]: + share, _share_root, relative_path, parts, _info = _resolved_item(item_id) + repository = trash_root(share_roots()[share]) + parent_fd = _open_directory_chain(repository, parts[:-1]) + try: + file_fd = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW, dir_fd=parent_fd) + finally: + os.close(parent_fd) + try: + info = os.fstat(file_fd) + if not stat.S_ISREG(info.st_mode): + raise ValueError("Nur reguläre Dateien können heruntergeladen werden") + payload = _item_payload(share, relative_path, info, retention_days()) + return os.fdopen(file_fd, "rb"), payload + except Exception: + os.close(file_fd) + raise + + +def _restore_directory_mode(share: str) -> int: + if share == "Data": + return 0o2770 + return 0o700 + + +def _remove_empty_trash_parents(repository: str, relative_path: str) -> None: + current = os.path.dirname(os.path.join(repository, relative_path)) + repository = os.path.abspath(repository) + while ( + os.path.commonpath((repository, current)) == repository + and current != repository + ): + try: + os.rmdir(current) + except OSError: + break + current = os.path.dirname(current) + + +def restore_item(item_id: str) -> Dict[str, object]: + share, share_root, relative_path, parts, info = _resolved_item(item_id) + original = _original_relative(relative_path) + original_parts = _relative_parts(original) + if original_parts[0] == TRASH_DIRECTORY: + raise ValueError("Ungültiger Wiederherstellungspfad") + + source_parent_fd = _open_directory_chain( + trash_root(share_root), parts[:-1] + ) + destination_parent_fd = _open_directory_chain( + share_root, + original_parts[:-1], + create=True, + uid=info.st_uid, + gid=info.st_gid, + mode=_restore_directory_mode(share), + ) + linked = False + try: + os.link( + parts[-1], + original_parts[-1], + src_dir_fd=source_parent_fd, + dst_dir_fd=destination_parent_fd, + follow_symlinks=False, + ) + linked = True + linked_info = os.stat( + original_parts[-1], + dir_fd=destination_parent_fd, + follow_symlinks=False, + ) + if ( + not stat.S_ISREG(linked_info.st_mode) + or linked_info.st_dev != info.st_dev + or linked_info.st_ino != info.st_ino + ): + os.unlink(original_parts[-1], dir_fd=destination_parent_fd) + linked = False + raise RuntimeError("Papierkorbeintrag wurde währenddessen verändert") + try: + os.unlink(parts[-1], dir_fd=source_parent_fd) + except Exception: + os.unlink(original_parts[-1], dir_fd=destination_parent_fd) + linked = False + raise + finally: + os.close(source_parent_fd) + os.close(destination_parent_fd) + + if not linked: + raise RuntimeError("Wiederherstellung konnte nicht abgeschlossen werden") + _remove_empty_trash_parents(trash_root(share_root), relative_path) + return { + "restored": True, + "share": share, + "path": original, + } + + +def cleanup_expired(now: Optional[dt.datetime] = None) -> Dict[str, int]: + days = retention_days() + current = now or dt.datetime.now(dt.timezone.utc) + cutoff = current.timestamp() - days * 86400 + removed = 0 + removed_bytes = 0 + + ensure_trash_roots() + for root in share_roots().values(): + repository = trash_root(root) + for directory, subdirectories, filenames in os.walk( + repository, topdown=False, followlinks=False + ): + for filename in filenames: + candidate = os.path.join(directory, filename) + try: + info = os.lstat(candidate) + if info.st_mtime >= cutoff: + continue + if not ( + stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) + ): + continue + os.unlink(candidate) + removed += 1 + if stat.S_ISREG(info.st_mode): + removed_bytes += int(info.st_size) + except OSError: + continue + for name in subdirectories: + candidate = os.path.join(directory, name) + try: + info = os.lstat(candidate) + if stat.S_ISLNK(info.st_mode): + if info.st_mtime < cutoff: + os.unlink(candidate) + removed += 1 + continue + os.rmdir(candidate) + except OSError: + continue + return {"removed": removed, "removedBytes": removed_bytes} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--cleanup", action="store_true") + args = parser.parse_args() + if not args.cleanup: + parser.error("--cleanup is required") + result = cleanup_expired() + print( + f"[trash] Removed {result['removed']} expired item(s) " + f"({result['removedBytes']} bytes)", + flush=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/web/app.js b/app/web/app.js index c0595cb..b97a53b 100644 --- a/app/web/app.js +++ b/app/web/app.js @@ -105,6 +105,7 @@ function routeFor(path) { if (path.startsWith("/reconciliation")) return "reconciliation"; if (path.startsWith("/activity/fslogix")) return "activity-fslogix"; if (path.startsWith("/activity")) return "activity"; + if (path.startsWith("/trash")) return "trash"; if (path.startsWith("/backup")) return "backup"; if (path.startsWith("/report")) return "report"; if (path.startsWith("/system")) return "system"; @@ -127,6 +128,7 @@ async function navigate(path, replace = false) { if (route === "storage-users") await renderStorage("users"); if (route === "activity") await renderActivity(); if (route === "activity-fslogix") await renderActivity("fslogix"); + if (route === "trash") await renderTrash(); if (route === "backup") await renderBackup(); if (route === "report") await renderReport(); if (route === "system") await renderSystem(); @@ -303,6 +305,68 @@ async function renderActivity(stream = "main") { await load(false); } +async function renderTrash() { + content.innerHTML = pageHead( + "Papierkorb", + "Gelöschte Dateien sieben Tage lang herunterladen oder am Originalpfad wiederherstellen.", + ) + ` +
+
+ + + +
+

+
Gelöscht (UTC)BenutzerFreigabeOriginalpfadGrößeVerfügbar bisAktionen
+
`; + const form = document.querySelector("#trash-filter"); + const load = async () => { + const params = new URLSearchParams(new FormData(form)); + params.set("limit", "500"); + const result = await api(`/api/trash?${params}`); + const body = document.querySelector("#trash-rows"); + if (!result.items.length) { + body.innerHTML = 'Keine gelöschten Dateien'; + } else { + body.innerHTML = result.items.map(item => ` + ${esc(utcTime(item.deletedAt))} + ${esc(item.deletedBy)} + ${esc(item.share)} + ${esc(item.path)} + ${bytes(item.size)} + ${esc(utcTime(item.expiresAt))} +
Herunterladen
+ `).join(""); + } + const shown = result.items.length.toLocaleString("de-DE"); + const total = Number(result.matched || 0).toLocaleString("de-DE"); + const suffix = result.truncated ? ` · ${shown} von ${total} angezeigt` : ""; + document.querySelector("#trash-summary").textContent = `${total} Dateien · Aufbewahrung ${result.retentionDays} Tage${suffix}`; + }; + form.addEventListener("submit", async event => { + event.preventDefault(); + try { await load(); } catch (error) { notice(error.message); } + }); + document.querySelector("#trash-rows").addEventListener("click", async event => { + const button = event.target.closest("button[data-restore]"); + if (!button) return; + if (!window.confirm("Datei am Originalpfad wiederherstellen?")) return; + button.disabled = true; + try { + const result = await api("/api/trash/restore", { + method: "POST", + body: JSON.stringify({id: button.dataset.restore}), + }); + notice(`${result.share}: ${result.path} wurde wiederhergestellt.`); + await load(); + } catch (error) { + notice(error.message); + button.disabled = false; + } + }); + await load(); +} + function triggerLabel(value) { return ({automatic: "Automatisch", web: "Weboberfläche", manual: "Befehlszeile", startup: "Systemstart"}[value] || value || "—"); } diff --git a/app/web/index.html b/app/web/index.html index aa244b6..dc6111d 100644 --- a/app/web/index.html +++ b/app/web/index.html @@ -33,6 +33,7 @@ Benutzerbelegung Aktivitätsprotokoll FSLogix-Protokoll + Papierkorb Sicherungen PDF-Bericht System diff --git a/app/web/styles.css b/app/web/styles.css index 5df86c8..873184b 100644 --- a/app/web/styles.css +++ b/app/web/styles.css @@ -8,7 +8,7 @@ body { margin: 0; min-height: 100vh; } a { color: #0645ad; } button, input, select { font: inherit; } -button { padding: .4rem .7rem; border: 1px solid #777; color: #111; background: #eee; cursor: pointer; } +button, .button { display: inline-block; padding: .4rem .7rem; border: 1px solid #777; color: #111; background: #eee; cursor: pointer; text-decoration: none; white-space: nowrap; } button:disabled { color: #777; cursor: wait; } input, select { width: 100%; padding: .4rem; border: 1px solid #999; background: #fff; } button:focus, input:focus, select:focus, a:focus { outline: 2px solid #0645ad; outline-offset: 1px; } @@ -64,6 +64,7 @@ th, td { padding: .5rem; border-bottom: 1px solid #ccc; vertical-align: top; } .badge.warn { color: #750; } .toolbar { display: flex; flex-wrap: wrap; gap: .5rem; } .toolbar input { max-width: 340px; } +.row-actions { display: flex; flex-wrap: wrap; gap: .4rem; } .list { margin: 0; padding: 0; list-style: none; } .select-row { width: 100%; display: grid; grid-template-columns: 1fr auto; gap: .5rem; border: 0; border-bottom: 1px solid #ccc; background: #fff; text-align: left; } .select-row.active { font-weight: bold; background: #eee; } diff --git a/app/web_ui.py b/app/web_ui.py index 053deeb..5d3095c 100644 --- a/app/web_ui.py +++ b/app/web_ui.py @@ -27,8 +27,10 @@ from typing import Dict, List, Optional, Tuple try: from app import reconcile_shares as directory + from app import trash except ImportError: # Container execution uses /app as the import root. import reconcile_shares as directory + import trash try: from app.audit_store import ( @@ -460,6 +462,8 @@ def scan_children(root: str) -> List[Dict[str, object]]: except OSError: return rows for entry in entries: + if entry.name == trash.TRASH_DIRECTORY: + continue try: if not entry.is_dir(follow_symlinks=False): continue @@ -1032,6 +1036,43 @@ class Handler(BaseHTTPRequestHandler): raise ValueError("Ein JSON-Objekt ist erforderlich") return value + def send_trash_download(self, params: Dict[str, List[str]]) -> None: + item_id = params.get("id", [""])[0] + try: + handle, item = trash.open_download(item_id) + except FileNotFoundError: + self.send_error_json(HTTPStatus.NOT_FOUND, "Datei nicht gefunden oder abgelaufen") + return + except ValueError as exc: + self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) + return + except OSError as exc: + log(f"Trash download failed: {exc}") + self.send_error_json(HTTPStatus.INTERNAL_SERVER_ERROR, "Download konnte nicht geöffnet werden") + return + + filename = str(item["name"]) + encoded_name = urllib.parse.quote(filename, safe="") + with handle: + self.send_response(HTTPStatus.OK) + self.security_headers() + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(item["size"])) + self.send_header( + "Content-Disposition", + f"attachment; filename*=UTF-8{chr(39) * 2}{encoded_name}", + ) + self.end_headers() + try: + while True: + chunk = handle.read(1024 * 1024) + if not chunk: + break + self.wfile.write(chunk) + except (BrokenPipeError, ConnectionResetError): + pass + + def do_POST(self) -> None: # pylint: disable=invalid-name parsed = urllib.parse.urlparse(self.path) if parsed.path == "/api/login": @@ -1060,6 +1101,42 @@ class Handler(BaseHTTPRequestHandler): cookie = f"{JWT_COOKIE}=; Path=/; Max-Age=0; HttpOnly; Secure; SameSite=Strict" self.send_json({"ok": True}, cookie=cookie) return + if parsed.path == "/api/trash/restore": + user = self.require_user() + if user is None: + return + try: + body = self.read_json_body() + result = trash.restore_item(str(body.get("id", ""))) + except FileExistsError: + self.send_error_json( + HTTPStatus.CONFLICT, + "Am Originalpfad existiert bereits eine Datei", + ) + return + except FileNotFoundError: + self.send_error_json( + HTTPStatus.NOT_FOUND, + "Datei nicht gefunden oder abgelaufen", + ) + return + except ValueError as exc: + self.send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) + return + except (OSError, RuntimeError) as exc: + log(f"Trash restore failed: {exc}") + self.send_error_json( + HTTPStatus.INTERNAL_SERVER_ERROR, + "Datei konnte nicht wiederhergestellt werden", + ) + return + log( + f"{user['sub']} restored " + f"{result['share']}:{result['path']}" + ) + self.send_json(result) + return + if parsed.path.startswith("/api/actions/"): user = self.require_user() if user is None: @@ -1105,6 +1182,16 @@ class Handler(BaseHTTPRequestHandler): self.send_json(query_audit(params)) elif path == "/api/fslogix-activity": self.send_json(query_audit(params, stream="fslogix")) + elif path == "/api/trash/download": + self.send_trash_download(params) + elif path == "/api/trash": + self.send_json( + trash.list_items( + share=params.get("share", [""])[0], + path=params.get("path", [""])[0], + limit=int(params.get("limit", ["200"])[0]), + ) + ) elif path == "/api/backup": self.send_json( backup_payload(include_log=query_includes_log(params)) diff --git a/dev/ad-dc.Dockerfile b/dev/ad-dc.Dockerfile index 7dbf440..27c8823 100644 --- a/dev/ad-dc.Dockerfile +++ b/dev/ad-dc.Dockerfile @@ -1,4 +1,4 @@ -FROM debian:12-slim +FROM docker.io/library/debian:12-slim ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/backup.Dockerfile b/dev/backup.Dockerfile index fedf982..3058b0a 100644 --- a/dev/backup.Dockerfile +++ b/dev/backup.Dockerfile @@ -1,4 +1,4 @@ -FROM debian:12-slim +FROM docker.io/library/debian:12-slim ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/e2e.py b/dev/e2e.py index 13cd783..6ef23de 100755 --- a/dev/e2e.py +++ b/dev/e2e.py @@ -222,6 +222,16 @@ def main() -> int: http("/api/actions/reconciliation", method="POST", value={}).status == 401, "anonymous reconciliation action was accepted", ) + check(http("/api/trash").status == 401, "anonymous trash listing was accepted") + check( + http( + "/api/trash/restore", + method="POST", + value={"id": "invalid"}, + ).status + == 401, + "anonymous trash restore was accepted", + ) non_admin = http( "/api/login", method="POST", @@ -304,6 +314,84 @@ def main() -> int: check_result=False, ) check(admin_access.returncode == 0, "Domain Admin cannot inspect Finance") + direct_trash_access = engine_run( + "exec", + CLIENT_CONTAINER, + "smbclient", + f"//files.{DNS_DOMAIN}/Data", + "-m", + "SMB3", + "-U", + f"{WORKGROUP}\\alice%{USER_PASSWORD}", + "-c", + "cd .trash; ls", + check_result=False, + ) + check( + direct_trash_access.returncode != 0, + "ordinary SMB user can browse the admin-managed trash repository", + ) + + announce("real Samba recycle, admin download, and conflict-safe restore") + trash_response = eventually( + "deleted SMB file in the seven-day trash", + lambda: http( + query_path( + "/api/trash", + {"share": "Data", "path": "audit-moved.txt", "limit": "10"}, + ), + token=token, + ), + lambda response: ( + response.status == 200 + and any( + item.get("path") == "Finance/Reports/audit-moved.txt" + for item in response.json().get("items", []) + ) + ), + timeout=30, + ) + trash_items = trash_response.json().get("items", []) + trash_item = next( + item + for item in trash_items + if item.get("path") == "Finance/Reports/audit-moved.txt" + ) + trash_download = http( + query_path("/api/trash/download", {"id": str(trash_item["id"])}), + token=token, + ) + check( + trash_download.status == 200 + and len(trash_download.body) == int(trash_item["size"]) + and "attachment" in trash_download.headers.get("Content-Disposition", ""), + "trash download is missing, truncated, or not an attachment", + ) + restored = http( + "/api/trash/restore", + method="POST", + value={"id": trash_item["id"]}, + token=token, + ) + check( + restored.status == 200 + and restored.json().get("path") == "Finance/Reports/audit-moved.txt", + f"trash restore failed: {restored.body!r}", + ) + restored_read = engine_run( + "exec", + CLIENT_CONTAINER, + "smbclient", + f"//files.{DNS_DOMAIN}/Data", + "-m", + "SMB3", + "-U", + f"{WORKGROUP}\\alice%{USER_PASSWORD}", + "-c", + "cd Finance; cd Reports; get audit-moved.txt /tmp/restored-trash-file.txt", + check_result=False, + ) + check(restored_read.returncode == 0, "restored file is not readable over SMB") announce("group, Private, and FSLogix size accounting") storage = http("/api/storage", token=token) diff --git a/etc/samba/smb.conf b/etc/samba/smb.conf index 20c20ec..e995143 100644 --- a/etc/samba/smb.conf +++ b/etc/samba/smb.conf @@ -42,7 +42,15 @@ read only = no browseable = yes guest ok = no - vfs objects = acl_xattr full_audit + vfs objects = acl_xattr recycle full_audit + recycle:repository = .trash/%U + recycle:keeptree = yes + recycle:versions = yes + recycle:touch_mtime = yes + recycle:directory_mode = 0700 + recycle:subdir_mode = 0700 + recycle:exclude_dir = .trash + veto files = /.trash/ full_audit:prefix = %T|%u|%I|%m|%S full_audit:success = pread pread_recv read sendfile offload_read_recv pwrite pwrite_recv write recvfile offload_write_recv renameat unlinkat full_audit:failure = pread pread_recv read sendfile offload_read_recv pwrite pwrite_recv write recvfile offload_write_recv renameat unlinkat @@ -58,7 +66,15 @@ read only = no browseable = yes guest ok = no - vfs objects = acl_xattr full_audit + vfs objects = acl_xattr recycle full_audit + recycle:repository = .trash/%U + recycle:keeptree = yes + recycle:versions = yes + recycle:touch_mtime = yes + recycle:directory_mode = 0700 + recycle:subdir_mode = 0700 + recycle:exclude_dir = .trash + veto files = /.trash/ full_audit:prefix = %T|%u|%I|%m|%S full_audit:success = pread pread_recv read sendfile offload_read_recv pwrite pwrite_recv write recvfile offload_write_recv renameat unlinkat full_audit:failure = pread pread_recv read sendfile offload_read_recv pwrite pwrite_recv write recvfile offload_write_recv renameat unlinkat @@ -85,7 +101,15 @@ read only = no browseable = yes guest ok = no - vfs objects = acl_xattr full_audit + vfs objects = acl_xattr recycle full_audit + recycle:repository = .trash/%U + recycle:keeptree = yes + recycle:versions = yes + recycle:touch_mtime = yes + recycle:directory_mode = 0700 + recycle:subdir_mode = 0700 + recycle:exclude_dir = .trash + veto files = /.trash/ full_audit:prefix = %T|%u|%I|%m|%S full_audit:success = pread pread_recv read sendfile offload_read_recv pwrite pwrite_recv write recvfile offload_write_recv renameat unlinkat full_audit:failure = pread pread_recv read sendfile offload_read_recv pwrite pwrite_recv write recvfile offload_write_recv renameat unlinkat diff --git a/tests/test_trash.py b/tests/test_trash.py new file mode 100644 index 0000000..34d5d91 --- /dev/null +++ b/tests/test_trash.py @@ -0,0 +1,193 @@ +import base64 +import datetime as dt +import os +import tempfile +import unittest +from unittest import mock + +from app import trash + + +class TrashTests(unittest.TestCase): + def roots(self, tmpdir): + return { + "GROUP_ROOT": os.path.join(tmpdir, "data"), + "PRIVATE_ROOT": os.path.join(tmpdir, "private"), + "FSLOGIX_ROOT": os.path.join(tmpdir, "fslogix"), + "TRASH_RETENTION_DAYS": "7", + } + + def recycled_file(self, root, relative, content=b"content"): + path = os.path.join(root, trash.TRASH_DIRECTORY, *relative.split("/")) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as handle: + handle.write(content) + return path + + def test_lists_downloads_and_restores_versioned_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + env = self.roots(tmpdir) + with mock.patch.dict(os.environ, env): + trash.ensure_trash_roots() + self.assertEqual( + os.stat( + os.path.join(env["GROUP_ROOT"], trash.TRASH_DIRECTORY) + ).st_mode + & 0o7777, + 0o1733, + ) + source = self.recycled_file( + env["GROUP_ROOT"], + "alice/Finance/Copy #2 of report.xlsx", + b"spreadsheet", + ) + before = os.stat(source) + + result = trash.list_items(share="data", path="REPORT") + self.assertEqual(result["retentionDays"], 7) + self.assertEqual(result["matched"], 1) + item = result["items"][0] + self.assertEqual(item["share"], "Data") + self.assertEqual(item["path"], "Finance/report.xlsx") + self.assertEqual(item["deletedBy"], "alice") + self.assertEqual(item["size"], 11) + + handle, download = trash.open_download(str(item["id"])) + with handle: + self.assertEqual(handle.read(), b"spreadsheet") + self.assertEqual(download["name"], "report.xlsx") + + restored = trash.restore_item(str(item["id"])) + destination = os.path.join( + env["GROUP_ROOT"], "Finance", "report.xlsx" + ) + self.assertEqual( + restored, + { + "restored": True, + "share": "Data", + "path": "Finance/report.xlsx", + }, + ) + self.assertFalse(os.path.exists(source)) + with open(destination, "rb") as handle: + self.assertEqual(handle.read(), b"spreadsheet") + self.assertEqual(os.stat(destination).st_ino, before.st_ino) + + def test_restore_never_overwrites_existing_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + env = self.roots(tmpdir) + with mock.patch.dict(os.environ, env): + trash.ensure_trash_roots() + source = self.recycled_file( + env["PRIVATE_ROOT"], "alice/alice/document.txt", b"deleted" + ) + destination = os.path.join( + env["PRIVATE_ROOT"], "alice", "document.txt" + ) + os.makedirs(os.path.dirname(destination), exist_ok=True) + with open(destination, "wb") as handle: + handle.write(b"current") + item_id = trash.encode_item_id( + "Private", "alice/alice/document.txt" + ) + + with self.assertRaises(FileExistsError): + trash.restore_item(item_id) + + with open(destination, "rb") as handle: + self.assertEqual(handle.read(), b"current") + self.assertTrue(os.path.isfile(source)) + + def test_restore_does_not_follow_destination_directory_symlinks(self): + with tempfile.TemporaryDirectory() as tmpdir: + env = self.roots(tmpdir) + with mock.patch.dict(os.environ, env): + trash.ensure_trash_roots() + source = self.recycled_file( + env["GROUP_ROOT"], "alice/Finance/report.txt", b"deleted" + ) + outside = os.path.join(tmpdir, "outside") + os.makedirs(outside) + os.symlink(outside, os.path.join(env["GROUP_ROOT"], "Finance")) + item_id = trash.encode_item_id( + "Data", "alice/Finance/report.txt" + ) + + with self.assertRaises(OSError): + trash.restore_item(item_id) + + self.assertTrue(os.path.isfile(source)) + self.assertFalse(os.path.exists(os.path.join(outside, "report.txt"))) + + def test_cleanup_removes_only_files_older_than_seven_days(self): + with tempfile.TemporaryDirectory() as tmpdir: + env = self.roots(tmpdir) + with mock.patch.dict(os.environ, env): + trash.ensure_trash_roots() + old = self.recycled_file( + env["FSLOGIX_ROOT"], "alice/old.vhd", b"old" + ) + recent = self.recycled_file( + env["FSLOGIX_ROOT"], "alice/recent.vhd", b"recent" + ) + now = dt.datetime.now(dt.timezone.utc) + old_time = (now - dt.timedelta(days=8)).timestamp() + os.utime(old, (old_time, old_time)) + + result = trash.cleanup_expired(now) + + self.assertEqual(result, {"removed": 1, "removedBytes": 3}) + self.assertFalse(os.path.exists(old)) + self.assertTrue(os.path.isfile(recent)) + self.assertEqual(trash.list_items(now=now)["matched"], 1) + + def test_cleanup_removes_expired_symlinks_without_following_them(self): + with tempfile.TemporaryDirectory() as tmpdir: + env = self.roots(tmpdir) + with mock.patch.dict(os.environ, env): + trash.ensure_trash_roots() + outside = os.path.join(tmpdir, "outside") + os.makedirs(outside) + repository = os.path.join( + env["GROUP_ROOT"], trash.TRASH_DIRECTORY, "alice" + ) + os.makedirs(repository, exist_ok=True) + link = os.path.join(repository, "linked-directory") + os.symlink(outside, link) + now = dt.datetime.now(dt.timezone.utc) + old_time = (now - dt.timedelta(days=8)).timestamp() + os.utime(link, (old_time, old_time), follow_symlinks=False) + + result = trash.cleanup_expired(now) + + self.assertEqual(result, {"removed": 1, "removedBytes": 0}) + self.assertFalse(os.path.lexists(link)) + self.assertTrue(os.path.isdir(outside)) + + def test_ids_cannot_escape_repository_and_symlinks_are_not_exposed(self): + with tempfile.TemporaryDirectory() as tmpdir: + env = self.roots(tmpdir) + with mock.patch.dict(os.environ, env): + trash.ensure_trash_roots() + encoded = base64.urlsafe_b64encode( + b"Data\0../outside" + ).decode().rstrip("=") + with self.assertRaises(ValueError): + trash.decode_item_id(encoded) + + repository = os.path.join( + env["GROUP_ROOT"], trash.TRASH_DIRECTORY, "alice" + ) + os.makedirs(repository, exist_ok=True) + link = os.path.join(repository, "link.txt") + os.symlink("/etc/passwd", link) + self.assertEqual(trash.list_items()["matched"], 0) + with self.assertRaises(ValueError): + trash.open_download( + trash.encode_item_id("Data", "alice/link.txt") + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_web_ui.py b/tests/test_web_ui.py index fb9d696..5ff84af 100644 --- a/tests/test_web_ui.py +++ b/tests/test_web_ui.py @@ -558,7 +558,7 @@ class AuditParsingTests(unittest.TestCase): ).fetchall() fingerprints = store.conn.execute( """ - SELECT count(*) FROM audit_read_dedup + SELECT count(*) FROM audit_event_dedup """ ).fetchone()[0] finally: @@ -582,6 +582,85 @@ class AuditParsingTests(unittest.TestCase): ) + def test_deduplicates_fslogix_writes_but_preserves_private_writes(self): + with tempfile.TemporaryDirectory() as tmpdir: + database = os.path.join(tmpdir, "state.db") + store = audit_store.AuditStore(database) + day = dt.datetime.now(dt.timezone.utc).date() + timestamp = f"{day}T13:39:23+00:00" + fslogix_write = { + "timestamp": timestamp, + "ingestedAt": timestamp, + "user": "LOCAL\\ralf.schwientek", + "clientIp": "10.100.0.21", + "client": "PC01", + "share": "FSLogix", + "operation": "pwrite", + "action": "write", + "path": "/data/fslogix/profile.vhd", + "result": "OK", + "success": True, + "source": "log.pc01", + } + private_write = dict( + fslogix_write, + share="Private", + path="/data/private/file.txt", + ) + fslogix_read = dict( + fslogix_write, operation="pread", action="read" + ) + try: + inserted = store.append_batch( + [ + fslogix_write, + dict(fslogix_write, source="log.pc02"), + private_write, + dict(private_write, source="log.pc02"), + fslogix_read, + ], + {}, + set(), + ) + repeated_poll = store.append_batch( + [dict(fslogix_write, source="log.pc03")], {}, set() + ) + next_timestamp = f"{day}T13:39:24+00:00" + next_second = store.append_batch( + [dict( + fslogix_write, + timestamp=next_timestamp, + ingestedAt=next_timestamp, + source="log.pc04", + )], + {}, + set(), + ) + rows = store.conn.execute( + "SELECT action, share FROM audit_events ORDER BY id" + ).fetchall() + fingerprints = store.conn.execute( + "SELECT count(*) FROM audit_event_dedup" + ).fetchone()[0] + finally: + store.close() + + self.assertEqual(inserted, 4) + self.assertEqual(repeated_poll, 0) + self.assertEqual(next_second, 1) + self.assertEqual(fingerprints, 3) + self.assertEqual( + [(row["action"], row["share"]) for row in rows], + [ + ("write", "FSLogix"), + ("write", "Private"), + ("write", "Private"), + ("read", "FSLogix"), + ("write", "FSLogix"), + ], + ) + + class AuditQueryTests(unittest.TestCase): def make_event(self, timestamp, user, success=True): return { @@ -969,14 +1048,21 @@ class AuditQueryTests(unittest.TestCase): self.assertEqual(main["matched"], 1) self.assertEqual(fslogix["matched"], 1) self.assertEqual(total, 2) - self.assertEqual(policy_version, 4) + self.assertEqual(policy_version, 5) - def test_upgrade_collapses_recent_legacy_read_duplicates(self): + def test_upgrade_collapses_recent_fslogix_write_duplicates(self): with tempfile.TemporaryDirectory() as tmpdir: today = dt.datetime.now(dt.timezone.utc).date() database = os.path.join(tmpdir, "state.db") store = audit_store.AuditStore(database) event = self.make_event(f"{today}T12:00:00+00:00", "alice") + event = dict( + event, + share="FSLogix", + operation="pwrite", + action="write", + path="/data/fslogix/profile.vhd", + ) store.append_batch([event], {}, set()) for source in ("log.pc02", "log.pc03"): store.conn.execute( @@ -994,9 +1080,11 @@ class AuditQueryTests(unittest.TestCase): """, (source,), ) - store.conn.execute("DELETE FROM audit_read_dedup") store.conn.execute( - "UPDATE audit_rollup_state SET policy_version = 3" + "ALTER TABLE audit_event_dedup RENAME TO audit_read_dedup" + ) + store.conn.execute( + "UPDATE audit_rollup_state SET policy_version = 4" ) store.conn.commit() store.close() @@ -1014,7 +1102,7 @@ class AuditQueryTests(unittest.TestCase): "SELECT sum(event_count) FROM audit_daily_totals" ).fetchone()[0] dedup_keys = store.conn.execute( - "SELECT count(*) FROM audit_read_dedup" + "SELECT count(*) FROM audit_event_dedup" ).fetchone()[0] policy_version = store.conn.execute( "SELECT policy_version FROM audit_rollup_state" @@ -1030,7 +1118,7 @@ class AuditQueryTests(unittest.TestCase): self.assertEqual(live_inserted, 1) self.assertEqual(retained_source, "log.live") self.assertEqual(dedup_keys, 1) - self.assertEqual(policy_version, 4) + self.assertEqual(policy_version, 5) def test_schema_has_filter_indexes_and_rollup_tables(self): @@ -1066,7 +1154,7 @@ class AuditQueryTests(unittest.TestCase): "audit_events_fslogix_user_time", "audit_events_fslogix_account_time", "audit_events_fslogix_result_time", - "audit_read_dedup_time", + "audit_event_dedup_time", }.issubset(indexes) ) self.assertTrue( @@ -1078,7 +1166,7 @@ class AuditQueryTests(unittest.TestCase): "audit_paths", "audit_paths_fts", "audit_path_events", - "audit_read_dedup", + "audit_event_dedup", }.issubset(tables) ) @@ -1148,6 +1236,12 @@ class WebPresentationTests(unittest.TestCase): self.assertIn("/activity/fslogix", html) self.assertIn("/api/fslogix-activity", script) self.assertIn('renderActivity("fslogix")', script) + self.assertIn('href="/trash" data-route="trash">Papierkorb', html) + self.assertIn("/api/trash?", script) + self.assertIn("/api/trash/download?id=", script) + self.assertIn('api("/api/trash/restore"', script) + self.assertIn("Herunterladen", script) + self.assertIn("Wiederherstellen", script) self.assertNotIn(">Auflisten<", script) self.assertNotIn(">Metadaten<", script) self.assertNotIn(">Sitzung<", script) @@ -1218,6 +1312,39 @@ class WebPresentationTests(unittest.TestCase): for line in success_lines + failure_lines: self.assertEqual(set(line.split("=", 1)[1].split()), expected) + def test_samba_recycles_all_three_shares_for_seven_day_cleanup(self): + root = os.path.join(os.path.dirname(__file__), "..") + with open( + os.path.join(root, "etc", "samba", "smb.conf"), + encoding="utf-8", + ) as handle: + config = handle.read() + with open(os.path.join(root, "app", "init.sh"), encoding="utf-8") as handle: + init_script = handle.read() + with open(os.path.join(root, "Dockerfile"), encoding="utf-8") as handle: + dockerfile = handle.read() + + for share, next_share in ( + ("Private", "Data"), + ("Data", "FSLogix"), + ("FSLogix", None), + ): + share_config = config.split(f"[{share}]", 1)[1] + if next_share: + share_config = share_config.split(f"[{next_share}]", 1)[0] + self.assertIn("vfs objects = acl_xattr recycle full_audit", share_config) + self.assertIn("recycle:repository = .trash/%U", share_config) + self.assertIn("recycle:keeptree = yes", share_config) + self.assertIn("recycle:versions = yes", share_config) + self.assertIn("recycle:touch_mtime = yes", share_config) + self.assertIn("recycle:exclude_dir = .trash", share_config) + self.assertIn("veto files = /.trash/", share_config) + + self.assertIn("acl_xattr recycle full_audit", init_script) + self.assertIn("/app/trash.py --cleanup", init_script) + self.assertIn("TRASH_RETENTION_DAYS", init_script) + self.assertIn("COPY app/trash.py /app/trash.py", dockerfile) + class TlsSummaryTests(unittest.TestCase): @mock.patch("app.web_ui.ssl._ssl._test_decode_cert") @@ -1236,12 +1363,21 @@ class UsageScannerTests(unittest.TestCase): os.makedirs(os.path.join(group_root, "Finance")) os.makedirs(os.path.join(private_root, "alice")) os.makedirs(os.path.join(fslogix_root, "alice_S-1-5-21-1-2-3-1001")) + os.makedirs(os.path.join(group_root, ".trash", "alice")) + os.makedirs(os.path.join(private_root, ".trash", "alice")) + os.makedirs(os.path.join(fslogix_root, ".trash", "alice")) with open(os.path.join(group_root, "Finance", "a"), "wb") as handle: handle.write(b"a" * 7) with open(os.path.join(private_root, "alice", "b"), "wb") as handle: handle.write(b"b" * 3) with open(os.path.join(fslogix_root, "alice_S-1-5-21-1-2-3-1001", "c"), "wb") as handle: handle.write(b"c" * 5) + for root in (group_root, private_root, fslogix_root): + with open( + os.path.join(root, ".trash", "alice", "deleted"), + "wb", + ) as handle: + handle.write(b"x" * 100) database = os.path.join(tmpdir, "state.db") env = {"GROUP_ROOT": group_root, "PRIVATE_ROOT": private_root, "FSLOGIX_ROOT": fslogix_root}