higher level logging; sqlite db for logs

2026-08-01 04:36:25 +00:00
parent 17f5ef8560
commit fdd5649198
19 changed files with 1092 additions and 377 deletions
@@ -1,30 +1,26 @@
 #!/usr/bin/env python3
-"""Persist Samba full_audit records as immutable daily NDJSON archives."""
+"""Persist selected Samba full_audit records in indexed SQLite tables."""

 import datetime as dt
 import glob
-import gzip
-import json
 import os
 import re
 import signal
 import sys
 import time
-from typing import Dict, Iterable, Optional
+from typing import Dict, Optional

 try:
    from .audit_policy import action_for, skip_user
+    from .audit_store import AuditStore, drop_legacy_audit_files
 except ImportError:
    from audit_policy import action_for, skip_user
+    from audit_store import AuditStore, drop_legacy_audit_files


 SAMBA_LOG_GLOB = os.getenv("AUDIT_SOURCE_GLOB", "/var/log/samba/log.*")
-ARCHIVE_DIR = os.getenv("AUDIT_ARCHIVE_DIR", "/state/audit")
-STATE_FILE = os.path.join(ARCHIVE_DIR, "collector-state.json")
+LEGACY_AUDIT_DIR = os.getenv("AUDIT_ARCHIVE_DIR", "/state/audit")
 POLL_SECONDS = max(0.2, float(os.getenv("AUDIT_POLL_SECONDS", "1")))
-COMPRESS_AFTER_HOURS = max(
-    1, int(os.getenv("AUDIT_COMPRESS_AFTER_HOURS", "24"))
-)
 AUDIT_MARKER_RE = re.compile(r"smbd_audit:\s*(.*)$")
 AUDIT_PAYLOAD_RE = re.compile(
    r"^\s*\d{4}/\d{2}/\d{2}\s+\d{2}:\d{2}:\d{2}(?:\.\d+)?\|"
@@ -43,26 +39,6 @@ def utc_now() -> dt.datetime:
    return dt.datetime.now(dt.timezone.utc)


-def atomic_json(path: str, value: object) -> None:
-    temp_path = f"{path}.tmp"
-    with open(temp_path, "w", encoding="utf-8") as handle:
-        json.dump(value, handle, separators=(",", ":"), sort_keys=True)
-        handle.flush()
-        os.fsync(handle.fileno())
-    os.replace(temp_path, path)
-
-
-def load_state() -> Dict[str, Dict[str, object]]:
-    try:
-        with open(STATE_FILE, encoding="utf-8") as handle:
-            value = json.load(handle)
-        if isinstance(value, dict):
-            return value
-    except (OSError, ValueError):
-        pass
-    return {}
-
-
 def parse_samba_timestamp(raw_line: str, fallback: dt.datetime) -> str:
    match = SAMBA_LOG_TIME_RE.match(raw_line)
    if not match:
@@ -108,29 +84,6 @@ def parse_audit_line(raw_line: str, source: str) -> Optional[Dict[str, object]]:
    }


-def archive_events(events: Iterable[Dict[str, object]]) -> int:
-    handles: Dict[str, object] = {}
-    count = 0
-    try:
-        for event in events:
-            day = str(event["ingestedAt"])[:10]
-            path = os.path.join(ARCHIVE_DIR, f"{day}.jsonl")
-            handle = handles.get(path)
-            if handle is None:
-                handle = open(path, "a", encoding="utf-8")
-                handles[path] = handle
-            handle.write(json.dumps(event, separators=(",", ":"), sort_keys=True))
-            handle.write("\n")
-            count += 1
-        for handle in handles.values():
-            handle.flush()
-            os.fsync(handle.fileno())
-    finally:
-        for handle in handles.values():
-            handle.close()
-    return count
-
-
 def read_new_events(path: str, entry: Dict[str, object]):
    stat = os.stat(path)
    inode = int(stat.st_ino)
@@ -157,60 +110,40 @@ def read_new_events(path: str, entry: Dict[str, object]):
    return events, {"inode": inode, "offset": new_offset}


-def compress_old_archives() -> None:
-    cutoff = utc_now() - dt.timedelta(hours=COMPRESS_AFTER_HOURS)
-    today = utc_now().date().isoformat()
-    for path in glob.glob(os.path.join(ARCHIVE_DIR, "????-??-??.jsonl")):
-        day = os.path.basename(path)[:10]
-        if day == today:
-            continue
-        try:
-            modified = dt.datetime.fromtimestamp(os.path.getmtime(path), dt.timezone.utc)
-            if modified > cutoff:
-                continue
-            target = f"{path}.gz"
-            temp_target = f"{target}.tmp"
-            with open(path, "rb") as source, gzip.open(temp_target, "wb", compresslevel=6) as output:
-                while True:
-                    chunk = source.read(1024 * 1024)
-                    if not chunk:
-                        break
-                    output.write(chunk)
-            os.replace(temp_target, target)
-            os.remove(path)
-            log(f"Compressed {os.path.basename(path)}")
-        except OSError as exc:
-            log(f"Unable to compress {path}: {exc}")
-
-
-def collect_once(state: Dict[str, Dict[str, object]]) -> int:
-    total = 0
+def collect_once(store: AuditStore) -> int:
+    state = store.source_entries()
+    all_events = []
+    source_updates = {}
    seen = set()
    initial_by_inode = {
        int(entry.get("inode", -1)): entry
        for entry in state.values()
-        if isinstance(entry, dict) and int(entry.get("inode", -1)) >= 0
+        if int(entry.get("inode", -1)) >= 0
    }
-    for path in sorted(glob.glob(SAMBA_LOG_GLOB)):
+    candidates = []
+    for path in glob.glob(SAMBA_LOG_GLOB):
        if not os.path.isfile(path):
            continue
+        try:
+            stat = os.stat(path)
+        except OSError as exc:
+            log(f"Unable to inspect {path}: {exc}")
+            continue
+        known_inode = int(stat.st_ino) in initial_by_inode
+        candidates.append((not known_inode, stat.st_mtime_ns, path, int(stat.st_ino)))
+
+    for _is_new, _mtime_ns, path, current_inode in sorted(candidates):
        seen.add(path)
        try:
            path_entry = state.get(path, {})
-            current_inode = os.stat(path).st_ino
            if int(path_entry.get("inode", -1)) != current_inode:
                path_entry = initial_by_inode.get(current_inode, {})
            events, new_entry = read_new_events(path, path_entry)
-            if events:
-                total += archive_events(events)
-            state[path] = new_entry
+            all_events.extend(events)
+            source_updates[path] = new_entry
        except OSError as exc:
            log(f"Unable to read {path}: {exc}")
-    for stale_path in list(state):
-        if stale_path not in seen:
-            state.pop(stale_path, None)
-    atomic_json(STATE_FILE, state)
-    return total
+    return store.append_batch(all_events, source_updates, seen)


 def stop(_signum, _frame) -> None:
@@ -219,23 +152,24 @@ def stop(_signum, _frame) -> None:


 def main() -> int:
-    os.makedirs(ARCHIVE_DIR, mode=0o750, exist_ok=True)
    signal.signal(signal.SIGTERM, stop)
    signal.signal(signal.SIGINT, stop)
-    state = load_state()
-    last_compress = 0.0
-    log(f"Watching {SAMBA_LOG_GLOB}")
-    while not STOP:
-        try:
-            count = collect_once(state)
-            if count:
-                log(f"Archived {count} event(s)")
-            if time.monotonic() - last_compress >= 3600:
-                compress_old_archives()
-                last_compress = time.monotonic()
-        except Exception as exc:  # pylint: disable=broad-except
-            log(f"Collector cycle failed: {exc}")
-        time.sleep(POLL_SECONDS)
+    store = AuditStore()
+    removed = drop_legacy_audit_files(LEGACY_AUDIT_DIR)
+    if removed:
+        log(f"Dropped {removed} legacy audit archive file(s)")
+    log(f"Watching {SAMBA_LOG_GLOB}; storing events in SQLite")
+    try:
+        while not STOP:
+            try:
+                count = collect_once(store)
+                if count:
+                    log(f"Stored {count} event(s)")
+            except Exception as exc:  # pylint: disable=broad-except
+                log(f"Collector cycle failed: {exc}")
+            time.sleep(POLL_SECONDS)
+    finally:
+        store.close()
    return 0


@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 """Shared policy for the small set of user-facing audit events."""

+import datetime as dt
 import os
-from typing import Optional, Tuple
+from typing import Mapping, Optional, Tuple


 AUDIT_ACTIONS = frozenset({"read", "write", "move", "delete"})
@@ -51,3 +52,35 @@ def account_name(user: str) -> str:
 def skip_user(user: str) -> bool:
    account = account_name(user).casefold()
    return any(account.endswith(suffix) for suffix in skipped_user_suffixes())
+
+
+ReadEventKey = Tuple[str, ...]
+
+
+def read_deduplication_key(event: Mapping[str, object]) -> Optional[ReadEventKey]:
+    if str(event.get("action", "")).casefold() != "read":
+        return None
+    try:
+        timestamp = dt.datetime.fromisoformat(
+            str(event.get("timestamp", "")).replace("Z", "+00:00")
+        )
+        if timestamp.tzinfo is None:
+            timestamp = timestamp.replace(tzinfo=dt.timezone.utc)
+        second = (
+            timestamp.astimezone(dt.timezone.utc)
+            .replace(microsecond=0)
+            .isoformat()
+        )
+    except ValueError:
+        second = str(event.get("timestamp", ""))
+
+    success = bool(event.get("success", False))
+    return (
+        second,
+        str(event.get("user", "")),
+        str(event.get("clientIp", "")),
+        str(event.get("share", "")),
+        str(event.get("path", "")),
+        "success" if success else "failure",
+        "" if success else str(event.get("result", "")),
+    )
@@ -0,0 +1,369 @@
+#!/usr/bin/env python3
+"""Indexed SQLite storage for normalized Samba audit events."""
+
+import datetime as dt
+import glob
+import os
+import sqlite3
+from typing import Dict, Iterable, List, Optional, Set, Tuple
+
+try:
+    from .audit_policy import (
+        account_name,
+        read_deduplication_key,
+        skipped_user_suffixes,
+    )
+    from .state_db import connect_state_db
+except ImportError:
+    from audit_policy import account_name, read_deduplication_key, skipped_user_suffixes
+    from state_db import connect_state_db
+
+
+AUDIT_SCHEMA = """
+CREATE TABLE IF NOT EXISTS audit_events (
+    id INTEGER PRIMARY KEY,
+    occurred_at TEXT NOT NULL,
+    occurred_second INTEGER NOT NULL,
+    ingested_at TEXT NOT NULL,
+    user TEXT NOT NULL,
+    account TEXT NOT NULL,
+    client_ip TEXT NOT NULL,
+    client TEXT NOT NULL,
+    share TEXT NOT NULL,
+    action TEXT NOT NULL CHECK (action IN ('read', 'write', 'move', 'delete')),
+    result TEXT NOT NULL,
+    success INTEGER NOT NULL CHECK (success IN (0, 1)),
+    path TEXT NOT NULL,
+    source TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS audit_sources (
+    path TEXT PRIMARY KEY,
+    inode INTEGER NOT NULL,
+    offset INTEGER NOT NULL
+) WITHOUT ROWID;
+CREATE INDEX IF NOT EXISTS audit_events_time
+    ON audit_events (occurred_second DESC, id DESC);
+CREATE INDEX IF NOT EXISTS audit_events_action_time
+    ON audit_events (action, occurred_second DESC, id DESC);
+CREATE INDEX IF NOT EXISTS audit_events_success_time
+    ON audit_events (success, occurred_second DESC, id DESC);
+CREATE INDEX IF NOT EXISTS audit_events_user_time
+    ON audit_events (user COLLATE NOCASE, occurred_second DESC, id DESC);
+CREATE INDEX IF NOT EXISTS audit_events_account_time
+    ON audit_events (account, occurred_second DESC, id DESC);
+CREATE INDEX IF NOT EXISTS audit_events_share_time
+    ON audit_events (share COLLATE NOCASE, occurred_second DESC, id DESC);
+CREATE INDEX IF NOT EXISTS audit_events_result_time
+    ON audit_events (result COLLATE NOCASE, occurred_second DESC, id DESC);
+"""
+
+
+def ensure_audit_schema(conn: sqlite3.Connection) -> None:
+    conn.executescript(AUDIT_SCHEMA)
+    conn.commit()
+
+
+def parse_event_second(timestamp: object) -> int:
+    parsed = dt.datetime.fromisoformat(str(timestamp).replace("Z", "+00:00"))
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=dt.timezone.utc)
+    return int(parsed.astimezone(dt.timezone.utc).timestamp())
+
+
+def row_to_event(row: sqlite3.Row) -> Dict[str, object]:
+    action = str(row["action"])
+    return {
+        "id": int(row["id"]),
+        "timestamp": str(row["occurred_at"]),
+        "ingestedAt": str(row["ingested_at"]),
+        "user": str(row["user"]),
+        "clientIp": str(row["client_ip"]),
+        "client": str(row["client"]),
+        "share": str(row["share"]),
+        "operation": action,
+        "action": action,
+        "result": str(row["result"]),
+        "success": bool(row["success"]),
+        "path": str(row["path"]),
+        "source": str(row["source"]),
+    }
+
+
+def last_event(conn: sqlite3.Connection) -> Optional[Dict[str, object]]:
+    row = conn.execute(
+        """
+        SELECT id, occurred_at, ingested_at, user, client_ip, client, share,
+               action, result, success, path, source
+        FROM audit_events
+        ORDER BY id DESC
+        LIMIT 1
+        """
+    ).fetchone()
+    return row_to_event(row) if row is not None else None
+
+
+class AuditStore:
+    def __init__(self, database_path: Optional[str] = None):
+        self.conn = connect_state_db(database_path)
+        ensure_audit_schema(self.conn)
+
+    def close(self) -> None:
+        self.conn.close()
+
+    def source_entries(self) -> Dict[str, Dict[str, object]]:
+        return {
+            str(row["path"]): {
+                "inode": int(row["inode"]),
+                "offset": int(row["offset"]),
+            }
+            for row in self.conn.execute(
+                "SELECT path, inode, offset FROM audit_sources"
+            )
+        }
+
+    def append_batch(
+        self,
+        events: Iterable[Dict[str, object]],
+        source_updates: Dict[str, Dict[str, object]],
+        seen_paths: Set[str],
+    ) -> int:
+        inserted = 0
+        self.conn.execute("BEGIN IMMEDIATE")
+        try:
+            previous_key = read_deduplication_key(last_event(self.conn) or {})
+            for event in events:
+                current_key = read_deduplication_key(event)
+                if current_key is not None and current_key == previous_key:
+                    continue
+                self.conn.execute(
+                    """
+                    INSERT INTO audit_events (
+                        occurred_at, occurred_second, ingested_at, user, account,
+                        client_ip, client, share, action, result, success, path,
+                        source
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        str(event["timestamp"]),
+                        parse_event_second(event["timestamp"]),
+                        str(event["ingestedAt"]),
+                        str(event["user"]),
+                        account_name(str(event["user"])).casefold(),
+                        str(event["clientIp"]),
+                        str(event["client"]),
+                        str(event["share"]),
+                        str(event["action"]),
+                        str(event["result"]),
+                        int(bool(event["success"])),
+                        str(event["path"]),
+                        str(event["source"]),
+                    ),
+                )
+                inserted += 1
+                previous_key = current_key
+
+            for path, entry in source_updates.items():
+                self.conn.execute(
+                    """
+                    INSERT INTO audit_sources (path, inode, offset)
+                    VALUES (?, ?, ?)
+                    ON CONFLICT(path) DO UPDATE SET
+                        inode = excluded.inode,
+                        offset = excluded.offset
+                    """,
+                    (path, int(entry["inode"]), int(entry["offset"])),
+                )
+            for row in self.conn.execute("SELECT path FROM audit_sources").fetchall():
+                path = str(row["path"])
+                if path not in seen_paths:
+                    self.conn.execute("DELETE FROM audit_sources WHERE path = ?", (path,))
+            self.conn.commit()
+        except Exception:
+            self.conn.rollback()
+            raise
+        return inserted
+
+
+def escape_like(value: str) -> str:
+    return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+
+
+def date_seconds(day: dt.date) -> int:
+    return int(
+        dt.datetime.combine(day, dt.time.min, tzinfo=dt.timezone.utc).timestamp()
+    )
+
+
+def activity_conditions(
+    start: dt.date,
+    end: dt.date,
+    params: Dict[str, List[str]],
+    *,
+    include_filters: bool,
+) -> Tuple[List[str], List[object]]:
+    conditions = ["occurred_second >= ?", "occurred_second < ?"]
+    values: List[object] = [
+        date_seconds(start),
+        date_seconds(end + dt.timedelta(days=1)),
+    ]
+    for suffix in skipped_user_suffixes():
+        conditions.append("lower(account) NOT LIKE ? ESCAPE '\\'")
+        values.append(f"%{escape_like(suffix)}")
+
+    if not include_filters:
+        return conditions, values
+
+    filters = {
+        key: params.get(key, [""])[0].casefold().strip()
+        for key in ("user", "share", "operation", "action", "path", "result")
+    }
+    user = filters["user"]
+    if user:
+        if "\\" in user or "@" in user:
+            conditions.append("user = ? COLLATE NOCASE")
+        else:
+            conditions.append("account = ?")
+        values.append(user)
+    share = filters["share"]
+    if share:
+        conditions.append("share = ? COLLATE NOCASE")
+        values.append(share)
+    path = filters["path"]
+    if path:
+        conditions.append("lower(path) LIKE ? ESCAPE '\\'")
+        values.append(f"%{escape_like(path)}%")
+    action = filters["action"] or filters["operation"]
+    if action:
+        conditions.append("action = ?")
+        values.append(action)
+    result = filters["result"]
+    if result == "fail":
+        conditions.append("success = 0")
+    elif result:
+        conditions.append("result = ? COLLATE NOCASE")
+        values.append(result)
+    return conditions, values
+
+
+def query_activity(
+    conn: sqlite3.Connection,
+    start: dt.date,
+    end: dt.date,
+    params: Dict[str, List[str]],
+) -> Dict[str, object]:
+    limit = min(500, max(1, int(params.get("limit", ["100"])[0])))
+    conditions, values = activity_conditions(start, end, params, include_filters=True)
+    where_sql = " AND ".join(conditions)
+    matched = int(
+        conn.execute(
+            f"SELECT count(*) FROM audit_events WHERE {where_sql}",
+            values,
+        ).fetchone()[0]
+    )
+
+    cursor = params.get("cursor", [""])[0].strip()
+    page_conditions = list(conditions)
+    page_values = list(values)
+    if cursor:
+        try:
+            cursor_second, cursor_id = (int(part) for part in cursor.split(":", 1))
+        except (TypeError, ValueError) as exc:
+            raise ValueError("Ungültiger Seitenzeiger") from exc
+        page_conditions.append(
+            "(occurred_second < ? OR (occurred_second = ? AND id < ?))"
+        )
+        page_values.extend((cursor_second, cursor_second, cursor_id))
+
+    rows = conn.execute(
+        f"""
+        SELECT id, occurred_at, occurred_second, ingested_at, user, client_ip,
+               client, share, action, result, success, path, source
+        FROM audit_events
+        WHERE {" AND ".join(page_conditions)}
+        ORDER BY occurred_second DESC, id DESC
+        LIMIT ?
+        """,
+        (*page_values, limit + 1),
+    ).fetchall()
+    has_more = len(rows) > limit
+    rows = rows[:limit]
+    events = [row_to_event(row) for row in rows]
+    next_cursor = None
+    if has_more and rows:
+        next_cursor = f"{int(rows[-1]['occurred_second'])}:{int(rows[-1]['id'])}"
+
+    facet_conditions, facet_values = activity_conditions(
+        start, end, params, include_filters=False
+    )
+    facet_where = " AND ".join(facet_conditions)
+
+    def distinct(column: str) -> List[str]:
+        return [
+            str(row[0])
+            for row in conn.execute(
+                f"""
+                SELECT DISTINCT {column}
+                FROM audit_events
+                WHERE {facet_where} AND {column} <> ''
+                ORDER BY {column} COLLATE NOCASE
+                """,
+                facet_values,
+            )
+        ]
+
+    actions = distinct("action")
+    return {
+        "events": events,
+        "nextCursor": next_cursor,
+        "matched": matched,
+        "facets": {
+            "users": distinct("user"),
+            "shares": distinct("share"),
+            "operations": actions,
+            "actions": actions,
+        },
+    }
+
+
+def audit_summary(conn: sqlite3.Connection, database_path: str) -> Dict[str, object]:
+    row = conn.execute(
+        """
+        SELECT count(DISTINCT substr(occurred_at, 1, 10)),
+               min(substr(occurred_at, 1, 10)),
+               max(substr(occurred_at, 1, 10))
+        FROM audit_events
+        """
+    ).fetchone()
+    database_bytes = 0
+    for suffix in ("", "-wal"):
+        try:
+            database_bytes += os.path.getsize(f"{database_path}{suffix}")
+        except OSError:
+            pass
+    return {
+        "days": int(row[0] or 0),
+        "bytes": database_bytes,
+        "oldest": row[1],
+        "newest": row[2],
+    }
+
+
+def drop_legacy_audit_files(directory: str) -> int:
+    removed = 0
+    patterns = (
+        "????-??-??.jsonl",
+        "????-??-??.jsonl.gz",
+        "collector-state.json",
+        "collector-state.json.tmp",
+    )
+    for pattern in patterns:
+        for path in glob.glob(os.path.join(directory, pattern)):
+            if not os.path.isfile(path):
+                continue
+            os.remove(path)
+            removed += 1
+    try:
+        os.rmdir(directory)
+    except OSError:
+        pass
+    return removed
@@ -5,6 +5,8 @@ import fcntl
 import json
 import os
 import re
+import shutil
+import sqlite3
 import subprocess
 import sys
 import tempfile
@@ -14,7 +16,12 @@ from typing import Callable, Dict, List, Optional, Set, TextIO, Tuple
 from urllib.parse import SplitResult, unquote, urlsplit


-LOCK_PATH = "/state/backup.lock"
+STATE_ROOT = os.getenv("STATE_ROOT", "/state")
+STATE_DB_PATH = os.getenv(
+    "STATE_DB_PATH",
+    os.getenv("SHARE_DB_PATH", os.path.join(STATE_ROOT, "shares.db")),
+)
+LOCK_PATH = os.path.join(STATE_ROOT, "backup.lock")
 DEFAULT_BACKUP_LOG_FILE = "/var/log/backup.log"
 DEFAULT_BACKUP_STATUS_FILE = "/state/backup-status.json"
 DEFAULT_PROGRESS_MODE = "auto"
@@ -43,7 +50,7 @@ BACKUP_SOURCES: List[Tuple[str, str]] = [
    ("/data/private", "data/private"),
    ("/data/groups", "data/groups"),
    ("/data/fslogix", "data/fslogix"),
-    ("/state", "state"),
+    (STATE_ROOT, "state"),
    ("/var/lib/samba/private", "samba/private"),
 ]

@@ -917,11 +924,75 @@ def redact_destination(raw_url: str) -> str:
    return f"{parts.scheme}://{netloc}{parts.path}"


-def available_sources() -> List[Tuple[str, str]]:
+def prepare_state_snapshot(
+    state_root: str = STATE_ROOT,
+    database_path: str = STATE_DB_PATH,
+) -> Tuple[str, str]:
+    """Copy state and replace the live WAL database with an online snapshot."""
+    state_root = os.path.abspath(state_root)
+    database_path = os.path.abspath(database_path)
+    database_relative = os.path.relpath(database_path, state_root)
+    if database_relative == ".." or database_relative.startswith(f"..{os.sep}"):
+        raise RuntimeError("STATE_DB_PATH must be located below STATE_ROOT")
+
+    temporary_root = tempfile.mkdtemp(prefix="backup-state-")
+    staged_state = os.path.join(temporary_root, "state")
+    excluded = {
+        database_relative,
+        f"{database_relative}-wal",
+        f"{database_relative}-shm",
+    }
+
+    def ignore_live_database(directory: str, names: List[str]) -> Set[str]:
+        relative_directory = os.path.relpath(directory, state_root)
+        return {
+            name
+            for name in names
+            if os.path.normpath(os.path.join(relative_directory, name)) in excluded
+        }
+
+    try:
+        shutil.copytree(
+            state_root,
+            staged_state,
+            symlinks=True,
+            ignore=ignore_live_database,
+        )
+        if os.path.isfile(database_path):
+            staged_database = os.path.join(staged_state, database_relative)
+            os.makedirs(os.path.dirname(staged_database), exist_ok=True)
+            source = sqlite3.connect(
+                f"file:{database_path}?mode=ro",
+                uri=True,
+                timeout=30,
+            )
+            destination = sqlite3.connect(staged_database, timeout=30)
+            try:
+                source.execute("PRAGMA busy_timeout = 30000")
+                source.backup(destination)
+                if destination.execute("PRAGMA quick_check").fetchone()[0] != "ok":
+                    raise RuntimeError("SQLite state snapshot failed its integrity check")
+            finally:
+                destination.close()
+                source.close()
+        return temporary_root, staged_state
+    except Exception:
+        shutil.rmtree(temporary_root, ignore_errors=True)
+        raise
+
+
+def available_sources(
+    state_snapshot: Optional[str] = None,
+) -> List[Tuple[str, str]]:
    sources: List[Tuple[str, str]] = []
    for source_path, destination_path in BACKUP_SOURCES:
-        if os.path.isdir(source_path):
-            sources.append((source_path, destination_path))
+        effective_source = (
+            state_snapshot
+            if destination_path == "state" and state_snapshot is not None
+            else source_path
+        )
+        if os.path.isdir(effective_source):
+            sources.append((effective_source, destination_path))
        else:
            log(f"Skipping missing source: {source_path}")
    return sources
@@ -1455,11 +1526,6 @@ def run_backup() -> int:
        return 0

    policy = parse_retention_policy()
-    sources = available_sources()
-    if not sources:
-        log("No backup sources are available, skipping backup")
-        return 0
-
    progress_interval_seconds = parse_int_env(
        "BACKUP_PROGRESS_INTERVAL_SECONDS",
        DEFAULT_PROGRESS_INTERVAL_SECONDS,
@@ -1474,8 +1540,19 @@ def run_backup() -> int:
        os.getenv("BACKUP_STATUS_FILE", DEFAULT_BACKUP_STATUS_FILE).strip()
    )
    BACKUP_STATUS.begin(redact_destination(destination.raw_url))
-    backend = build_backend(destination)
+    backend = None
+    state_snapshot_root = None
    try:
+        state_snapshot = None
+        if os.path.isdir(STATE_ROOT):
+            BACKUP_STATUS.write(message="Creating consistent state database snapshot")
+            state_snapshot_root, state_snapshot = prepare_state_snapshot()
+        sources = available_sources(state_snapshot)
+        if not sources:
+            log("No backup sources are available, skipping backup")
+            return 0
+
+        backend = build_backend(destination)
        log(f"Starting backup to {redact_destination(destination.raw_url)}")

        existing = set(backend.list_snapshots())
@@ -1494,7 +1571,7 @@ def run_backup() -> int:
        )

        for source_path, destination_path in sources:
-            log(f"Syncing {source_path}")
+            log(f"Syncing {destination_path}")
            backend.sync_source(
                snapshot_name,
                source_path,
@@ -1527,7 +1604,10 @@ def run_backup() -> int:
        )
        return 0
    finally:
-        backend.close()
+        if backend is not None:
+            backend.close()
+        if state_snapshot_root is not None:
+            shutil.rmtree(state_snapshot_root, ignore_errors=True)


 def with_lock() -> int:
@@ -584,7 +584,7 @@ if [[ -n "${JOIN_PASSWORD:-}" ]]; then
  export JOIN_PASSWORD
 fi

-mkdir -p /data/private /data/fslogix /data/groups/data /data/groups/archive /state /state/audit /state/web /var/log/samba
+mkdir -p /data/private /data/fslogix /data/groups/data /data/groups/archive /state /var/log/samba
 touch /var/log/reconcile.log /var/log/backup.log

 append_winbind_to_nss
@@ -15,8 +15,13 @@ import uuid
 from dataclasses import dataclass, field
 from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple

+try:
+    from .state_db import STATE_DB_PATH, connect_state_db
+except ImportError:
+    from state_db import STATE_DB_PATH, connect_state_db

-DB_PATH = "/state/shares.db"
+
+DB_PATH = STATE_DB_PATH
 LOCK_PATH = "/state/reconcile.lock"
 GROUP_ROOT = "/data/groups/data"
 GROUP_ARCHIVE_ROOT = "/data/groups/archive"
@@ -720,9 +725,7 @@ def ensure_share_db_schema(conn: sqlite3.Connection) -> None:


 def open_db() -> sqlite3.Connection:
-    os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
-    conn = sqlite3.connect(DB_PATH)
-    conn.row_factory = sqlite3.Row
+    conn = connect_state_db(DB_PATH)
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS shares (
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Connections to the shared persistent SQLite state database."""
+
+import os
+import sqlite3
+from typing import Optional
+
+
+STATE_DB_PATH = os.getenv(
+    "STATE_DB_PATH",
+    os.getenv("SHARE_DB_PATH", "/state/shares.db"),
+)
+
+
+def connect_state_db(
+    path: Optional[str] = None,
+    *,
+    read_only: bool = False,
+) -> sqlite3.Connection:
+    database_path = path or STATE_DB_PATH
+    if read_only:
+        conn = sqlite3.connect(
+            f"file:{database_path}?mode=ro",
+            uri=True,
+            timeout=30,
+        )
+    else:
+        directory = os.path.dirname(database_path)
+        if directory:
+            os.makedirs(directory, exist_ok=True)
+        conn = sqlite3.connect(database_path, timeout=30)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA busy_timeout = 30000")
+    conn.execute("PRAGMA foreign_keys = ON")
+    if not read_only:
+        conn.execute("PRAGMA journal_mode = WAL")
+        conn.execute("PRAGMA synchronous = NORMAL")
+    return conn
+
+
+def ensure_web_cache_schema(conn: sqlite3.Connection) -> None:
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS web_cache (
+            key TEXT PRIMARY KEY,
+            value TEXT NOT NULL,
+            updated_at TEXT NOT NULL
+        )
+        """
+    )
+    conn.commit()
+
+
+def drop_legacy_web_cache(directory: Optional[str] = None) -> int:
+    legacy_directory = directory or os.getenv("WEB_STATE_DIR", "/state/web")
+    removed = 0
+    for name in ("usage.json", "usage.json.tmp"):
+        path = os.path.join(legacy_directory, name)
+        try:
+            os.remove(path)
+            removed += 1
+        except FileNotFoundError:
+            pass
+    try:
+        os.rmdir(legacy_directory)
+    except OSError:
+        pass
+    return removed
+
@@ -306,12 +306,12 @@ async function renderSystem() {
    <section class="panel"><div class="panel-head"><h2>Dienstprüfungen</h2><span>${esc(data.hostname)}</span></div><div class="check-list">
      <div class="check"><span>Domänenvertrauen</span>${data.checks.domainTrust ? badge("In Ordnung") : badge("Fehlgeschlagen", "error")}</div>
      <div class="check"><span>Samba-Konfiguration</span>${data.checks.sambaConfig ? badge("Gültig") : badge("Fehlgeschlagen", "error")}</div>
-      <div class="check"><span>Aktivitätsarchiv</span><strong>${data.audit.days} Tage · ${bytes(data.audit.bytes)}</strong></div>
+      <div class="check"><span>Aktivitätsdatenbank</span><strong>${data.audit.days} Tage · ${bytes(data.audit.bytes)}</strong></div>
      <div class="check"><span>Speicherprüfung</span><strong>${usage.scannedAt ? esc(utcTime(usage.scannedAt)) : "Ausstehend"}</strong></div>
    </div></section>
    <section class="split">
      <article class="panel"><h2>TLS-Zertifikat</h2><dl><dt>Allgemeiner Name</dt><dd>${esc(data.tls.subject?.commonName || "—")}</dd><dt>Aussteller</dt><dd>${esc(data.tls.issuer?.commonName || "—")}</dd><dt>Gültig bis</dt><dd>${esc(utcTime(data.tls.notAfter))}</dd><dt>Namen</dt><dd>${esc((data.tls.sans || []).map(value => value[1]).join(", ") || "—")}</dd></dl></article>
-      <article class="panel"><h2>Protokollaufbewahrung</h2><dl><dt>Ältester UTC-Tag</dt><dd>${esc(data.audit.oldest || "—")}</dd><dt>Neuester UTC-Tag</dt><dd>${esc(data.audit.newest || "—")}</dd><dt>Archivgröße</dt><dd>${bytes(data.audit.bytes)}</dd></dl><p class="muted">Abgeschlossene Tagesdateien werden automatisch komprimiert und aufbewahrt, bis ein Administrator sie entfernt.</p></article>
+      <article class="panel"><h2>Protokollspeicherung</h2><dl><dt>Ältester UTC-Tag</dt><dd>${esc(data.audit.oldest || "—")}</dd><dt>Neuester UTC-Tag</dt><dd>${esc(data.audit.newest || "—")}</dd><dt>Datenbankgröße</dt><dd>${bytes(data.audit.bytes)}</dd></dl><p class="muted">Ereignisse werden dauerhaft und indexiert in der gemeinsamen SQLite-Datenbank gespeichert.</p></article>
    </section>`;
 }

@@ -3,7 +3,6 @@

 import base64
 import datetime as dt
-import gzip
 import hashlib
 import hmac
 import http.cookies
@@ -23,7 +22,7 @@ import urllib.parse
 from collections import deque
 from http import HTTPStatus
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from typing import Dict, Iterable, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple

 try:
    from app import reconcile_shares as directory
@@ -31,20 +30,37 @@ except ImportError:  # Container execution uses /app as the import root.
    import reconcile_shares as directory

 try:
-    from app.audit_policy import AUDIT_ACTIONS, action_for, skip_user
+    from app.audit_store import (
+        audit_summary as sqlite_audit_summary,
+        ensure_audit_schema,
+        query_activity,
+    )
+    from app.state_db import (
+        STATE_DB_PATH,
+        connect_state_db,
+        drop_legacy_web_cache,
+        ensure_web_cache_schema,
+    )
 except ImportError:  # Container execution uses /app as the import root.
-    from audit_policy import AUDIT_ACTIONS, action_for, skip_user
+    from audit_store import (
+        audit_summary as sqlite_audit_summary,
+        ensure_audit_schema,
+        query_activity,
+    )
+    from state_db import (
+        STATE_DB_PATH,
+        connect_state_db,
+        drop_legacy_web_cache,
+        ensure_web_cache_schema,
+    )


-STATE_ROOT = os.getenv("WEB_STATE_DIR", "/state/web")
-AUDIT_ROOT = os.getenv("AUDIT_ARCHIVE_DIR", "/state/audit")
 STATIC_ROOT = os.getenv("WEB_STATIC_DIR", "/app/web")
-SHARE_DB = os.getenv("SHARE_DB_PATH", "/state/shares.db")
+STATE_DB = STATE_DB_PATH
 BACKUP_STATUS_FILE = os.getenv("BACKUP_STATUS_FILE", "/state/backup-status.json")
 BACKUP_LOG_FILE = os.getenv("BACKUP_LOG_FILE", "/var/log/backup.log")
 TLS_CERT_FILE = os.getenv("WEB_TLS_CERT_FILE", "/state/tls/web.crt")
 TLS_KEY_FILE = os.getenv("WEB_TLS_KEY_FILE", "/state/tls/web.key")
-USAGE_CACHE_FILE = os.path.join(STATE_ROOT, "usage.json")
 JWT_COOKIE = "adfs_session"
 JWT_ISSUER = "ad-file-server-web"
 JWT_AUDIENCE = "domain-admins"
@@ -69,16 +85,6 @@ def env_int(name: str, default: int, minimum: int, maximum: int) -> int:
        return default


-def atomic_json(path: str, value: object) -> None:
-    os.makedirs(os.path.dirname(path), exist_ok=True)
-    temp = f"{path}.tmp"
-    with open(temp, "w", encoding="utf-8") as handle:
-        json.dump(value, handle, separators=(",", ":"), sort_keys=True)
-        handle.flush()
-        os.fsync(handle.fileno())
-    os.replace(temp, path)
-
-
 def read_json(path: str, default):
    try:
        with open(path, encoding="utf-8") as handle:
@@ -316,11 +322,49 @@ def scan_children(root: str) -> List[Dict[str, object]]:
    return rows


+def load_usage_cache() -> Dict[str, object]:
+    try:
+        conn = connect_state_db(STATE_DB)
+        try:
+            ensure_web_cache_schema(conn)
+            row = conn.execute(
+                "SELECT value FROM web_cache WHERE key = 'usage'"
+            ).fetchone()
+            value = json.loads(str(row[0])) if row is not None else {}
+            return value if isinstance(value, dict) else {}
+        finally:
+            conn.close()
+    except (sqlite3.Error, ValueError):
+        return {}
+
+
+def store_usage_cache(value: Dict[str, object]) -> None:
+    conn = connect_state_db(STATE_DB)
+    try:
+        ensure_web_cache_schema(conn)
+        with conn:
+            conn.execute(
+                """
+                INSERT INTO web_cache (key, value, updated_at)
+                VALUES ('usage', ?, ?)
+                ON CONFLICT(key) DO UPDATE SET
+                    value = excluded.value,
+                    updated_at = excluded.updated_at
+                """,
+                (
+                    json.dumps(value, separators=(",", ":"), sort_keys=True),
+                    now_utc().isoformat(timespec="seconds"),
+                ),
+            )
+    finally:
+        conn.close()
+
+
 class UsageScanner:
    def __init__(self):
        self.interval = env_int("WEB_USAGE_SCAN_INTERVAL_SECONDS", 900, 60, 86400)
        self.lock = threading.Lock()
-        self.data = read_json(USAGE_CACHE_FILE, {})
+        self.data = load_usage_cache()
        self.stop = threading.Event()

    def snapshot(self) -> Dict[str, object]:
@@ -368,7 +412,7 @@ class UsageScanner:
                "fslogixBytes": sum(int(row["fslogixBytes"]) for row in user_rows),
            },
        }
-        atomic_json(USAGE_CACHE_FILE, value)
+        store_usage_cache(value)
        with self.lock:
            self.data = value
        return value
@@ -445,7 +489,7 @@ class DirectoryCache:

        folder_map = {}
        try:
-            conn = sqlite3.connect(f"file:{SHARE_DB}?mode=ro", uri=True)
+            conn = connect_state_db(STATE_DB, read_only=True)
            try:
                folder_map = {
                    row[0]: {"folder": os.path.basename(row[1]), "active": bool(row[2])}
@@ -515,60 +559,6 @@ class DirectoryCache:
        }


-def iter_audit_file(path: str) -> Iterable[Dict[str, object]]:
-    opener = gzip.open if path.endswith(".gz") else open
-    try:
-        with opener(path, "rt", encoding="utf-8", errors="replace") as handle:
-            for line in handle:
-                try:
-                    value = json.loads(line)
-                    if isinstance(value, dict):
-                        yield value
-                except json.JSONDecodeError:
-                    continue
-    except OSError:
-        return
-
-
-def iter_audit_file_reverse(path: str) -> Iterable[Dict[str, object]]:
-    if path.endswith(".gz"):
-        yield from reversed(list(iter_audit_file(path)))
-        return
-    try:
-        with open(path, "rb") as handle:
-            position = handle.seek(0, os.SEEK_END)
-            remainder = b""
-            while position > 0:
-                size = min(1024 * 1024, position)
-                position -= size
-                handle.seek(position)
-                parts = (handle.read(size) + remainder).split(b"\n")
-                remainder = parts[0]
-                for line in reversed(parts[1:]):
-                    if not line:
-                        continue
-                    try:
-                        value = json.loads(line.decode("utf-8", errors="replace"))
-                        if isinstance(value, dict):
-                            yield value
-                    except (json.JSONDecodeError, UnicodeDecodeError):
-                        continue
-            if remainder:
-                try:
-                    value = json.loads(remainder.decode("utf-8", errors="replace"))
-                    if isinstance(value, dict):
-                        yield value
-                except (json.JSONDecodeError, UnicodeDecodeError):
-                    pass
-    except OSError:
-        return
-
-
-def date_range(start: dt.date, end: dt.date):
-    day = start
-    while day <= end:
-        yield day.isoformat()
-        day += dt.timedelta(days=1)


 def query_audit(params: Dict[str, List[str]]) -> Dict[str, object]:
@@ -582,62 +572,11 @@ def query_audit(params: Dict[str, List[str]]) -> Dict[str, object]:
    max_days = env_int("AUDIT_QUERY_MAX_DAYS", 31, 1, 366)
    if end < start or (end - start).days >= max_days:
        raise ValueError(f"Der Datumsbereich darf höchstens {max_days} Tage umfassen")
-    limit = min(500, max(1, int(params.get("limit", ["100"])[0])))
-    offset = max(0, int(params.get("cursor", ["0"])[0]))
-    filters = {
-        key: params.get(key, [""])[0].casefold().strip()
-        for key in ("user", "share", "operation", "action", "path", "result")
-    }
-    page = []
-    matched = 0
-    facets = {"users": set(), "shares": set(), "operations": set(), "actions": set()}
-    for day in reversed(list(date_range(start, end))):
-        candidates = [os.path.join(AUDIT_ROOT, f"{day}.jsonl"), os.path.join(AUDIT_ROOT, f"{day}.jsonl.gz")]
-        for path in candidates:
-            if not os.path.isfile(path):
-                continue
-            for event in iter_audit_file_reverse(path):
-                user = str(event.get("user", ""))
-                operation = str(event.get("operation", ""))
-                action = action_for(operation)
-                if action is None and not operation:
-                    stored_action = str(event.get("action", "")).casefold()
-                    action = stored_action if stored_action in AUDIT_ACTIONS else None
-                if action is None or skip_user(user):
-                    continue
-                if event.get("action") != action:
-                    event = {**event, "action": action}
-                facets["users"].add(user)
-                facets["shares"].add(str(event.get("share", "")))
-                facets["operations"].add(operation)
-                facets["actions"].add(action)
-                failed_filter = filters["result"] == "fail"
-                if failed_filter and bool(event.get("success", False)):
-                    continue
-                if (
-                    filters["result"]
-                    and not failed_filter
-                    and filters["result"]
-                    not in str(event.get("result", "")).casefold()
-                ):
-                    continue
-                if any(
-                    value and value not in str(event.get(key, "")).casefold()
-                    for key, value in filters.items()
-                    if key != "result"
-                ):
-                    continue
-                if matched >= offset and len(page) < limit:
-                    page.append(event)
-                matched += 1
-    page.sort(key=lambda event: str(event.get("timestamp", "")), reverse=True)
-    next_cursor = offset + limit if offset + limit < matched else None
-    return {
-        "events": page,
-        "nextCursor": next_cursor,
-        "matched": matched,
-        "facets": {key: sorted(value, key=str.casefold) for key, value in facets.items()},
-    }
+    conn = connect_state_db(STATE_DB, read_only=True)
+    try:
+        return query_activity(conn, start, end, params)
+    finally:
+        conn.close()


 def tail_lines(path: str, count: int) -> List[str]:
@@ -666,7 +605,7 @@ def backup_payload() -> Dict[str, object]:

 def share_count() -> int:
    try:
-        conn = sqlite3.connect(f"file:{SHARE_DB}?mode=ro", uri=True)
+        conn = connect_state_db(STATE_DB, read_only=True)
        try:
            return int(conn.execute("SELECT count(*) FROM shares WHERE isActive = 1").fetchone()[0])
        finally:
@@ -676,21 +615,14 @@ def share_count() -> int:


 def audit_archive_summary() -> Dict[str, object]:
-    files = []
-    total_bytes = 0
    try:
-        names = os.listdir(AUDIT_ROOT)
-    except OSError:
-        names = []
-    for name in names:
-        if re.match(r"^\d{4}-\d{2}-\d{2}\.jsonl(?:\.gz)?$", name):
-            path = os.path.join(AUDIT_ROOT, name)
-            try:
-                total_bytes += os.path.getsize(path)
-                files.append(name)
-            except OSError:
-                continue
-    return {"days": len(files), "bytes": total_bytes, "oldest": min(files)[:10] if files else None, "newest": max(files)[:10] if files else None}
+        conn = connect_state_db(STATE_DB, read_only=True)
+        try:
+            return sqlite_audit_summary(conn, STATE_DB)
+        finally:
+            conn.close()
+    except sqlite3.Error:
+        return {"days": 0, "bytes": 0, "oldest": None, "newest": None}


 def tls_summary() -> Dict[str, object]:
@@ -711,7 +643,16 @@ class App:
    def __init__(self):
        secret = os.environ.get("WEB_JWT_SECRET", "")
        self.tokens = TokenManager(secret, env_int("WEB_JWT_TTL_SECONDS", 28800, 300, 604800))
+        removed = drop_legacy_web_cache()
+        if removed:
+            log(f"Dropped {removed} legacy web cache file(s)")
        self.usage = UsageScanner()
+        conn = connect_state_db(STATE_DB)
+        try:
+            ensure_audit_schema(conn)
+            ensure_web_cache_schema(conn)
+        finally:
+            conn.close()
        self.directory = DirectoryCache()
        threading.Thread(target=self.usage.run, name="usage-scanner", daemon=True).start()

@@ -916,7 +857,6 @@ def serve_https() -> None:

 def main() -> int:
    global APP
-    os.makedirs(STATE_ROOT, mode=0o750, exist_ok=True)
    if not os.path.isfile(TLS_CERT_FILE) or not os.path.isfile(TLS_KEY_FILE):
        raise RuntimeError("TLS certificate or key is missing")
    APP = App()