lehnert.cloud/ad-ds-simple-file-server
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

even more performance benefits

Browse Source
This commit is contained in:
Ludwig Lehnert
2026-08-12 13:26:31 +00:00
parent 0ea17c6401
commit 29340d778d
5 changed files with 353 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/audit_policy.py
+37 -7
View File
@@ -2,10 +2,10 @@
"""Shared policy for the small set of user-facing audit events."""
import datetime as dt
import hashlib
import os
from typing import Mapping, Optional, Tuple
AUDIT_ACTIONS = frozenset({"read", "write", "move", "delete"})
OPERATION_ACTIONS = {
@@ -30,11 +30,9 @@ OPERATION_ACTIONS = {
"rmdir": "delete",
}
def action_for(operation: str) -> Optional[str]:
return OPERATION_ACTIONS.get(operation.strip().casefold())
def skipped_user_suffixes() -> Tuple[str, ...]:
raw = os.getenv("AUDIT_SKIP_USER_SUFFIXES", "_svc,_ServiceAcc")
return tuple(
@@ -43,20 +41,16 @@ def skipped_user_suffixes() -> Tuple[str, ...]:
if suffix.strip()
)
def account_name(user: str) -> str:
account = user.strip().rsplit("\\", 1)[-1]
return account.split("@", 1)[0]
def skip_user(user: str) -> bool:
account = account_name(user).casefold()
return any(account.endswith(suffix) for suffix in skipped_user_suffixes())
ReadEventKey = Tuple[str, ...]
def read_deduplication_key(event: Mapping[str, object]) -> Optional[ReadEventKey]:
if str(event.get("action", "")).casefold() != "read":
return None
@@ -84,3 +78,39 @@ def read_deduplication_key(event: Mapping[str, object]) -> Optional[ReadEventKey
"success" if success else "failure",
"" if success else str(event.get("result", "")),
)
def read_deduplication_fingerprint(
event: Mapping[str, object]
) -> Optional[bytes]:
key = read_deduplication_key(event)
if key is None:
return None
digest = hashlib.blake2b(digest_size=16)
for value in key:
encoded = value.encode("utf-8", errors="surrogatepass")
digest.update(len(encoded).to_bytes(4, "big"))
digest.update(encoded)
return digest.digest()
def read_deduplication_fingerprint_values(
timestamp: object,
user: object,
client_ip: object,
share: object,
path: object,
success: object,
result: object,
) -> bytes:
"""Fingerprint legacy SQLite columns with the live-ingest policy."""
fingerprint = read_deduplication_fingerprint({
"action": "read",
"timestamp": timestamp,
"user": user,
"clientIp": client_ip,
"share": share,
"path": path,
"success": bool(success),
"result": result,
})
assert fingerprint is not None
return fingerprint
app/audit_store.py
+163 -29
View File
@@ -11,14 +11,16 @@ from typing import Dict, Iterable, List, Optional, Set, Tuple
try:
from .audit_policy import (
account_name,
read_deduplication_key,
read_deduplication_fingerprint,
read_deduplication_fingerprint_values,
skipped_user_suffixes,
)
from .state_db import connect_state_db
except ImportError:
from audit_policy import (
account_name,
read_deduplication_key,
read_deduplication_fingerprint,
read_deduplication_fingerprint_values,
skipped_user_suffixes,
)
from state_db import connect_state_db
@@ -47,6 +49,13 @@ CREATE TABLE IF NOT EXISTS audit_sources (
inode INTEGER NOT NULL,
offset INTEGER NOT NULL
) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS audit_read_dedup (
fingerprint BLOB PRIMARY KEY,
occurred_second INTEGER NOT NULL,
event_id INTEGER
) WITHOUT ROWID;
CREATE INDEX IF NOT EXISTS audit_read_dedup_time
ON audit_read_dedup (occurred_second);
CREATE INDEX IF NOT EXISTS audit_events_main_time
ON audit_events (occurred_second DESC, id DESC)
WHERE share <> 'FSLogix' COLLATE NOCASE;
@@ -138,7 +147,7 @@ CREATE TABLE IF NOT EXISTS audit_rollup_state (
backfill_next_id INTEGER NOT NULL,
backfill_max_id INTEGER NOT NULL,
ready INTEGER NOT NULL CHECK (ready IN (0, 1)),
policy_version INTEGER NOT NULL DEFAULT 3
policy_version INTEGER NOT NULL DEFAULT 4
);
"""
@@ -212,7 +221,7 @@ def ensure_audit_schema(conn: sqlite3.Connection) -> None:
WHERE singleton = 1
"""
).fetchone()
if state is not None and int(state["policy_version"]) < 3:
if state is not None and int(state["policy_version"]) < 4:
for table in (
"audit_daily_totals",
"audit_daily_counts",
@@ -223,7 +232,7 @@ def ensure_audit_schema(conn: sqlite3.Connection) -> None:
"""
UPDATE audit_rollup_state
SET backfill_next_id = 1, backfill_max_id = ?, ready = ?,
policy_version = 3
policy_version = 4
WHERE singleton = 1
""",
(max_id, int(max_id == 0)),
@@ -231,6 +240,21 @@ def ensure_audit_schema(conn: sqlite3.Connection) -> None:
conn.commit()
READ_DEDUP_DEFAULT_WINDOW_SECONDS = 2 * 86400
def read_deduplication_cutoff() -> int:
try:
window = int(os.getenv(
"AUDIT_READ_DEDUP_WINDOW_SECONDS",
str(READ_DEDUP_DEFAULT_WINDOW_SECONDS),
))
except ValueError:
window = READ_DEDUP_DEFAULT_WINDOW_SECONDS
now = int(dt.datetime.now(dt.timezone.utc).timestamp())
return now - max(1, window)
def parse_event_second(timestamp: object) -> int:
parsed = dt.datetime.fromisoformat(str(timestamp).replace("Z", "+00:00"))
if parsed.tzinfo is None:
@@ -257,22 +281,13 @@ def row_to_event(row: sqlite3.Row) -> Dict[str, object]:
}
def last_event(conn: sqlite3.Connection) -> Optional[Dict[str, object]]:
row = conn.execute(
"""
SELECT id, occurred_at, ingested_at, user, client_ip, client, share,
action, result, success, path, source
FROM audit_events
ORDER BY id DESC
LIMIT 1
"""
).fetchone()
return row_to_event(row) if row is not None else None
class AuditStore:
def __init__(self, database_path: Optional[str] = None):
self.conn = connect_state_db(database_path)
self.conn.create_function(
"audit_read_fingerprint", 7,
read_deduplication_fingerprint_values, deterministic=True,
)
ensure_audit_schema(self.conn)
def close(self) -> None:
@@ -299,10 +314,13 @@ class AuditStore:
totals = Counter()
counts = Counter()
facets = set()
previous_key = read_deduplication_key(last_event(self.conn) or {})
seen_read_fingerprints = set()
for event in events:
current_key = read_deduplication_key(event)
if current_key is not None and current_key == previous_key:
read_fingerprint = read_deduplication_fingerprint(event)
if (
read_fingerprint is not None
and read_fingerprint in seen_read_fingerprints
):
continue
occurred_second = parse_event_second(event["timestamp"])
user = str(event["user"])
@@ -311,7 +329,6 @@ class AuditStore:
action = str(event["action"])
result = str(event["result"])
success = int(bool(event["success"]))
day = occurred_second // 86400
event_rows.append(
(
str(event["timestamp"]),
@@ -327,17 +344,62 @@ class AuditStore:
success,
str(event["path"]),
str(event["source"]),
read_fingerprint,
)
)
totals[(day, account)] += 1
counts[(day, account, user, share, action, result, success)] += 1
for kind, value in (("user", user), ("share", share), ("action", action)):
if value:
facets.add((kind, day, account, value))
previous_key = current_key
if read_fingerprint is not None:
seen_read_fingerprints.add(read_fingerprint)
self.conn.execute("BEGIN IMMEDIATE")
try:
fingerprints = [
row[13] for row in event_rows if row[13] is not None
]
existing_fingerprints = set()
for offset in range(0, len(fingerprints), 500):
chunk = fingerprints[offset : offset + 500]
placeholders = ",".join("?" for _ in chunk)
existing_fingerprints.update(
bytes(row[0])
for row in self.conn.execute(
f"""
SELECT fingerprint
FROM audit_read_dedup
WHERE fingerprint IN ({placeholders})
""",
chunk,
)
)
if existing_fingerprints:
event_rows = [
row
for row in event_rows
if (
row[13] is None
or row[13] not in existing_fingerprints
)
]
for row in event_rows:
day = int(row[1]) // 86400
account = str(row[4])
user = str(row[3])
share = str(row[7])
action = str(row[8])
result = str(row[9])
success = int(row[10])
totals[(day, account)] += 1
counts[(
day, account, user, share, action, result, success
)] += 1
for kind, value in (
("user", user),
("share", share),
("action", action),
):
if value:
facets.add((kind, day, account, value))
if event_rows:
paths = sorted({str(row[11]) for row in event_rows})
self.conn.executemany(
@@ -383,6 +445,24 @@ class AuditStore:
""",
(len(indexed_rows),),
)
last_inserted_id = int(
self.conn.execute(
"SELECT max(id) FROM audit_events"
).fetchone()[0]
)
first_inserted_id = last_inserted_id - len(event_rows) + 1
self.conn.executemany(
"""
INSERT INTO audit_read_dedup (
fingerprint, occurred_second, event_id
) VALUES (?, ?, ?)
""",
(
(row[13], int(row[1]), first_inserted_id + offset)
for offset, row in enumerate(event_rows)
if row[13] is not None
),
)
self.conn.executemany(
ROLLUP_TOTAL_SQL,
((*key, count) for key, count in totals.items()),
@@ -393,6 +473,11 @@ class AuditStore:
)
self.conn.executemany(ROLLUP_FACET_SQL, facets)
self.conn.execute(
"DELETE FROM audit_read_dedup WHERE occurred_second < ?",
(read_deduplication_cutoff(),),
)
if source_updates:
self.conn.executemany(
"""
@@ -422,7 +507,7 @@ class AuditStore:
raise
return len(event_rows)
def backfill_rollups(self, limit: int = 50000) -> int:
def backfill_rollups(self, limit: int = 5000) -> int:
"""Backfill one bounded legacy-event chunk without delaying live appends."""
state = self.conn.execute(
"""
@@ -459,6 +544,55 @@ class AuditStore:
end_id = int(chunk[1])
self.conn.execute("BEGIN IMMEDIATE")
try:
dedup_cutoff = read_deduplication_cutoff()
self.conn.execute(
"DELETE FROM audit_read_dedup WHERE occurred_second < ?",
(dedup_cutoff,),
)
self.conn.execute(
"""
INSERT INTO audit_read_dedup (
fingerprint, occurred_second, event_id
)
SELECT audit_read_fingerprint(
e.occurred_at, e.user, e.client_ip, e.share, e.path,
e.success, e.result
), e.occurred_second, e.id
FROM audit_events AS e
WHERE e.id BETWEEN ? AND ?
AND e.action = 'read'
AND e.occurred_second >= ?
ORDER BY e.id
ON CONFLICT (fingerprint) DO UPDATE SET
event_id = coalesce(
audit_read_dedup.event_id, excluded.event_id
)
""",
(start_id, end_id, dedup_cutoff),
)
duplicate_read_sql = """
SELECT e.id
FROM audit_events AS e
JOIN audit_read_dedup AS d
ON d.fingerprint = audit_read_fingerprint(
e.occurred_at, e.user, e.client_ip, e.share, e.path,
e.success, e.result
)
WHERE e.id BETWEEN ? AND ?
AND e.action = 'read'
AND e.occurred_second >= ?
AND (d.event_id IS NULL OR d.event_id <> e.id)
"""
self.conn.execute(
f"DELETE FROM audit_path_events "
f"WHERE event_id IN ({duplicate_read_sql})",
(start_id, end_id, dedup_cutoff),
)
self.conn.execute(
f"DELETE FROM audit_events "
f"WHERE id IN ({duplicate_read_sql})",
(start_id, end_id, dedup_cutoff),
)
self.conn.execute(
"""
INSERT OR IGNORE INTO audit_paths (path)
app/web/app.js
+8 -2
View File
@@ -341,6 +341,7 @@ async function renderBackup() {
'<button id="start-backup" type="button">Sicherung jetzt starten</button>'
) + '<div id="backup-body"></div>';
const button = document.querySelector("#start-backup");
const body = document.querySelector("#backup-body");
let logLines = [];
let previousActive = false;
let polling = false;
@@ -349,9 +350,10 @@ async function renderBackup() {
const currentLog = document.querySelector("#backup-log");
const previousScrollTop = currentLog?.scrollTop || 0;
const data = await api(includeLog ? "/api/backup" : "/api/backup?log=0");
if (!body.isConnected || !button.isConnected) return null;
if (Array.isArray(data.log)) logLines = data.log;
data.log = logLines;
document.querySelector("#backup-body").innerHTML = backupMarkup(data);
body.innerHTML = backupMarkup(data);
button.disabled = !data.manualEnabled || processIsActive(data);
button.title = data.manualEnabled ? "" : "Es ist kein Sicherungsziel eingerichtet.";
const logView = document.querySelector("#backup-log");
@@ -394,6 +396,7 @@ async function renderBackup() {
}
});
const initial = await refresh(true);
if (!body.isConnected || !button.isConnected) return;
previousActive = processIsActive(initial);
state.timer = window.setInterval(() => { void poll(); }, 2000);
}
@@ -449,6 +452,7 @@ async function renderReconciliation() {
'<button id="start-reconciliation" type="button">Freigaben jetzt abgleichen</button>'
) + '<div id="reconciliation-body"></div>';
const button = document.querySelector("#start-reconciliation");
const body = document.querySelector("#reconciliation-body");
let logLines = [];
let previousActive = false;
let polling = false;
@@ -459,9 +463,10 @@ async function renderReconciliation() {
const data = await api(
includeLog ? "/api/reconciliation" : "/api/reconciliation?log=0"
);
if (!body.isConnected || !button.isConnected) return null;
if (Array.isArray(data.log)) logLines = data.log;
data.log = logLines;
document.querySelector("#reconciliation-body").innerHTML = reconciliationMarkup(data);
body.innerHTML = reconciliationMarkup(data);
button.disabled = processIsActive(data);
const logView = document.querySelector("#reconciliation-log");
if (logView) {
@@ -503,6 +508,7 @@ async function renderReconciliation() {
}
});
const initial = await refresh(true);
if (!body.isConnected || !button.isConnected) return;
previousActive = processIsActive(initial);
state.timer = window.setInterval(() => { void poll(); }, 1500);
}