370 lines
12 KiB
Python
370 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Indexed SQLite storage for normalized Samba audit events."""
|
|
|
|
import datetime as dt
|
|
import glob
|
|
import os
|
|
import sqlite3
|
|
from typing import Dict, Iterable, List, Optional, Set, Tuple
|
|
|
|
try:
|
|
from .audit_policy import (
|
|
account_name,
|
|
read_deduplication_key,
|
|
skipped_user_suffixes,
|
|
)
|
|
from .state_db import connect_state_db
|
|
except ImportError:
|
|
from audit_policy import account_name, read_deduplication_key, skipped_user_suffixes
|
|
from state_db import connect_state_db
|
|
|
|
|
|
AUDIT_SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS audit_events (
|
|
id INTEGER PRIMARY KEY,
|
|
occurred_at TEXT NOT NULL,
|
|
occurred_second INTEGER NOT NULL,
|
|
ingested_at TEXT NOT NULL,
|
|
user TEXT NOT NULL,
|
|
account TEXT NOT NULL,
|
|
client_ip TEXT NOT NULL,
|
|
client TEXT NOT NULL,
|
|
share TEXT NOT NULL,
|
|
action TEXT NOT NULL CHECK (action IN ('read', 'write', 'move', 'delete')),
|
|
result TEXT NOT NULL,
|
|
success INTEGER NOT NULL CHECK (success IN (0, 1)),
|
|
path TEXT NOT NULL,
|
|
source TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS audit_sources (
|
|
path TEXT PRIMARY KEY,
|
|
inode INTEGER NOT NULL,
|
|
offset INTEGER NOT NULL
|
|
) WITHOUT ROWID;
|
|
CREATE INDEX IF NOT EXISTS audit_events_time
|
|
ON audit_events (occurred_second DESC, id DESC);
|
|
CREATE INDEX IF NOT EXISTS audit_events_action_time
|
|
ON audit_events (action, occurred_second DESC, id DESC);
|
|
CREATE INDEX IF NOT EXISTS audit_events_success_time
|
|
ON audit_events (success, occurred_second DESC, id DESC);
|
|
CREATE INDEX IF NOT EXISTS audit_events_user_time
|
|
ON audit_events (user COLLATE NOCASE, occurred_second DESC, id DESC);
|
|
CREATE INDEX IF NOT EXISTS audit_events_account_time
|
|
ON audit_events (account, occurred_second DESC, id DESC);
|
|
CREATE INDEX IF NOT EXISTS audit_events_share_time
|
|
ON audit_events (share COLLATE NOCASE, occurred_second DESC, id DESC);
|
|
CREATE INDEX IF NOT EXISTS audit_events_result_time
|
|
ON audit_events (result COLLATE NOCASE, occurred_second DESC, id DESC);
|
|
"""
|
|
|
|
|
|
def ensure_audit_schema(conn: sqlite3.Connection) -> None:
|
|
conn.executescript(AUDIT_SCHEMA)
|
|
conn.commit()
|
|
|
|
|
|
def parse_event_second(timestamp: object) -> int:
|
|
parsed = dt.datetime.fromisoformat(str(timestamp).replace("Z", "+00:00"))
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=dt.timezone.utc)
|
|
return int(parsed.astimezone(dt.timezone.utc).timestamp())
|
|
|
|
|
|
def row_to_event(row: sqlite3.Row) -> Dict[str, object]:
|
|
action = str(row["action"])
|
|
return {
|
|
"id": int(row["id"]),
|
|
"timestamp": str(row["occurred_at"]),
|
|
"ingestedAt": str(row["ingested_at"]),
|
|
"user": str(row["user"]),
|
|
"clientIp": str(row["client_ip"]),
|
|
"client": str(row["client"]),
|
|
"share": str(row["share"]),
|
|
"operation": action,
|
|
"action": action,
|
|
"result": str(row["result"]),
|
|
"success": bool(row["success"]),
|
|
"path": str(row["path"]),
|
|
"source": str(row["source"]),
|
|
}
|
|
|
|
|
|
def last_event(conn: sqlite3.Connection) -> Optional[Dict[str, object]]:
|
|
row = conn.execute(
|
|
"""
|
|
SELECT id, occurred_at, ingested_at, user, client_ip, client, share,
|
|
action, result, success, path, source
|
|
FROM audit_events
|
|
ORDER BY id DESC
|
|
LIMIT 1
|
|
"""
|
|
).fetchone()
|
|
return row_to_event(row) if row is not None else None
|
|
|
|
|
|
class AuditStore:
|
|
def __init__(self, database_path: Optional[str] = None):
|
|
self.conn = connect_state_db(database_path)
|
|
ensure_audit_schema(self.conn)
|
|
|
|
def close(self) -> None:
|
|
self.conn.close()
|
|
|
|
def source_entries(self) -> Dict[str, Dict[str, object]]:
|
|
return {
|
|
str(row["path"]): {
|
|
"inode": int(row["inode"]),
|
|
"offset": int(row["offset"]),
|
|
}
|
|
for row in self.conn.execute(
|
|
"SELECT path, inode, offset FROM audit_sources"
|
|
)
|
|
}
|
|
|
|
def append_batch(
|
|
self,
|
|
events: Iterable[Dict[str, object]],
|
|
source_updates: Dict[str, Dict[str, object]],
|
|
seen_paths: Set[str],
|
|
) -> int:
|
|
inserted = 0
|
|
self.conn.execute("BEGIN IMMEDIATE")
|
|
try:
|
|
previous_key = read_deduplication_key(last_event(self.conn) or {})
|
|
for event in events:
|
|
current_key = read_deduplication_key(event)
|
|
if current_key is not None and current_key == previous_key:
|
|
continue
|
|
self.conn.execute(
|
|
"""
|
|
INSERT INTO audit_events (
|
|
occurred_at, occurred_second, ingested_at, user, account,
|
|
client_ip, client, share, action, result, success, path,
|
|
source
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
str(event["timestamp"]),
|
|
parse_event_second(event["timestamp"]),
|
|
str(event["ingestedAt"]),
|
|
str(event["user"]),
|
|
account_name(str(event["user"])).casefold(),
|
|
str(event["clientIp"]),
|
|
str(event["client"]),
|
|
str(event["share"]),
|
|
str(event["action"]),
|
|
str(event["result"]),
|
|
int(bool(event["success"])),
|
|
str(event["path"]),
|
|
str(event["source"]),
|
|
),
|
|
)
|
|
inserted += 1
|
|
previous_key = current_key
|
|
|
|
for path, entry in source_updates.items():
|
|
self.conn.execute(
|
|
"""
|
|
INSERT INTO audit_sources (path, inode, offset)
|
|
VALUES (?, ?, ?)
|
|
ON CONFLICT(path) DO UPDATE SET
|
|
inode = excluded.inode,
|
|
offset = excluded.offset
|
|
""",
|
|
(path, int(entry["inode"]), int(entry["offset"])),
|
|
)
|
|
for row in self.conn.execute("SELECT path FROM audit_sources").fetchall():
|
|
path = str(row["path"])
|
|
if path not in seen_paths:
|
|
self.conn.execute("DELETE FROM audit_sources WHERE path = ?", (path,))
|
|
self.conn.commit()
|
|
except Exception:
|
|
self.conn.rollback()
|
|
raise
|
|
return inserted
|
|
|
|
|
|
def escape_like(value: str) -> str:
|
|
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
|
|
|
|
def date_seconds(day: dt.date) -> int:
|
|
return int(
|
|
dt.datetime.combine(day, dt.time.min, tzinfo=dt.timezone.utc).timestamp()
|
|
)
|
|
|
|
|
|
def activity_conditions(
|
|
start: dt.date,
|
|
end: dt.date,
|
|
params: Dict[str, List[str]],
|
|
*,
|
|
include_filters: bool,
|
|
) -> Tuple[List[str], List[object]]:
|
|
conditions = ["occurred_second >= ?", "occurred_second < ?"]
|
|
values: List[object] = [
|
|
date_seconds(start),
|
|
date_seconds(end + dt.timedelta(days=1)),
|
|
]
|
|
for suffix in skipped_user_suffixes():
|
|
conditions.append("lower(account) NOT LIKE ? ESCAPE '\\'")
|
|
values.append(f"%{escape_like(suffix)}")
|
|
|
|
if not include_filters:
|
|
return conditions, values
|
|
|
|
filters = {
|
|
key: params.get(key, [""])[0].casefold().strip()
|
|
for key in ("user", "share", "operation", "action", "path", "result")
|
|
}
|
|
user = filters["user"]
|
|
if user:
|
|
if "\\" in user or "@" in user:
|
|
conditions.append("user = ? COLLATE NOCASE")
|
|
else:
|
|
conditions.append("account = ?")
|
|
values.append(user)
|
|
share = filters["share"]
|
|
if share:
|
|
conditions.append("share = ? COLLATE NOCASE")
|
|
values.append(share)
|
|
path = filters["path"]
|
|
if path:
|
|
conditions.append("lower(path) LIKE ? ESCAPE '\\'")
|
|
values.append(f"%{escape_like(path)}%")
|
|
action = filters["action"] or filters["operation"]
|
|
if action:
|
|
conditions.append("action = ?")
|
|
values.append(action)
|
|
result = filters["result"]
|
|
if result == "fail":
|
|
conditions.append("success = 0")
|
|
elif result:
|
|
conditions.append("result = ? COLLATE NOCASE")
|
|
values.append(result)
|
|
return conditions, values
|
|
|
|
|
|
def query_activity(
|
|
conn: sqlite3.Connection,
|
|
start: dt.date,
|
|
end: dt.date,
|
|
params: Dict[str, List[str]],
|
|
) -> Dict[str, object]:
|
|
limit = min(500, max(1, int(params.get("limit", ["100"])[0])))
|
|
conditions, values = activity_conditions(start, end, params, include_filters=True)
|
|
where_sql = " AND ".join(conditions)
|
|
matched = int(
|
|
conn.execute(
|
|
f"SELECT count(*) FROM audit_events WHERE {where_sql}",
|
|
values,
|
|
).fetchone()[0]
|
|
)
|
|
|
|
cursor = params.get("cursor", [""])[0].strip()
|
|
page_conditions = list(conditions)
|
|
page_values = list(values)
|
|
if cursor:
|
|
try:
|
|
cursor_second, cursor_id = (int(part) for part in cursor.split(":", 1))
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("Ungültiger Seitenzeiger") from exc
|
|
page_conditions.append(
|
|
"(occurred_second < ? OR (occurred_second = ? AND id < ?))"
|
|
)
|
|
page_values.extend((cursor_second, cursor_second, cursor_id))
|
|
|
|
rows = conn.execute(
|
|
f"""
|
|
SELECT id, occurred_at, occurred_second, ingested_at, user, client_ip,
|
|
client, share, action, result, success, path, source
|
|
FROM audit_events
|
|
WHERE {" AND ".join(page_conditions)}
|
|
ORDER BY occurred_second DESC, id DESC
|
|
LIMIT ?
|
|
""",
|
|
(*page_values, limit + 1),
|
|
).fetchall()
|
|
has_more = len(rows) > limit
|
|
rows = rows[:limit]
|
|
events = [row_to_event(row) for row in rows]
|
|
next_cursor = None
|
|
if has_more and rows:
|
|
next_cursor = f"{int(rows[-1]['occurred_second'])}:{int(rows[-1]['id'])}"
|
|
|
|
facet_conditions, facet_values = activity_conditions(
|
|
start, end, params, include_filters=False
|
|
)
|
|
facet_where = " AND ".join(facet_conditions)
|
|
|
|
def distinct(column: str) -> List[str]:
|
|
return [
|
|
str(row[0])
|
|
for row in conn.execute(
|
|
f"""
|
|
SELECT DISTINCT {column}
|
|
FROM audit_events
|
|
WHERE {facet_where} AND {column} <> ''
|
|
ORDER BY {column} COLLATE NOCASE
|
|
""",
|
|
facet_values,
|
|
)
|
|
]
|
|
|
|
actions = distinct("action")
|
|
return {
|
|
"events": events,
|
|
"nextCursor": next_cursor,
|
|
"matched": matched,
|
|
"facets": {
|
|
"users": distinct("user"),
|
|
"shares": distinct("share"),
|
|
"operations": actions,
|
|
"actions": actions,
|
|
},
|
|
}
|
|
|
|
|
|
def audit_summary(conn: sqlite3.Connection, database_path: str) -> Dict[str, object]:
|
|
row = conn.execute(
|
|
"""
|
|
SELECT count(DISTINCT substr(occurred_at, 1, 10)),
|
|
min(substr(occurred_at, 1, 10)),
|
|
max(substr(occurred_at, 1, 10))
|
|
FROM audit_events
|
|
"""
|
|
).fetchone()
|
|
database_bytes = 0
|
|
for suffix in ("", "-wal"):
|
|
try:
|
|
database_bytes += os.path.getsize(f"{database_path}{suffix}")
|
|
except OSError:
|
|
pass
|
|
return {
|
|
"days": int(row[0] or 0),
|
|
"bytes": database_bytes,
|
|
"oldest": row[1],
|
|
"newest": row[2],
|
|
}
|
|
|
|
|
|
def drop_legacy_audit_files(directory: str) -> int:
|
|
removed = 0
|
|
patterns = (
|
|
"????-??-??.jsonl",
|
|
"????-??-??.jsonl.gz",
|
|
"collector-state.json",
|
|
"collector-state.json.tmp",
|
|
)
|
|
for pattern in patterns:
|
|
for path in glob.glob(os.path.join(directory, pattern)):
|
|
if not os.path.isfile(path):
|
|
continue
|
|
os.remove(path)
|
|
removed += 1
|
|
try:
|
|
os.rmdir(directory)
|
|
except OSError:
|
|
pass
|
|
return removed
|