higher level logging; sqlite db for logs
This commit is contained in:
Ludwig Lehnert
@@ -0,0 +1,369 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Indexed SQLite storage for normalized Samba audit events."""
|
||||
|
||||
import datetime as dt
|
||||
import glob
|
||||
import os
|
||||
import sqlite3
|
||||
from typing import Dict, Iterable, List, Optional, Set, Tuple
|
||||
|
||||
try:
|
||||
from .audit_policy import (
|
||||
account_name,
|
||||
read_deduplication_key,
|
||||
skipped_user_suffixes,
|
||||
)
|
||||
from .state_db import connect_state_db
|
||||
except ImportError:
|
||||
from audit_policy import account_name, read_deduplication_key, skipped_user_suffixes
|
||||
from state_db import connect_state_db
|
||||
|
||||
|
||||
AUDIT_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS audit_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
occurred_at TEXT NOT NULL,
|
||||
occurred_second INTEGER NOT NULL,
|
||||
ingested_at TEXT NOT NULL,
|
||||
user TEXT NOT NULL,
|
||||
account TEXT NOT NULL,
|
||||
client_ip TEXT NOT NULL,
|
||||
client TEXT NOT NULL,
|
||||
share TEXT NOT NULL,
|
||||
action TEXT NOT NULL CHECK (action IN ('read', 'write', 'move', 'delete')),
|
||||
result TEXT NOT NULL,
|
||||
success INTEGER NOT NULL CHECK (success IN (0, 1)),
|
||||
path TEXT NOT NULL,
|
||||
source TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS audit_sources (
|
||||
path TEXT PRIMARY KEY,
|
||||
inode INTEGER NOT NULL,
|
||||
offset INTEGER NOT NULL
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX IF NOT EXISTS audit_events_time
|
||||
ON audit_events (occurred_second DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS audit_events_action_time
|
||||
ON audit_events (action, occurred_second DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS audit_events_success_time
|
||||
ON audit_events (success, occurred_second DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS audit_events_user_time
|
||||
ON audit_events (user COLLATE NOCASE, occurred_second DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS audit_events_account_time
|
||||
ON audit_events (account, occurred_second DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS audit_events_share_time
|
||||
ON audit_events (share COLLATE NOCASE, occurred_second DESC, id DESC);
|
||||
CREATE INDEX IF NOT EXISTS audit_events_result_time
|
||||
ON audit_events (result COLLATE NOCASE, occurred_second DESC, id DESC);
|
||||
"""
|
||||
|
||||
|
||||
def ensure_audit_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.executescript(AUDIT_SCHEMA)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def parse_event_second(timestamp: object) -> int:
|
||||
parsed = dt.datetime.fromisoformat(str(timestamp).replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=dt.timezone.utc)
|
||||
return int(parsed.astimezone(dt.timezone.utc).timestamp())
|
||||
|
||||
|
||||
def row_to_event(row: sqlite3.Row) -> Dict[str, object]:
|
||||
action = str(row["action"])
|
||||
return {
|
||||
"id": int(row["id"]),
|
||||
"timestamp": str(row["occurred_at"]),
|
||||
"ingestedAt": str(row["ingested_at"]),
|
||||
"user": str(row["user"]),
|
||||
"clientIp": str(row["client_ip"]),
|
||||
"client": str(row["client"]),
|
||||
"share": str(row["share"]),
|
||||
"operation": action,
|
||||
"action": action,
|
||||
"result": str(row["result"]),
|
||||
"success": bool(row["success"]),
|
||||
"path": str(row["path"]),
|
||||
"source": str(row["source"]),
|
||||
}
|
||||
|
||||
|
||||
def last_event(conn: sqlite3.Connection) -> Optional[Dict[str, object]]:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, occurred_at, ingested_at, user, client_ip, client, share,
|
||||
action, result, success, path, source
|
||||
FROM audit_events
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
return row_to_event(row) if row is not None else None
|
||||
|
||||
|
||||
class AuditStore:
|
||||
def __init__(self, database_path: Optional[str] = None):
|
||||
self.conn = connect_state_db(database_path)
|
||||
ensure_audit_schema(self.conn)
|
||||
|
||||
def close(self) -> None:
|
||||
self.conn.close()
|
||||
|
||||
def source_entries(self) -> Dict[str, Dict[str, object]]:
|
||||
return {
|
||||
str(row["path"]): {
|
||||
"inode": int(row["inode"]),
|
||||
"offset": int(row["offset"]),
|
||||
}
|
||||
for row in self.conn.execute(
|
||||
"SELECT path, inode, offset FROM audit_sources"
|
||||
)
|
||||
}
|
||||
|
||||
def append_batch(
|
||||
self,
|
||||
events: Iterable[Dict[str, object]],
|
||||
source_updates: Dict[str, Dict[str, object]],
|
||||
seen_paths: Set[str],
|
||||
) -> int:
|
||||
inserted = 0
|
||||
self.conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
previous_key = read_deduplication_key(last_event(self.conn) or {})
|
||||
for event in events:
|
||||
current_key = read_deduplication_key(event)
|
||||
if current_key is not None and current_key == previous_key:
|
||||
continue
|
||||
self.conn.execute(
|
||||
"""
|
||||
INSERT INTO audit_events (
|
||||
occurred_at, occurred_second, ingested_at, user, account,
|
||||
client_ip, client, share, action, result, success, path,
|
||||
source
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
str(event["timestamp"]),
|
||||
parse_event_second(event["timestamp"]),
|
||||
str(event["ingestedAt"]),
|
||||
str(event["user"]),
|
||||
account_name(str(event["user"])).casefold(),
|
||||
str(event["clientIp"]),
|
||||
str(event["client"]),
|
||||
str(event["share"]),
|
||||
str(event["action"]),
|
||||
str(event["result"]),
|
||||
int(bool(event["success"])),
|
||||
str(event["path"]),
|
||||
str(event["source"]),
|
||||
),
|
||||
)
|
||||
inserted += 1
|
||||
previous_key = current_key
|
||||
|
||||
for path, entry in source_updates.items():
|
||||
self.conn.execute(
|
||||
"""
|
||||
INSERT INTO audit_sources (path, inode, offset)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
inode = excluded.inode,
|
||||
offset = excluded.offset
|
||||
""",
|
||||
(path, int(entry["inode"]), int(entry["offset"])),
|
||||
)
|
||||
for row in self.conn.execute("SELECT path FROM audit_sources").fetchall():
|
||||
path = str(row["path"])
|
||||
if path not in seen_paths:
|
||||
self.conn.execute("DELETE FROM audit_sources WHERE path = ?", (path,))
|
||||
self.conn.commit()
|
||||
except Exception:
|
||||
self.conn.rollback()
|
||||
raise
|
||||
return inserted
|
||||
|
||||
|
||||
def escape_like(value: str) -> str:
|
||||
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
|
||||
def date_seconds(day: dt.date) -> int:
|
||||
return int(
|
||||
dt.datetime.combine(day, dt.time.min, tzinfo=dt.timezone.utc).timestamp()
|
||||
)
|
||||
|
||||
|
||||
def activity_conditions(
|
||||
start: dt.date,
|
||||
end: dt.date,
|
||||
params: Dict[str, List[str]],
|
||||
*,
|
||||
include_filters: bool,
|
||||
) -> Tuple[List[str], List[object]]:
|
||||
conditions = ["occurred_second >= ?", "occurred_second < ?"]
|
||||
values: List[object] = [
|
||||
date_seconds(start),
|
||||
date_seconds(end + dt.timedelta(days=1)),
|
||||
]
|
||||
for suffix in skipped_user_suffixes():
|
||||
conditions.append("lower(account) NOT LIKE ? ESCAPE '\\'")
|
||||
values.append(f"%{escape_like(suffix)}")
|
||||
|
||||
if not include_filters:
|
||||
return conditions, values
|
||||
|
||||
filters = {
|
||||
key: params.get(key, [""])[0].casefold().strip()
|
||||
for key in ("user", "share", "operation", "action", "path", "result")
|
||||
}
|
||||
user = filters["user"]
|
||||
if user:
|
||||
if "\\" in user or "@" in user:
|
||||
conditions.append("user = ? COLLATE NOCASE")
|
||||
else:
|
||||
conditions.append("account = ?")
|
||||
values.append(user)
|
||||
share = filters["share"]
|
||||
if share:
|
||||
conditions.append("share = ? COLLATE NOCASE")
|
||||
values.append(share)
|
||||
path = filters["path"]
|
||||
if path:
|
||||
conditions.append("lower(path) LIKE ? ESCAPE '\\'")
|
||||
values.append(f"%{escape_like(path)}%")
|
||||
action = filters["action"] or filters["operation"]
|
||||
if action:
|
||||
conditions.append("action = ?")
|
||||
values.append(action)
|
||||
result = filters["result"]
|
||||
if result == "fail":
|
||||
conditions.append("success = 0")
|
||||
elif result:
|
||||
conditions.append("result = ? COLLATE NOCASE")
|
||||
values.append(result)
|
||||
return conditions, values
|
||||
|
||||
|
||||
def query_activity(
|
||||
conn: sqlite3.Connection,
|
||||
start: dt.date,
|
||||
end: dt.date,
|
||||
params: Dict[str, List[str]],
|
||||
) -> Dict[str, object]:
|
||||
limit = min(500, max(1, int(params.get("limit", ["100"])[0])))
|
||||
conditions, values = activity_conditions(start, end, params, include_filters=True)
|
||||
where_sql = " AND ".join(conditions)
|
||||
matched = int(
|
||||
conn.execute(
|
||||
f"SELECT count(*) FROM audit_events WHERE {where_sql}",
|
||||
values,
|
||||
).fetchone()[0]
|
||||
)
|
||||
|
||||
cursor = params.get("cursor", [""])[0].strip()
|
||||
page_conditions = list(conditions)
|
||||
page_values = list(values)
|
||||
if cursor:
|
||||
try:
|
||||
cursor_second, cursor_id = (int(part) for part in cursor.split(":", 1))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError("Ungültiger Seitenzeiger") from exc
|
||||
page_conditions.append(
|
||||
"(occurred_second < ? OR (occurred_second = ? AND id < ?))"
|
||||
)
|
||||
page_values.extend((cursor_second, cursor_second, cursor_id))
|
||||
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT id, occurred_at, occurred_second, ingested_at, user, client_ip,
|
||||
client, share, action, result, success, path, source
|
||||
FROM audit_events
|
||||
WHERE {" AND ".join(page_conditions)}
|
||||
ORDER BY occurred_second DESC, id DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(*page_values, limit + 1),
|
||||
).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
events = [row_to_event(row) for row in rows]
|
||||
next_cursor = None
|
||||
if has_more and rows:
|
||||
next_cursor = f"{int(rows[-1]['occurred_second'])}:{int(rows[-1]['id'])}"
|
||||
|
||||
facet_conditions, facet_values = activity_conditions(
|
||||
start, end, params, include_filters=False
|
||||
)
|
||||
facet_where = " AND ".join(facet_conditions)
|
||||
|
||||
def distinct(column: str) -> List[str]:
|
||||
return [
|
||||
str(row[0])
|
||||
for row in conn.execute(
|
||||
f"""
|
||||
SELECT DISTINCT {column}
|
||||
FROM audit_events
|
||||
WHERE {facet_where} AND {column} <> ''
|
||||
ORDER BY {column} COLLATE NOCASE
|
||||
""",
|
||||
facet_values,
|
||||
)
|
||||
]
|
||||
|
||||
actions = distinct("action")
|
||||
return {
|
||||
"events": events,
|
||||
"nextCursor": next_cursor,
|
||||
"matched": matched,
|
||||
"facets": {
|
||||
"users": distinct("user"),
|
||||
"shares": distinct("share"),
|
||||
"operations": actions,
|
||||
"actions": actions,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def audit_summary(conn: sqlite3.Connection, database_path: str) -> Dict[str, object]:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT count(DISTINCT substr(occurred_at, 1, 10)),
|
||||
min(substr(occurred_at, 1, 10)),
|
||||
max(substr(occurred_at, 1, 10))
|
||||
FROM audit_events
|
||||
"""
|
||||
).fetchone()
|
||||
database_bytes = 0
|
||||
for suffix in ("", "-wal"):
|
||||
try:
|
||||
database_bytes += os.path.getsize(f"{database_path}{suffix}")
|
||||
except OSError:
|
||||
pass
|
||||
return {
|
||||
"days": int(row[0] or 0),
|
||||
"bytes": database_bytes,
|
||||
"oldest": row[1],
|
||||
"newest": row[2],
|
||||
}
|
||||
|
||||
|
||||
def drop_legacy_audit_files(directory: str) -> int:
|
||||
removed = 0
|
||||
patterns = (
|
||||
"????-??-??.jsonl",
|
||||
"????-??-??.jsonl.gz",
|
||||
"collector-state.json",
|
||||
"collector-state.json.tmp",
|
||||
)
|
||||
for pattern in patterns:
|
||||
for path in glob.glob(os.path.join(directory, pattern)):
|
||||
if not os.path.isfile(path):
|
||||
continue
|
||||
os.remove(path)
|
||||
removed += 1
|
||||
try:
|
||||
os.rmdir(directory)
|
||||
except OSError:
|
||||
pass
|
||||
return removed
|
||||
Reference in New Issue
Block a user