413 lines
19 KiB
Python
Executable File
413 lines
19 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""End-to-end checks for the disposable preview domain and file server."""
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
import socket
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from dataclasses import dataclass
|
|
from typing import Callable, Dict, Optional
|
|
|
|
|
|
ENGINE = os.environ["PREVIEW_ENGINE"]
|
|
FILES_CONTAINER = os.environ["PREVIEW_FILES_CONTAINER"]
|
|
CLIENT_CONTAINER = os.environ["PREVIEW_CLIENT_CONTAINER"]
|
|
BACKUP_CONTAINER = os.environ["PREVIEW_BACKUP_CONTAINER"]
|
|
CA_ROOT = os.environ["PREVIEW_CA_ROOT"]
|
|
HTTPS_PORT = int(os.environ["PREVIEW_HTTPS_PORT"])
|
|
REALM = os.environ["PREVIEW_REALM"]
|
|
WORKGROUP = os.environ["PREVIEW_WORKGROUP"]
|
|
DNS_DOMAIN = os.environ["PREVIEW_DNS_DOMAIN"]
|
|
DOMAIN_SID = os.environ["PREVIEW_DOMAIN_SID"]
|
|
ADMIN_USER = os.environ["PREVIEW_ADMIN_USER"]
|
|
ADMIN_PASSWORD = os.environ["PREVIEW_ADMIN_PASSWORD"]
|
|
USER_PASSWORD = os.environ["PREVIEW_USER_PASSWORD"]
|
|
HTTPS_HOST = os.environ["PREVIEW_HTTPS_HOST"]
|
|
BASE_URL = f"https://{HTTPS_HOST}:{HTTPS_PORT}"
|
|
_ORIGINAL_GETADDRINFO = socket.getaddrinfo
|
|
|
|
|
|
def preview_getaddrinfo(host, port, *args, **kwargs):
|
|
if host == HTTPS_HOST:
|
|
host = "127.0.0.1"
|
|
return _ORIGINAL_GETADDRINFO(host, port, *args, **kwargs)
|
|
|
|
|
|
socket.getaddrinfo = preview_getaddrinfo
|
|
TLS_CONTEXT = ssl.create_default_context(cafile=CA_ROOT)
|
|
HTTP_OPENER = urllib.request.build_opener(
|
|
urllib.request.ProxyHandler({}),
|
|
urllib.request.HTTPSHandler(context=TLS_CONTEXT),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class Response:
|
|
status: int
|
|
headers: object
|
|
body: bytes
|
|
|
|
def json(self):
|
|
return json.loads(self.body.decode("utf-8"))
|
|
|
|
|
|
def fail(message: str) -> None:
|
|
raise AssertionError(message)
|
|
|
|
|
|
def check(condition: bool, message: str) -> None:
|
|
if not condition:
|
|
fail(message)
|
|
|
|
|
|
def announce(message: str) -> None:
|
|
print(f"[e2e] {message}", flush=True)
|
|
|
|
|
|
def engine_run(*args: str, check_result: bool = True) -> subprocess.CompletedProcess:
|
|
result = subprocess.run(
|
|
[ENGINE, *args], capture_output=True, text=True, check=False
|
|
)
|
|
if check_result and result.returncode != 0:
|
|
output = result.stderr.strip() or result.stdout.strip()
|
|
fail(f"container command failed ({' '.join(args)}): {output}")
|
|
return result
|
|
|
|
|
|
def http(
|
|
path: str,
|
|
*,
|
|
method: str = "GET",
|
|
value: Optional[Dict[str, object]] = None,
|
|
token: str = "",
|
|
) -> Response:
|
|
body = None
|
|
headers = {"Accept": "application/json"}
|
|
if value is not None:
|
|
body = json.dumps(value).encode("utf-8")
|
|
headers["Content-Type"] = "application/json"
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
request = urllib.request.Request(
|
|
f"{BASE_URL}{path}", data=body, headers=headers, method=method
|
|
)
|
|
try:
|
|
with HTTP_OPENER.open(request, timeout=30) as response:
|
|
return Response(response.status, response.headers, response.read())
|
|
except urllib.error.HTTPError as exc:
|
|
return Response(exc.code, exc.headers, exc.read())
|
|
|
|
|
|
def eventually(
|
|
description: str,
|
|
callback: Callable[[], object],
|
|
predicate: Callable[[object], bool],
|
|
timeout: float = 90,
|
|
interval: float = 1,
|
|
):
|
|
deadline = time.monotonic() + timeout
|
|
last_value = None
|
|
last_error: Optional[Exception] = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
last_value = callback()
|
|
if predicate(last_value):
|
|
return last_value
|
|
except Exception as exc: # pylint: disable=broad-except
|
|
last_error = exc
|
|
time.sleep(interval)
|
|
detail = f"; last value={last_value!r}"
|
|
if last_error is not None:
|
|
detail += f"; last error={last_error}"
|
|
fail(f"timed out waiting for {description}{detail}")
|
|
|
|
|
|
def decode_jwt_payload(token: str) -> Dict[str, object]:
|
|
parts = token.split(".")
|
|
check(len(parts) == 3, "login did not return a compact JWT")
|
|
padding = "=" * (-len(parts[1]) % 4)
|
|
return json.loads(base64.urlsafe_b64decode(parts[1] + padding))
|
|
|
|
|
|
def flatten_members(nodes):
|
|
values = []
|
|
for node in nodes:
|
|
values.append((node.get("type"), node.get("sam"), node.get("name")))
|
|
values.extend(flatten_members(node.get("members", [])))
|
|
return values
|
|
|
|
|
|
def query_path(path: str, params: Dict[str, str]) -> str:
|
|
return f"{path}?{urllib.parse.urlencode(params)}"
|
|
|
|
|
|
def main() -> int:
|
|
announce("TLS chain, hostname, public health, and browser security headers")
|
|
health = http("/healthz")
|
|
check(health.status == 200 and health.json() == {"status": "ok"}, "health check failed")
|
|
index = http("/")
|
|
check(index.status == 200 and b'Dateiserver-Verwaltung' in index.body, "German web shell was not served")
|
|
check(b'<html lang="de">' in index.body, "web shell language is not German")
|
|
styles = http("/assets/styles.css")
|
|
check(b"[hidden]" in styles.body and b"display: none !important" in styles.body, "hidden views can be made visible by author CSS")
|
|
script = http("/assets/app.js")
|
|
check(b"getUTCHours()" in script.body and b" UTC`" in script.body, "UI does not format timestamps explicitly in UTC")
|
|
check(b"localTime" not in script.body and b"eyebrow" not in script.body, "obsolete local-time or decorative UI code remains")
|
|
check(b"brand-mark" not in index.body, "decorative brand mark remains")
|
|
check("max-age=" in index.headers.get("Strict-Transport-Security", ""), "HSTS missing")
|
|
check("default-src 'self'" in index.headers.get("Content-Security-Policy", ""), "CSP missing")
|
|
with socket.create_connection(("localhost", HTTPS_PORT), timeout=10) as raw:
|
|
with TLS_CONTEXT.wrap_socket(raw, server_hostname=HTTPS_HOST) as secured:
|
|
certificate = secured.getpeercert()
|
|
sans = {value for kind, value in certificate.get("subjectAltName", ()) if kind == "DNS"}
|
|
check(HTTPS_HOST in sans, f"issued certificate does not cover {HTTPS_HOST}")
|
|
check(certificate.get("issuer") != certificate.get("subject"), "web certificate is self-signed instead of CA-issued")
|
|
|
|
announce("JWT boundary, real Kerberos credentials, and Domain Admin authorization")
|
|
unauthenticated = http("/api/session")
|
|
check(unauthenticated.status == 401, "protected API accepted an anonymous request")
|
|
non_admin = http(
|
|
"/api/login",
|
|
method="POST",
|
|
value={"username": "alice", "password": USER_PASSWORD},
|
|
)
|
|
check(non_admin.status == 401, "valid non-admin domain user was allowed into the UI")
|
|
wrong_password = http(
|
|
"/api/login",
|
|
method="POST",
|
|
value={"username": ADMIN_USER, "password": "wrong-password"},
|
|
)
|
|
check(wrong_password.status == 401, "invalid admin password was accepted")
|
|
login = http(
|
|
"/api/login",
|
|
method="POST",
|
|
value={"username": ADMIN_USER, "password": ADMIN_PASSWORD},
|
|
)
|
|
check(login.status == 200, f"Domain Admin login failed: {login.body!r}")
|
|
login_payload = login.json()
|
|
token = str(login_payload.get("token", ""))
|
|
claims = decode_jwt_payload(token)
|
|
check(claims.get("iss") == "ad-file-server-web", "JWT issuer is wrong")
|
|
check(claims.get("aud") == "domain-admins", "JWT audience is wrong")
|
|
check(claims.get("role") == "domain-admin", "JWT role is wrong")
|
|
check(claims.get("sub") == f"{WORKGROUP}\\{ADMIN_USER}", "JWT subject is wrong")
|
|
cookie = login.headers.get("Set-Cookie", "")
|
|
for attribute in ("HttpOnly", "Secure", "SameSite=Strict"):
|
|
check(attribute in cookie, f"session cookie is missing {attribute}")
|
|
session = http("/api/session", token=token)
|
|
check(session.status == 200 and session.json().get("user") == claims["sub"], "Bearer JWT was not accepted")
|
|
check(http("/api/session", token=f"{token}corrupt").status == 401, "corrupt JWT was accepted")
|
|
readonly = http("/api/groups", method="POST", value={}, token=token)
|
|
check(readonly.status == 404, "a mutation-like API method was accepted")
|
|
|
|
announce("AD trust, nested group tree, folders, and domain membership")
|
|
engine_run("exec", FILES_CONTAINER, "wbinfo", "-t")
|
|
admin_identity = engine_run(
|
|
"exec", FILES_CONTAINER, "wbinfo", "--name-to-sid", f"{WORKGROUP}\\{ADMIN_USER}"
|
|
)
|
|
admin_sid = admin_identity.stdout.split()[0]
|
|
admin_sids = engine_run(
|
|
"exec", FILES_CONTAINER, "wbinfo", "--user-sids", admin_sid
|
|
)
|
|
check(f"{DOMAIN_SID}-512" in admin_sids.stdout, "preview admin SID set lacks Domain Admins")
|
|
groups_response = http("/api/groups", token=token)
|
|
check(groups_response.status == 200, f"group endpoint failed: {groups_response.body!r}")
|
|
groups_payload = groups_response.json()
|
|
groups = {row["name"]: row for row in groups_payload.get("groups", [])}
|
|
check({"Finance", "Engineering", "Projects"}.issubset(groups), f"seed groups missing: {sorted(groups)}")
|
|
check(groups["Finance"]["folder"] == "Finance", "Finance group/folder mapping is wrong")
|
|
finance_nodes = flatten_members(groups["Finance"].get("members", []))
|
|
check(any(kind == "group" and sam == "Finance_Analysts" for kind, sam, _ in finance_nodes), "nested Finance Analysts group missing")
|
|
check({"alice", "bob"}.issubset({sam for kind, sam, _ in finance_nodes if kind == "user"}), "Finance users missing")
|
|
project_nodes = flatten_members(groups["Projects"].get("members", []))
|
|
check({"alice", "bob", "carol", "dave", "eve", "frank"}.issubset({sam for kind, sam, _ in project_nodes if kind == "user"}), "transitive or primary-group Project membership is incomplete")
|
|
check(any(kind == "group" and sam == "Domain Users" for kind, sam, _ in project_nodes), "nested Domain Users group missing")
|
|
|
|
announce("SMB authorization and real share reads/writes")
|
|
alice_access = engine_run(
|
|
"exec", CLIENT_CONTAINER, "smbclient", f"//files.{DNS_DOMAIN}/Data", "-m", "SMB3",
|
|
"-U", f"{WORKGROUP}\\alice%{USER_PASSWORD}", "-c", "cd Finance; ls",
|
|
check_result=False,
|
|
)
|
|
check(alice_access.returncode == 0, f"Alice cannot access Finance: {alice_access.stderr}")
|
|
dave_denied = engine_run(
|
|
"exec", CLIENT_CONTAINER, "smbclient", f"//files.{DNS_DOMAIN}/Data", "-m", "SMB3",
|
|
"-U", f"{WORKGROUP}\\dave%{USER_PASSWORD}", "-c", "cd Finance; ls",
|
|
check_result=False,
|
|
)
|
|
check(dave_denied.returncode != 0, "unrelated user Dave can access Finance")
|
|
frank_projects = engine_run(
|
|
"exec", CLIENT_CONTAINER, "smbclient", f"//files.{DNS_DOMAIN}/Data", "-m", "SMB3",
|
|
"-U", f"{WORKGROUP}\\frank%{USER_PASSWORD}", "-c", "cd Projects; ls",
|
|
check_result=False,
|
|
)
|
|
check(frank_projects.returncode == 0, "primary Domain Users membership did not grant Projects access")
|
|
admin_access = engine_run(
|
|
"exec", CLIENT_CONTAINER, "smbclient", f"//files.{DNS_DOMAIN}/Data", "-m", "SMB3",
|
|
"-U", f"{WORKGROUP}\\{ADMIN_USER}%{ADMIN_PASSWORD}", "-c", "cd Finance; ls",
|
|
check_result=False,
|
|
)
|
|
check(admin_access.returncode == 0, "Domain Admin cannot inspect Finance")
|
|
|
|
announce("group, Private, and FSLogix size accounting")
|
|
storage = http("/api/storage", token=token)
|
|
check(storage.status == 200, f"storage endpoint failed: {storage.body!r}")
|
|
storage_payload = storage.json()
|
|
storage_groups = {row["name"]: int(row["bytes"]) for row in storage_payload.get("groups", [])}
|
|
check(storage_groups.get("Finance", 0) >= 1024 * 1024, "Finance usage was not scanned")
|
|
check(storage_groups.get("Engineering", 0) >= 1024 * 1024, "Engineering usage was not scanned")
|
|
users = {row["name"].casefold(): row for row in storage_payload.get("users", [])}
|
|
check(int(users.get("alice", {}).get("privateBytes", 0)) > 0, "Alice Private usage missing")
|
|
check(int(users.get("alice", {}).get("fslogixBytes", 0)) > 0, "Alice FSLogix usage missing")
|
|
check(int(storage_payload.get("totals", {}).get("dataBytes", 0)) > 0, "Data total is empty")
|
|
|
|
announce("high-level Samba audit ingestion, suffix exclusions, filters, facets, and pagination")
|
|
required_actions = {"read", "write", "move", "delete"}
|
|
activity = eventually(
|
|
"all four live alice audit actions",
|
|
lambda: http(query_path("/api/activity", {"user": "alice", "limit": "100"}), token=token),
|
|
lambda response: (
|
|
response.status == 200
|
|
and required_actions.issubset(
|
|
{event.get("action") for event in response.json().get("events", [])}
|
|
)
|
|
),
|
|
timeout=60,
|
|
)
|
|
activity_payload = activity.json()
|
|
alice_actions = {event.get("action") for event in activity_payload["events"]}
|
|
check(all("alice" in str(event.get("user", "")).casefold() for event in activity_payload["events"]), "activity user filter leaked unrelated events")
|
|
check(alice_actions == required_actions, f"unexpected alice audit actions: {sorted(alice_actions)}")
|
|
check(set(activity_payload.get("facets", {}).get("actions", [])) <= required_actions, "low-level activity category leaked into facets")
|
|
check("Data" in activity_payload.get("facets", {}).get("shares", []), "activity share facets missing Data")
|
|
|
|
moved = http(query_path("/api/activity", {"user": "alice", "action": "move", "limit": "100"}), token=token).json()
|
|
check(moved.get("matched", 0) >= 1, "move action filter returned no event")
|
|
check(all(event.get("action") == "move" for event in moved.get("events", [])), "move action filter leaked another action")
|
|
|
|
eventually(
|
|
"raw service-account SMB audit source",
|
|
lambda: engine_run(
|
|
"exec", FILES_CONTAINER, "grep", "-R", "report_svc", "/var/log/samba",
|
|
check_result=False,
|
|
).returncode,
|
|
lambda returncode: returncode == 0,
|
|
timeout=30,
|
|
)
|
|
service_activity = http(query_path("/api/activity", {"user": "report_svc", "limit": "100"}), token=token).json()
|
|
check(service_activity.get("matched") == 0, "_svc account was persisted in the activity archive")
|
|
|
|
one_event = http(query_path("/api/activity", {"limit": "1"}), token=token).json()
|
|
check(len(one_event.get("events", [])) == 1, "activity limit was ignored")
|
|
check(one_event.get("nextCursor") is not None, "activity pagination cursor missing")
|
|
|
|
announce("shared SQLite state, indexes, integrity, and ordered read deduplication")
|
|
integrity = engine_run(
|
|
"exec", FILES_CONTAINER, "sqlite3", "/state/shares.db", "PRAGMA quick_check;"
|
|
)
|
|
check(integrity.stdout.strip() == "ok", "shared SQLite database failed quick_check")
|
|
tables = set(
|
|
engine_run(
|
|
"exec",
|
|
FILES_CONTAINER,
|
|
"sqlite3",
|
|
"/state/shares.db",
|
|
"SELECT name FROM sqlite_schema WHERE type='table' ORDER BY name;",
|
|
).stdout.splitlines()
|
|
)
|
|
check(
|
|
{"shares", "audit_events", "audit_sources", "web_cache"}.issubset(tables),
|
|
f"shared SQLite tables are incomplete: {sorted(tables)}",
|
|
)
|
|
indexes = set(
|
|
engine_run(
|
|
"exec",
|
|
FILES_CONTAINER,
|
|
"sqlite3",
|
|
"/state/shares.db",
|
|
"SELECT name FROM sqlite_schema WHERE type='index' AND name LIKE 'audit_events_%';",
|
|
).stdout.splitlines()
|
|
)
|
|
check(
|
|
{"audit_events_time", "audit_events_user_time", "audit_events_action_time"}.issubset(indexes),
|
|
f"audit indexes are incomplete: {sorted(indexes)}",
|
|
)
|
|
duplicate_reads = engine_run(
|
|
"exec",
|
|
FILES_CONTAINER,
|
|
"sqlite3",
|
|
"/state/shares.db",
|
|
"""SELECT count(*) FROM (
|
|
SELECT action, occurred_second, user, client_ip, share, path, success, result,
|
|
lag(action) OVER (ORDER BY id) AS previous_action,
|
|
lag(occurred_second) OVER (ORDER BY id) AS previous_second,
|
|
lag(user) OVER (ORDER BY id) AS previous_user,
|
|
lag(client_ip) OVER (ORDER BY id) AS previous_client_ip,
|
|
lag(share) OVER (ORDER BY id) AS previous_share,
|
|
lag(path) OVER (ORDER BY id) AS previous_path,
|
|
lag(success) OVER (ORDER BY id) AS previous_success,
|
|
lag(result) OVER (ORDER BY id) AS previous_result
|
|
FROM audit_events
|
|
) WHERE action='read' AND previous_action='read'
|
|
AND occurred_second=previous_second AND user=previous_user
|
|
AND client_ip=previous_client_ip AND share=previous_share
|
|
AND path=previous_path AND success=previous_success
|
|
AND (success=1 OR result=previous_result);""",
|
|
)
|
|
check(duplicate_reads.stdout.strip() == "0", "uninterrupted duplicate reads remain")
|
|
legacy_archive = engine_run(
|
|
"exec", FILES_CONTAINER, "test", "!", "-e", "/state/audit", check_result=False
|
|
)
|
|
check(legacy_archive.returncode == 0, "legacy JSONL audit archive still exists")
|
|
|
|
announce("real rsync backup, status API, log tail, and remote completion marker")
|
|
backup = eventually(
|
|
"completed backup",
|
|
lambda: http("/api/backup", token=token),
|
|
lambda response: response.status == 200 and response.json().get("state") in {"completed", "failed"},
|
|
timeout=240,
|
|
interval=2,
|
|
)
|
|
backup_payload = backup.json()
|
|
check(backup_payload.get("state") == "completed", f"backup failed: {backup_payload}")
|
|
check(float(backup_payload.get("percent", 0)) == 100.0, "completed backup is not at 100%")
|
|
check(any("completed" in line.casefold() for line in backup_payload.get("log", [])), "backup completion absent from log tail")
|
|
marker = engine_run(
|
|
"exec", BACKUP_CONTAINER, "sh", "-ec",
|
|
"find /backup/fileserver/snapshots -name .backup_complete -type f | grep -q .",
|
|
check_result=False,
|
|
)
|
|
check(marker.returncode == 0, "backup target has no completed snapshot marker")
|
|
|
|
announce("overview and system health aggregation")
|
|
overview = http("/api/overview", token=token)
|
|
check(overview.status == 200 and overview.json().get("activeGroups") == 3, "overview group count is wrong")
|
|
system = http("/api/system", token=token)
|
|
check(system.status == 200, f"system endpoint failed: {system.body!r}")
|
|
system_payload = system.json()
|
|
check(system_payload.get("checks", {}).get("domainTrust") is True, "system reports broken AD trust")
|
|
check(system_payload.get("checks", {}).get("sambaConfig") is True, "system reports invalid Samba config")
|
|
check(system_payload.get("tls", {}).get("sans"), "system TLS summary is empty")
|
|
check(system_payload.get("audit", {}).get("days", 0) >= 1, "system activity database summary is incomplete")
|
|
|
|
logout = http("/api/logout", method="POST", value={}, token=token)
|
|
check(logout.status == 200 and "Max-Age=0" in logout.headers.get("Set-Cookie", ""), "logout did not clear session cookie")
|
|
|
|
announce("PASS: all end-to-end assertions succeeded")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.exit(main())
|
|
except Exception as exc: # pylint: disable=broad-except
|
|
print(f"[e2e] FAIL: {exc}", file=sys.stderr, flush=True)
|
|
sys.exit(1)
|