webui (2)

.env.example

@@ -18,6 +18,9 @@ WEB_JWT_SECRET=ReplaceWithAtLeast32RandomBytes
 WEB_TLS_MODE=acme
 ACME_CA_SERVER=https://ca.internal/acme/acme/directory
 ACME_CA_CERTIFICATES=/state/tls/acme-ca-certificates.pem
+# Optional one-time HTTPS download when ACME_CA_CERTIFICATES does not exist:
+# ACME_CA_CERTIFICATES_URL=https://ca.internal/roots.pem
+# ACME_CA_CERTIFICATES_INSECURE_DOWNLOAD=true
 ACME_HTTP_LISTEN=:80
 ACME_HTTP_PORT=80
 # ACME_RENEW_CHECK_SECONDS=900

README.md

@@ -211,10 +211,10 @@ The runner returns non-zero on the first failed assertion, prints bounded logs f
    - `WEB_HOSTNAME` (the DNS name in the HTTPS certificate)
    - optional `WEB_HTTPS_PORT` (host port, default `443`)
    - `ACME_CA_SERVER` (ACME directory URL)
-   - ACME CA root bundle (a host PEM file imported into the persistent state volume)
+   - either an ACME CA root bundle on the host or an `ACME_CA_CERTIFICATES_URL` for one-time retrieval
    - optional `ACME_HTTP_PORT` (HTTP-01 host port, default `80`)
 
-   Setup generates a random `WEB_JWT_SECRET`, stores only the generic ACME settings, and imports the public CA root as `/state/tls/acme-ca-certificates.pem`.
+   Setup generates a random `WEB_JWT_SECRET` and either imports the selected public CA root or configures its one-time download to `/state/tls/acme-ca-certificates.pem`.
 
    Optional:
    - `SAMBA_HOSTNAME` (defaults to `adsambafsrv`)
@@ -313,11 +313,17 @@ WEB_JWT_SECRET=<at-least-32-random-bytes>
 WEB_TLS_MODE=acme
 ACME_CA_SERVER=https://ca.internal/acme/acme/directory
 ACME_CA_CERTIFICATES=/state/tls/acme-ca-certificates.pem
+ACME_CA_CERTIFICATES_URL=https://ca.internal/roots.pem
+ACME_CA_CERTIFICATES_INSECURE_DOWNLOAD=true
 ACME_HTTP_LISTEN=:80
 ACME_HTTP_PORT=80
 ```
 
-`ACME_CA_CERTIFICATES` is a path inside the file-server container. The setup script imports the selected host PEM bundle at the path above; manual deployments must copy or mount the bundle there. The CA must be able to resolve `WEB_HOSTNAME` and reach `http://WEB_HOSTNAME/.well-known/acme-challenge/...` on port 80. `ACME_HTTP_LISTEN` controls the in-container standalone challenge listener and `ACME_HTTP_PORT` controls the Compose host-port mapping.
+`ACME_CA_CERTIFICATES` is the persistent destination inside the file-server container. If that path does not exist and `ACME_CA_CERTIFICATES_URL` is set, the service downloads at most 4 MiB over HTTPS, validates that the result contains readable certificate data, and atomically installs it. Once the destination exists, it is never downloaded or overwritten again—not even when it is empty or invalid. Removing the file is therefore an explicit operator action that permits a new bootstrap download.
+
+HTTPS verification is enabled by default. `ACME_CA_CERTIFICATES_INSECURE_DOWNLOAD=true` explicitly enables a one-time trust-on-first-use bootstrap equivalent to `curl -k`; it is useful when the server certificate is signed by the very root being fetched. This protects subsequent starts but cannot protect the first download from a man-in-the-middle attack, so compare the downloaded root fingerprint through an independent channel when possible. Instead of downloading, setup can import a local PEM bundle into the same destination.
+
+The CA must be able to resolve `WEB_HOSTNAME` and reach `http://WEB_HOSTNAME/.well-known/acme-challenge/...` on port 80. `ACME_HTTP_LISTEN` controls the in-container standalone challenge listener and `ACME_HTTP_PORT` controls the Compose host-port mapping.
 
 The equivalent Traefik settings translate as follows:
 
@@ -326,7 +332,8 @@ The equivalent Traefik settings translate as follows:
 | `acme.httpchallenge=true` | `WEB_TLS_MODE=acme` |
 | `acme.httpchallenge.entrypoint=http` | `ACME_HTTP_LISTEN=:80` and `ACME_HTTP_PORT=80` |
 | `acme.caserver=https://ca.internal/acme/acme/directory` | `ACME_CA_SERVER=https://ca.internal/acme/acme/directory` |
-| `acme.cacertificates=/traefik/stepca/roots.pem` | `ACME_CA_CERTIFICATES=<the mounted PEM path inside this container>` |
+| `acme.cacertificates=/traefik/stepca/roots.pem` | `ACME_CA_CERTIFICATES=<persistent PEM destination inside this container>` |
+| one-time equivalent of `curl https://ca.internal/roots.pem -k` | `ACME_CA_CERTIFICATES_URL=https://ca.internal/roots.pem` and `ACME_CA_CERTIFICATES_INSECURE_DOWNLOAD=true` |
 
 The client uses [`step ca certificate --acme --root --standalone`](https://smallstep.com/docs/step-cli/reference/ca/certificate/) and periodically checks whether the certificate needs renewal. Renewal is a fresh ACME order, so this mode is not coupled to a proprietary renewal endpoint. The HTTPS listener reloads the replaced certificate files.
 
@@ -356,6 +363,8 @@ Useful optional settings:
 
 | Variable | Default | Purpose |
 | --- | ---: | --- |
+| `ACME_CA_CERTIFICATES_URL` | unset | HTTPS URL used only when the persistent root bundle does not exist |
+| `ACME_CA_CERTIFICATES_INSECURE_DOWNLOAD` | `false` | Disable TLS verification only for that initial root download |
 | `WEB_TLS_AUTORENEW` | `true` | Renew managed certificates automatically |
 | `ACME_RENEW_CHECK_SECONDS` | `900` | Interval for ACME renewal checks; minimum 60 seconds |
 | `WEB_JWT_TTL_SECONDS` | `28800` | Session lifetime, 5 minutes to 7 days |

app/init.sh

@@ -343,6 +343,77 @@ selected_tls_mode() {
   esac
 }
 
+ensure_acme_ca_certificates() {
+  local target="$ACME_CA_CERTIFICATES"
+  local download_url="${ACME_CA_CERTIFICATES_URL:-}"
+  local download_tmp="${target}.download.tmp"
+  local insecure=false
+
+  if [[ ! -e "$target" ]]; then
+    if [[ -z "$download_url" ]]; then
+      printf '[init] ERROR: ACME_CA_CERTIFICATES does not exist and ACME_CA_CERTIFICATES_URL is unset: %s\n' "$target" >&2
+      return 1
+    fi
+    case "$download_url" in
+      https://*) ;;
+      *)
+        printf '[init] ERROR: ACME_CA_CERTIFICATES_URL must use https://\n' >&2
+        return 1
+        ;;
+    esac
+    if env_is_true "${ACME_CA_CERTIFICATES_INSECURE_DOWNLOAD:-false}"; then
+      insecure=true
+      log 'WARNING: downloading the initial ACME CA certificate bundle without TLS verification (one-time TOFU bootstrap).'
+    else
+      log 'Downloading the initial ACME CA certificate bundle with TLS verification.'
+    fi
+
+    mkdir -p "$(dirname "$target")"
+    rm -f "$download_tmp"
+    if ! python3 - "$download_url" "$download_tmp" "$insecure" <<'PY'
+import ssl
+import sys
+import urllib.parse
+import urllib.request
+
+url, output, insecure_value = sys.argv[1:]
+context = ssl.create_default_context()
+if insecure_value == "true":
+    context = ssl._create_unverified_context()  # noqa: SLF001
+request = urllib.request.Request(url, headers={"User-Agent": "ad-file-server-ca-bootstrap/1"})
+with urllib.request.urlopen(request, context=context, timeout=30) as response:
+    if urllib.parse.urlsplit(response.geturl()).scheme != "https":
+        raise RuntimeError("CA certificate download redirected away from HTTPS")
+    payload = response.read(4 * 1024 * 1024 + 1)
+if not payload or len(payload) > 4 * 1024 * 1024:
+    raise RuntimeError("CA certificate download is empty or exceeds 4 MiB")
+with open(output, "xb") as handle:
+    handle.write(payload)
+PY
+    then
+      rm -f "$download_tmp"
+      return 1
+    fi
+    if ! step certificate inspect "$download_tmp" >/dev/null 2>&1; then
+      printf '[init] ERROR: downloaded ACME CA certificate bundle is not valid PEM certificate data\n' >&2
+      rm -f "$download_tmp"
+      return 1
+    fi
+    chmod 0644 "$download_tmp"
+    mv -f "$download_tmp" "$target"
+    log "Stored the initial ACME CA certificate bundle at ${target}; it will not be downloaded again while this file exists."
+  fi
+
+  if [[ ! -s "$target" ]]; then
+    printf '[init] ERROR: ACME_CA_CERTIFICATES exists but is empty: %s\n' "$target" >&2
+    return 1
+  fi
+  if ! step certificate inspect "$target" >/dev/null 2>&1; then
+    printf '[init] ERROR: ACME_CA_CERTIFICATES is not valid PEM certificate data: %s\n' "$target" >&2
+    return 1
+  fi
+}
+
 request_acme_certificate() {
   local cert_tmp="${WEB_TLS_CERT_FILE}.acme.tmp"
   local key_tmp="${WEB_TLS_KEY_FILE}.acme.tmp"
@@ -392,10 +463,7 @@ configure_web_tls() {
   if [[ "$tls_mode" == "acme" ]]; then
     require_env ACME_CA_SERVER
     require_env ACME_CA_CERTIFICATES
-    if [[ ! -s "$ACME_CA_CERTIFICATES" ]]; then
+    ensure_acme_ca_certificates
-      printf '[init] ERROR: ACME_CA_CERTIFICATES is not a readable PEM bundle: %s\n' "$ACME_CA_CERTIFICATES" >&2
-      return 1
-    fi
     if [[ ! -s "$WEB_TLS_CERT_FILE" || ! -s "$WEB_TLS_KEY_FILE" ]]; then
       request_acme_certificate
     fi

scripts/dev

@@ -241,12 +241,6 @@ fi
 preview_tmp_root=${TMPDIR:-/tmp}
 ca_root_file="${preview_tmp_root%/}/${run_id}-root-ca.crt"
 "$engine" exec "$ca_container" cat /home/step/certs/root_ca.crt > "$ca_root_file"
-"$engine" run --rm -i \
-  -v "$state_volume:/state" \
-  --entrypoint /bin/bash \
-  "$server_image" -lc \
-  'install -d -m 0755 /state/tls && dd of=/state/tls/acme-ca-certificates.pem status=none && chmod 0644 /state/tls/acme-ca-certificates.pem' \
-  < "$ca_root_file"
 
 printf 'starting disposable rsync backup target\n'
 "$engine" run -d \
@@ -325,6 +319,8 @@ printf 'starting actual file server and HTTPS web UI\n'
   -e "WEB_TLS_MODE=acme" \
   -e "ACME_CA_SERVER=https://ca.${dev_dns_domain}:9000/acme/acme/directory" \
   -e "ACME_CA_CERTIFICATES=/state/tls/acme-ca-certificates.pem" \
+  -e "ACME_CA_CERTIFICATES_URL=https://ca.${dev_dns_domain}:9000/roots.pem" \
+  -e "ACME_CA_CERTIFICATES_INSECURE_DOWNLOAD=true" \
   -e "ACME_HTTP_LISTEN=:80" \
   -e "WEB_USAGE_SCAN_INTERVAL_SECONDS=60" \
   -e "WEB_DIRECTORY_CACHE_SECONDS=30" \
@@ -350,6 +346,27 @@ if ! wait_for_exec "$files_container" 240 python3 -c \
   die 'file server web UI did not become healthy'
 fi
 
+if [[ $run_e2e == 1 ]]; then
+  printf 'validating one-time untrusted CA root retrieval across a restart\n'
+  initial_ca_bundle_stat=$("$engine" exec "$files_container" stat -c '%i:%s:%Y' /state/tls/acme-ca-certificates.pem)
+  "$engine" restart "$files_container" >/dev/null
+  if ! wait_for_exec "$files_container" 240 python3 -c \
+    'import ssl,urllib.request; print(urllib.request.urlopen("https://127.0.0.1:8443/healthz", context=ssl._create_unverified_context(), timeout=3).status)'; then
+    show_logs
+    die 'file server web UI did not become healthy after the one-time CA download restart check'
+  fi
+  restarted_ca_bundle_stat=$("$engine" exec "$files_container" stat -c '%i:%s:%Y' /state/tls/acme-ca-certificates.pem)
+  if [[ "$initial_ca_bundle_stat" != "$restarted_ca_bundle_stat" ]]; then
+    show_logs
+    die 'persisted ACME CA certificate bundle changed during restart'
+  fi
+  ca_bundle_download_count=$("$engine" logs "$files_container" 2>&1 | awk '/Stored the initial ACME CA certificate bundle/ { count++ } END { print count + 0 }')
+  if [[ "$ca_bundle_download_count" != 1 ]]; then
+    show_logs
+    die "expected exactly one ACME CA certificate download, got ${ca_bundle_download_count}"
+  fi
+fi
+
 printf 'starting continuous authenticated SMB activity client\n'
 "$engine" run -d \
   --name "$client_container" \

setup

@@ -141,6 +141,9 @@ write_env_file() {
   local acme_ca_server=""
   local acme_ca_certificates_source=""
   local acme_ca_certificates_source_dir=""
+  local acme_ca_certificates_url=""
+  local acme_ca_certificates_insecure_download="false"
+  local acme_ca_certificates_insecure_input=""
   local acme_http_port="80"
   local acme_http_port_input=""
   local web_hostname_input=""
@@ -219,6 +222,21 @@ write_env_file() {
   read -r -p "WEB_HTTPS_PORT [443]: " web_https_port
   web_https_port="${web_https_port:-443}"
   prompt_value acme_ca_server "ACME_CA_SERVER (e.g. https://ca.internal/acme/acme/directory)"
+  read -r -p "ACME_CA_CERTIFICATES_URL (optional one-time download; Enter to import a local PEM file): " acme_ca_certificates_url
+  if [[ -n "$acme_ca_certificates_url" ]]; then
+    case "$acme_ca_certificates_url" in
+      https://*) ;;
+      *)
+        printf "ACME_CA_CERTIFICATES_URL must use https://\n" >&2
+        return 1
+        ;;
+    esac
+    read -r -p "Allow untrusted TLS for this one-time root download? [y/N]: " acme_ca_certificates_insecure_input
+    case "$acme_ca_certificates_insecure_input" in
+      y|Y|yes|YES) acme_ca_certificates_insecure_download=true ;;
+      *) acme_ca_certificates_insecure_download=false ;;
+    esac
+  else
     prompt_value acme_ca_certificates_source "ACME CA root bundle (PEM file on this host)"
     if [[ ! -s "$acme_ca_certificates_source" ]]; then
       printf "ACME CA root bundle is not a readable file: %s\n" "$acme_ca_certificates_source" >&2
@@ -226,6 +244,7 @@ write_env_file() {
     fi
     acme_ca_certificates_source_dir="$(cd "$(dirname "$acme_ca_certificates_source")" && pwd)"
     acme_ca_certificates_source="${acme_ca_certificates_source_dir}/$(basename "$acme_ca_certificates_source")"
+  fi
   read -r -p "ACME_HTTP_PORT (HTTP-01 host port) [80]: " acme_http_port_input
   acme_http_port="${acme_http_port_input:-80}"
 
@@ -289,6 +308,8 @@ WEB_JWT_SECRET=${web_jwt_secret}
 WEB_TLS_MODE=acme
 ACME_CA_SERVER=${acme_ca_server}
 ACME_CA_CERTIFICATES=/state/tls/acme-ca-certificates.pem
+ACME_CA_CERTIFICATES_URL=${acme_ca_certificates_url}
+ACME_CA_CERTIFICATES_INSECURE_DOWNLOAD=${acme_ca_certificates_insecure_download}
 ACME_HTTP_LISTEN=:80
 ACME_HTTP_PORT=${acme_http_port}
 EOF
@@ -300,11 +321,13 @@ EOF
   printf "Building image...\n"
   docker compose build samba
 
+  if [[ -n "$acme_ca_certificates_source" ]]; then
     printf "Importing ACME CA root bundle...\n"
     docker compose run --rm --no-deps -T \
       --entrypoint /bin/bash samba -lc \
       'install -d -m 0755 "$(dirname "$ACME_CA_CERTIFICATES")" && dd of="$ACME_CA_CERTIFICATES" status=none && chmod 0644 "$ACME_CA_CERTIFICATES"' \
       < "$acme_ca_certificates_source"
+  fi
 
   printf "Provisioning service account %s...\n" "$SERVICE_ACCOUNT_NAME"
   docker compose run --rm --entrypoint /bin/bash samba -lc '
@@ -364,6 +387,8 @@ WEB_JWT_SECRET=${web_jwt_secret}
 WEB_TLS_MODE=acme
 ACME_CA_SERVER=${acme_ca_server}
 ACME_CA_CERTIFICATES=/state/tls/acme-ca-certificates.pem
+ACME_CA_CERTIFICATES_URL=${acme_ca_certificates_url}
+ACME_CA_CERTIFICATES_INSECURE_DOWNLOAD=${acme_ca_certificates_insecure_download}
 ACME_HTTP_LISTEN=:80
 ACME_HTTP_PORT=${acme_http_port}
 # Optional overrides: