From ce8c1319f420a4ca15893a24cff2e9c819d30359 Mon Sep 17 00:00:00 2001 From: "every.channel" Date: Tue, 17 Feb 2026 02:26:09 -0800 Subject: [PATCH] runner: overlay-root appliance mode --- docs/RUNNER_IMAGES.md | 13 +++++ .../proposals/ECP-0065-nixos-runner-images.md | 1 + flake.nix | 12 +++++ nix/modules/ec-runner.nix | 52 +++++++++++++++++++ nix/nixos/ec-runner.nix | 2 +- 5 files changed, 79 insertions(+), 1 deletion(-) diff --git a/docs/RUNNER_IMAGES.md b/docs/RUNNER_IMAGES.md index fa96f5b..ffe5c22 100644 --- a/docs/RUNNER_IMAGES.md +++ b/docs/RUNNER_IMAGES.md @@ -15,6 +15,19 @@ The runner OS exposes this repo's flake source inside the system at: This allows a runner to self-build and verify artifacts from the same flake definition. +## Read-Only Root + tmpfs Writes + +The base runner profile enables an initrd overlay that: + +- remounts the real `/` read-only, and +- provides a tmpfs-backed writable overlay upperdir. + +For reliable upgrades and operation, mount persistent filesystems for: + +- `/boot` (so new boot entries persist) +- `/nix` (so store contents persist across reboots) +- `/var` or selected `/var/lib/*` paths (for any state you care about) + ## Build (OrbStack / Linux) These commands should be run inside a Linux environment with Nix enabled (e.g. OrbStack VM). diff --git a/evolution/proposals/ECP-0065-nixos-runner-images.md b/evolution/proposals/ECP-0065-nixos-runner-images.md index dd6b4b9..3899324 100644 --- a/evolution/proposals/ECP-0065-nixos-runner-images.md +++ b/evolution/proposals/ECP-0065-nixos-runner-images.md @@ -15,6 +15,7 @@ The runner system: - is defined in-repo as a `nixosConfiguration` in `flake.nix`, - exports the repo source tree inside the OS at a stable path (read-only) so the node can self-build and verify from the same flake, - uses `ec-node` as the primary long-running publisher binary, with orchestration via NixOS + systemd. +- defaults to a read-only root filesystem with a tmpfs-backed overlayfs upperdir (appliance semantics), while image/bootstrap variants (netboot/ISO/sdimage) may disable this where it conflicts with their initrd/root setup. Initial implementation targets `aarch64-linux` builds first (local builds via OrbStack). `x86_64-linux` is defined in the flake but may not be built until an x86 builder is available. diff --git a/flake.nix b/flake.nix index 36a7082..8fd87d0 100644 --- a/flake.nix +++ b/flake.nix @@ -40,6 +40,9 @@ ({ modulesPath, ... }: { imports = [ (modulesPath + "/installer/netboot/netboot-minimal.nix") ]; }) + ({ ... }: { + services.every-channel.runner.overlayRoot.enable = false; + }) ({ config, pkgs, ... }: { # Convenience output dir: { kernel, initrd, netboot.ipxe }. system.build.netboot = pkgs.linkFarm "ec-runner-netboot" [ @@ -62,6 +65,9 @@ ({ modulesPath, ... }: { imports = [ (modulesPath + "/installer/netboot/netboot-minimal.nix") ]; }) + ({ ... }: { + services.every-channel.runner.overlayRoot.enable = false; + }) ({ config, pkgs, ... }: { system.build.netboot = pkgs.linkFarm "ec-runner-netboot" [ { @@ -85,6 +91,9 @@ ({ modulesPath, ... }: { imports = [ (modulesPath + "/installer/cd-dvd/installation-cd-minimal.nix") ]; }) + ({ ... }: { + services.every-channel.runner.overlayRoot.enable = false; + }) ]; # aarch64 SD image (useful for quick ARM bring-up and as a "real image" build target). @@ -92,6 +101,9 @@ ({ modulesPath, ... }: { imports = [ (modulesPath + "/installer/sd-card/sd-image-aarch64.nix") ]; }) + ({ ... }: { + services.every-channel.runner.overlayRoot.enable = false; + }) ]; }; } diff --git a/nix/modules/ec-runner.nix b/nix/modules/ec-runner.nix index 5b8ef35..2604f98 100644 --- a/nix/modules/ec-runner.nix +++ b/nix/modules/ec-runner.nix @@ -6,6 +6,21 @@ in { options.services.every-channel.runner = { enable = lib.mkEnableOption "every.channel runner base system profile"; + + overlayRoot = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + If enabled, mount the real root filesystem read-only and layer a tmpfs-backed + overlayfs upperdir on top. This makes runtime mutations non-persistent while + still allowing normal operation. + + Note: for reliable in-place upgrades, mount `/boot` and `/nix` as separate + persistent filesystems outside the overlay. + ''; + }; + }; }; config = lib.mkIf cfg.enable { @@ -24,5 +39,42 @@ in jq curl ]; + + # Appliance defaults: avoid persistence-by-accident in logs. + services.journald.storage = lib.mkDefault "volatile"; + + # GC defaults to keep store growth bounded on unattended boxes. + nix.gc.automatic = lib.mkDefault true; + nix.gc.dates = lib.mkDefault "weekly"; + nix.gc.options = lib.mkDefault "--delete-older-than 14d"; + boot.loader.systemd-boot.configurationLimit = lib.mkDefault 10; + + boot.initrd.kernelModules = lib.mkIf cfg.overlayRoot.enable [ "overlay" ]; + + # Make the on-disk root read-only and provide a tmpfs-backed writable layer. + # This runs after the real root has been mounted at /mnt-root by stage-1. + boot.initrd.postMountCommands = lib.mkIf cfg.overlayRoot.enable '' + set -euo pipefail + + # If something else already overlaid the root (e.g. installer media), do nothing. + if mountpoint -q /mnt-root; then + if grep -q " /mnt-root overlay " /proc/mounts; then + exit 0 + fi + fi + + mkdir -p /mnt-root/.ec-overlay/ro /mnt-root/.ec-overlay/rw + + # Move the real root mount out of the way, then remount it read-only. + mount --move /mnt-root /mnt-root/.ec-overlay/ro + mount -o remount,ro /mnt-root/.ec-overlay/ro + + # Upper/work go on tmpfs so mutations disappear on reboot. + mount -t tmpfs tmpfs /mnt-root/.ec-overlay/rw -o mode=0755 + mkdir -p /mnt-root/.ec-overlay/rw/upper /mnt-root/.ec-overlay/rw/work + + mount -t overlay overlay /mnt-root \ + -o lowerdir=/mnt-root/.ec-overlay/ro,upperdir=/mnt-root/.ec-overlay/rw/upper,workdir=/mnt-root/.ec-overlay/rw/work + ''; }; } diff --git a/nix/nixos/ec-runner.nix b/nix/nixos/ec-runner.nix index 1e11bd2..7ed9aab 100644 --- a/nix/nixos/ec-runner.nix +++ b/nix/nixos/ec-runner.nix @@ -7,6 +7,7 @@ ]; services.every-channel.runner.enable = true; + services.every-channel.runner.overlayRoot.enable = lib.mkDefault true; # This is a role image; avoid baking per-host secrets/keys. SSH host keys will be # generated at first boot by NixOS defaults. @@ -26,4 +27,3 @@ # Required by NixOS. system.stateVersion = "24.11"; } -