runner: overlay-root appliance mode

docs/RUNNER_IMAGES.md

@@ -15,6 +15,19 @@ The runner OS exposes this repo's flake source inside the system at:
 
 This allows a runner to self-build and verify artifacts from the same flake definition.
 
+## Read-Only Root + tmpfs Writes
+
+The base runner profile enables an initrd overlay that:
+
+- remounts the real `/` read-only, and
+- provides a tmpfs-backed writable overlay upperdir.
+
+For reliable upgrades and operation, mount persistent filesystems for:
+
+- `/boot` (so new boot entries persist)
+- `/nix` (so store contents persist across reboots)
+- `/var` or selected `/var/lib/*` paths (for any state you care about)
+
 ## Build (OrbStack / Linux)
 
 These commands should be run inside a Linux environment with Nix enabled (e.g. OrbStack VM).

evolution/proposals/ECP-0065-nixos-runner-images.md

@@ -15,6 +15,7 @@ The runner system:
 - is defined in-repo as a `nixosConfiguration` in `flake.nix`,
 - exports the repo source tree inside the OS at a stable path (read-only) so the node can self-build and verify from the same flake,
 - uses `ec-node` as the primary long-running publisher binary, with orchestration via NixOS + systemd.
+- defaults to a read-only root filesystem with a tmpfs-backed overlayfs upperdir (appliance semantics), while image/bootstrap variants (netboot/ISO/sdimage) may disable this where it conflicts with their initrd/root setup.
 
 Initial implementation targets `aarch64-linux` builds first (local builds via OrbStack). `x86_64-linux` is defined in the flake but may not be built until an x86 builder is available.
 

flake.nix

@@ -40,6 +40,9 @@
             ({ modulesPath, ... }: {
               imports = [ (modulesPath + "/installer/netboot/netboot-minimal.nix") ];
             })
+            ({ ... }: {
+              services.every-channel.runner.overlayRoot.enable = false;
+            })
             ({ config, pkgs, ... }: {
               # Convenience output dir: { kernel, initrd, netboot.ipxe }.
               system.build.netboot = pkgs.linkFarm "ec-runner-netboot" [
@@ -62,6 +65,9 @@
             ({ modulesPath, ... }: {
               imports = [ (modulesPath + "/installer/netboot/netboot-minimal.nix") ];
             })
+            ({ ... }: {
+              services.every-channel.runner.overlayRoot.enable = false;
+            })
             ({ config, pkgs, ... }: {
               system.build.netboot = pkgs.linkFarm "ec-runner-netboot" [
                 {
@@ -85,6 +91,9 @@
             ({ modulesPath, ... }: {
               imports = [ (modulesPath + "/installer/cd-dvd/installation-cd-minimal.nix") ];
             })
+            ({ ... }: {
+              services.every-channel.runner.overlayRoot.enable = false;
+            })
           ];
 
           # aarch64 SD image (useful for quick ARM bring-up and as a "real image" build target).
@@ -92,6 +101,9 @@
             ({ modulesPath, ... }: {
               imports = [ (modulesPath + "/installer/sd-card/sd-image-aarch64.nix") ];
             })
+            ({ ... }: {
+              services.every-channel.runner.overlayRoot.enable = false;
+            })
           ];
         };
     }

nix/modules/ec-runner.nix

@@ -6,6 +6,21 @@ in
 {
   options.services.every-channel.runner = {
     enable = lib.mkEnableOption "every.channel runner base system profile";
+
+    overlayRoot = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          If enabled, mount the real root filesystem read-only and layer a tmpfs-backed
+          overlayfs upperdir on top. This makes runtime mutations non-persistent while
+          still allowing normal operation.
+
+          Note: for reliable in-place upgrades, mount `/boot` and `/nix` as separate
+          persistent filesystems outside the overlay.
+        '';
+      };
+    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -24,5 +39,42 @@ in
       jq
       curl
     ];
+
+    # Appliance defaults: avoid persistence-by-accident in logs.
+    services.journald.storage = lib.mkDefault "volatile";
+
+    # GC defaults to keep store growth bounded on unattended boxes.
+    nix.gc.automatic = lib.mkDefault true;
+    nix.gc.dates = lib.mkDefault "weekly";
+    nix.gc.options = lib.mkDefault "--delete-older-than 14d";
+    boot.loader.systemd-boot.configurationLimit = lib.mkDefault 10;
+
+    boot.initrd.kernelModules = lib.mkIf cfg.overlayRoot.enable [ "overlay" ];
+
+    # Make the on-disk root read-only and provide a tmpfs-backed writable layer.
+    # This runs after the real root has been mounted at /mnt-root by stage-1.
+    boot.initrd.postMountCommands = lib.mkIf cfg.overlayRoot.enable ''
+      set -euo pipefail
+
+      # If something else already overlaid the root (e.g. installer media), do nothing.
+      if mountpoint -q /mnt-root; then
+        if grep -q " /mnt-root overlay " /proc/mounts; then
+          exit 0
+        fi
+      fi
+
+      mkdir -p /mnt-root/.ec-overlay/ro /mnt-root/.ec-overlay/rw
+
+      # Move the real root mount out of the way, then remount it read-only.
+      mount --move /mnt-root /mnt-root/.ec-overlay/ro
+      mount -o remount,ro /mnt-root/.ec-overlay/ro
+
+      # Upper/work go on tmpfs so mutations disappear on reboot.
+      mount -t tmpfs tmpfs /mnt-root/.ec-overlay/rw -o mode=0755
+      mkdir -p /mnt-root/.ec-overlay/rw/upper /mnt-root/.ec-overlay/rw/work
+
+      mount -t overlay overlay /mnt-root \
+        -o lowerdir=/mnt-root/.ec-overlay/ro,upperdir=/mnt-root/.ec-overlay/rw/upper,workdir=/mnt-root/.ec-overlay/rw/work
+    '';
   };
 }

nix/nixos/ec-runner.nix

@@ -7,6 +7,7 @@
   ];
 
   services.every-channel.runner.enable = true;
+  services.every-channel.runner.overlayRoot.enable = lib.mkDefault true;
 
   # This is a role image; avoid baking per-host secrets/keys. SSH host keys will be
   # generated at first boot by NixOS defaults.
@@ -26,4 +27,3 @@
   # Required by NixOS.
   system.stateVersion = "24.11";
 }
-