Advance forge rollout, Ethereum rails, and NBC sources
This commit is contained in:
every.channel
parent
be26313225
commit
7d84510eac
88 changed files with 11230 additions and 302 deletions
377
nix/modules/ec-ipxe-qemu.nix
Normal file
377
nix/modules/ec-ipxe-qemu.nix
Normal file
|
|
@ -0,0 +1,377 @@
|
|||
{ lib, config, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.services.every-channel.ipxe-qemu;
|
||||
scriptsRoot = ../..;
|
||||
|
||||
firstDns = lib.head cfg.boot.userNet.dnsServers;
|
||||
qemuBin = "${cfg.package}/bin/qemu-system-x86_64";
|
||||
qemuImgBin = "${cfg.package}/bin/qemu-img";
|
||||
bootNetdevArg = lib.escapeShellArg "user,id=boot0,dns=${firstDns},hostname=${cfg.boot.userNet.hostname},tftp=${cfg.boot.userNet.tftpDir},bootfile=${cfg.boot.userNet.bootFilename}";
|
||||
bootDeviceArg = lib.escapeShellArg "${cfg.boot.userNet.model},netdev=boot0";
|
||||
lanNetdevArg = lib.escapeShellArg "tap,id=lan0,fd=3";
|
||||
lanDeviceArg = lib.escapeShellArg "${cfg.lan.model},netdev=lan0";
|
||||
driveArg = lib.escapeShellArg "if=virtio,format=qcow2,file=${cfg.disk.path}";
|
||||
effectiveChainUrl =
|
||||
if cfg.boot.chainUrl != null
|
||||
then cfg.boot.chainUrl
|
||||
else "http://10.0.2.2:${toString cfg.boot.http.port}/netboot.ipxe";
|
||||
|
||||
bootScript = pkgs.writeText "every-channel-qemu.ipxe" ''
|
||||
#!ipxe
|
||||
dhcp
|
||||
set dns ${firstDns}
|
||||
chain ${effectiveChainUrl} || shell
|
||||
'';
|
||||
bootAsset = pkgs.ipxe.override {
|
||||
embedScript = bootScript;
|
||||
additionalTargets = {
|
||||
"bin/undionly.kpxe" = null;
|
||||
};
|
||||
firmwareBinary = "undionly.kpxe";
|
||||
};
|
||||
|
||||
bootExtraArgs = lib.concatMapStringsSep "\n" (arg: " cmd+=(${lib.escapeShellArg arg})") cfg.extraArgs;
|
||||
|
||||
runner = pkgs.writeShellApplication {
|
||||
name = "every-channel-ipxe-qemu";
|
||||
runtimeInputs = [
|
||||
pkgs.coreutils
|
||||
pkgs.iproute2
|
||||
cfg.package
|
||||
];
|
||||
text = ''
|
||||
set -euo pipefail
|
||||
|
||||
state_dir=${lib.escapeShellArg cfg.stateDir}
|
||||
tftp_dir=${lib.escapeShellArg cfg.boot.userNet.tftpDir}
|
||||
boot_file=${lib.escapeShellArg cfg.boot.userNet.bootFilename}
|
||||
disk_path=${lib.escapeShellArg cfg.disk.path}
|
||||
disk_size=${lib.escapeShellArg cfg.disk.size}
|
||||
lan_ifname=${lib.escapeShellArg cfg.lan.macvtap.name}
|
||||
enable_kvm=${lib.escapeShellArg (lib.boolToString cfg.enableKvm)}
|
||||
enable_lan=${lib.escapeShellArg (lib.boolToString cfg.lan.enable)}
|
||||
serve_http=${lib.escapeShellArg (lib.boolToString cfg.boot.http.enable)}
|
||||
http_bind_ip=${lib.escapeShellArg cfg.boot.http.bindIp}
|
||||
http_port=${toString cfg.boot.http.port}
|
||||
http_root=${lib.escapeShellArg cfg.boot.http.root}
|
||||
|
||||
cleanup() {
|
||||
if [[ "$enable_lan" == true ]]; then
|
||||
ip link del "$lan_ifname" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
terminate() {
|
||||
if [[ -n "''${http_pid:-}" ]]; then
|
||||
kill "$http_pid" 2>/dev/null || true
|
||||
wait "$http_pid" 2>/dev/null || true
|
||||
fi
|
||||
if [[ -n "''${qemu_pid:-}" ]]; then
|
||||
kill "$qemu_pid" 2>/dev/null || true
|
||||
wait "$qemu_pid" 2>/dev/null || true
|
||||
fi
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
trap terminate TERM INT
|
||||
|
||||
install -d -m 0755 "$state_dir" "$tftp_dir" "$(dirname "$disk_path")"
|
||||
install -m 0644 ${bootAsset}/undionly.kpxe "$tftp_dir/$boot_file"
|
||||
|
||||
if [[ "$serve_http" == true ]]; then
|
||||
if [[ ! -d "$http_root" ]]; then
|
||||
echo "error: boot HTTP root not found: $http_root" >&2
|
||||
exit 1
|
||||
fi
|
||||
${pkgs.python3}/bin/python3 ${scriptsRoot}/scripts/netboot-http-server.py \
|
||||
--bind-ip "$http_bind_ip" \
|
||||
--port "$http_port" \
|
||||
--root "$http_root" &
|
||||
http_pid=$!
|
||||
fi
|
||||
|
||||
if [[ ! -f "$disk_path" ]]; then
|
||||
${qemuImgBin} create -f qcow2 "$disk_path" "$disk_size" >/dev/null
|
||||
fi
|
||||
|
||||
cmd=(
|
||||
${lib.escapeShellArg qemuBin}
|
||||
-name ${lib.escapeShellArg cfg.name}
|
||||
-machine ${lib.escapeShellArg cfg.machine}
|
||||
-cpu ${lib.escapeShellArg cfg.cpu}
|
||||
-smp ${toString cfg.vcpus}
|
||||
-m ${toString cfg.memoryMiB}
|
||||
-nographic
|
||||
-boot ${lib.escapeShellArg cfg.boot.order}
|
||||
-serial ${lib.escapeShellArg cfg.serial}
|
||||
-device virtio-rng-pci
|
||||
)
|
||||
|
||||
if [[ "$enable_kvm" == true ]]; then
|
||||
cmd+=(-enable-kvm)
|
||||
fi
|
||||
|
||||
cmd+=(
|
||||
-netdev ${bootNetdevArg}
|
||||
-device ${bootDeviceArg}
|
||||
)
|
||||
|
||||
if [[ "$enable_lan" == true ]]; then
|
||||
ip link del "$lan_ifname" 2>/dev/null || true
|
||||
ip link add link ${lib.escapeShellArg cfg.lan.macvtap.interface} name "$lan_ifname" type macvtap mode ${lib.escapeShellArg cfg.lan.macvtap.mode}
|
||||
ip link set "$lan_ifname" up
|
||||
tap_index="$(< /sys/class/net/$lan_ifname/ifindex)"
|
||||
exec 3<>"/dev/tap''${tap_index}"
|
||||
cmd+=(
|
||||
-netdev ${lanNetdevArg}
|
||||
-device ${lanDeviceArg}
|
||||
)
|
||||
fi
|
||||
|
||||
cmd+=(
|
||||
-drive ${driveArg}
|
||||
)
|
||||
${bootExtraArgs}
|
||||
|
||||
"''${cmd[@]}" &
|
||||
qemu_pid=$!
|
||||
wait "$qemu_pid"
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services.every-channel.ipxe-qemu = {
|
||||
enable = lib.mkEnableOption "every.channel iPXE/QEMU boot VM";
|
||||
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = pkgs.qemu_kvm;
|
||||
description = "QEMU package providing qemu-system-x86_64 and qemu-img.";
|
||||
};
|
||||
|
||||
name = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "every-channel-ipxe";
|
||||
description = "QEMU guest name.";
|
||||
};
|
||||
|
||||
machine = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "q35,accel=kvm";
|
||||
description = "QEMU machine string.";
|
||||
};
|
||||
|
||||
cpu = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "host";
|
||||
description = "QEMU CPU model.";
|
||||
};
|
||||
|
||||
enableKvm = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Use KVM acceleration when available.";
|
||||
};
|
||||
|
||||
vcpus = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 2;
|
||||
description = "Virtual CPU count.";
|
||||
};
|
||||
|
||||
memoryMiB = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 4096;
|
||||
description = "Guest memory in MiB.";
|
||||
};
|
||||
|
||||
serial = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "mon:stdio";
|
||||
description = "Serial/monitor wiring passed to QEMU.";
|
||||
};
|
||||
|
||||
stateDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/var/lib/every-channel/ipxe-qemu";
|
||||
description = "Persistent state directory for qcow2 disk and boot assets.";
|
||||
};
|
||||
|
||||
disk = {
|
||||
path = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "${cfg.stateDir}/disk.qcow2";
|
||||
description = "Persistent qcow2 disk path.";
|
||||
};
|
||||
|
||||
size = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "16G";
|
||||
description = "Size used when initializing the qcow2 disk.";
|
||||
};
|
||||
};
|
||||
|
||||
boot = {
|
||||
order = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "order=n";
|
||||
description = "QEMU boot order, defaulting to network first.";
|
||||
};
|
||||
|
||||
chainUrl = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional explicit iPXE chain target fetched by the guest; defaults to the local boot.http server.";
|
||||
};
|
||||
|
||||
http = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Serve a host-local HTTP root for the guest netboot flow.";
|
||||
};
|
||||
|
||||
bindIp = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "127.0.0.1";
|
||||
description = "Bind address used for the host-local boot HTTP server.";
|
||||
};
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8080;
|
||||
description = "Port used for the host-local boot HTTP server.";
|
||||
};
|
||||
|
||||
root = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Directory containing kernel/initrd/netboot.ipxe served to the guest.";
|
||||
};
|
||||
};
|
||||
|
||||
userNet = {
|
||||
hostname = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "every-channel-ipxe";
|
||||
description = "Hostname advertised on the user-mode boot network.";
|
||||
};
|
||||
|
||||
dnsServers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ "1.1.1.1" ];
|
||||
description = "DNS servers used by the user-mode boot NIC; the first entry is applied.";
|
||||
};
|
||||
|
||||
tftpDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "${cfg.stateDir}/tftp";
|
||||
description = "TFTP directory used by the user-mode boot NIC.";
|
||||
};
|
||||
|
||||
bootFilename = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "undionly.kpxe";
|
||||
description = "Boot asset filename exposed on the user-mode boot NIC.";
|
||||
};
|
||||
|
||||
model = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "e1000";
|
||||
description = "NIC model used for the boot network.";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
lan = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Attach a second NIC to a real LAN using macvtap.";
|
||||
};
|
||||
|
||||
model = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "virtio-net-pci";
|
||||
description = "NIC model used for the LAN attachment.";
|
||||
};
|
||||
|
||||
macvtap = {
|
||||
interface = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Host interface used as the lower device for macvtap.";
|
||||
};
|
||||
|
||||
name = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "ecqemu0";
|
||||
description = "macvtap link name created while the guest runs.";
|
||||
};
|
||||
|
||||
mode = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"private"
|
||||
"vepa"
|
||||
"bridge"
|
||||
"passthru"
|
||||
"source"
|
||||
];
|
||||
default = "bridge";
|
||||
description = "macvtap mode used for the LAN attachment.";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
extraArgs = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Additional arguments appended to the QEMU command.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.boot.userNet.dnsServers != [ ];
|
||||
message = "services.every-channel.ipxe-qemu.boot.userNet.dnsServers must not be empty";
|
||||
}
|
||||
{
|
||||
assertion = (!cfg.lan.enable) || (cfg.lan.macvtap.interface != null);
|
||||
message = "services.every-channel.ipxe-qemu.lan.macvtap.interface must be set when lan.enable is true";
|
||||
}
|
||||
{
|
||||
assertion = (!cfg.boot.http.enable) || (cfg.boot.http.root != null);
|
||||
message = "services.every-channel.ipxe-qemu.boot.http.root must be set when boot.http.enable is true";
|
||||
}
|
||||
];
|
||||
|
||||
environment.systemPackages = [ runner ];
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${cfg.stateDir} 0755 root root -"
|
||||
"d ${cfg.boot.userNet.tftpDir} 0755 root root -"
|
||||
];
|
||||
|
||||
systemd.services.every-channel-ipxe-qemu = {
|
||||
description = "every.channel iPXE QEMU VM";
|
||||
after = [ "local-fs.target" "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
pkgs.coreutils
|
||||
pkgs.iproute2
|
||||
cfg.package
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = "${runner}/bin/every-channel-ipxe-qemu";
|
||||
Restart = "always";
|
||||
RestartSec = 5;
|
||||
WorkingDirectory = cfg.stateDir;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
338
nix/modules/ec-netboot.nix
Normal file
338
nix/modules/ec-netboot.nix
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
{ lib, config, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.services.every-channel.netboot;
|
||||
scriptsRoot = ../..;
|
||||
|
||||
boolString = v: if v then "true" else "false";
|
||||
|
||||
mkExport = name: value: "export ${name}=${lib.escapeShellArg value}";
|
||||
|
||||
optionalExport = name: value:
|
||||
if value == null then "" else mkExport name value;
|
||||
|
||||
optionalFileExport = name: path:
|
||||
if path == null then ""
|
||||
else ''
|
||||
if [[ ! -r ${lib.escapeShellArg path} ]]; then
|
||||
echo "error: required file is not readable: ${path}" >&2
|
||||
exit 2
|
||||
fi
|
||||
export ${name}="$(tr -d '\\r\\n' < ${lib.escapeShellArg path})"
|
||||
'';
|
||||
|
||||
stageToolchain = with pkgs; [
|
||||
bash
|
||||
coreutils
|
||||
curl
|
||||
gawk
|
||||
gnugrep
|
||||
gnused
|
||||
gnutar
|
||||
gzip
|
||||
python3
|
||||
];
|
||||
|
||||
ipxeToolchain = with pkgs; [
|
||||
git
|
||||
gnumake
|
||||
gcc
|
||||
binutils
|
||||
perl
|
||||
mtools
|
||||
docker-client
|
||||
];
|
||||
in
|
||||
{
|
||||
options.services.every-channel.netboot = {
|
||||
enable = lib.mkEnableOption "every.channel persistent netboot stage/serve services";
|
||||
|
||||
listenIP = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "10.20.30.2";
|
||||
description = "IP address bound for TFTP/HTTP on the provisioning VLAN.";
|
||||
};
|
||||
|
||||
interface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "enp3s0";
|
||||
description = "Network interface name on the provisioning VLAN.";
|
||||
};
|
||||
|
||||
hostname = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "boot.every.channel";
|
||||
description = "Boot server hostname advertised to DHCP/iPXE clients.";
|
||||
};
|
||||
|
||||
rootDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/var/lib/every-channel-netboot";
|
||||
description = "Persistent root directory containing staged HTTP and TFTP artifacts.";
|
||||
};
|
||||
|
||||
httpPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8080;
|
||||
description = "HTTP port used for kernel/initrd/netboot script serving.";
|
||||
};
|
||||
|
||||
tftpBootFilename = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "ec-ipxe.efi";
|
||||
description = "Filename served via TFTP (DHCP option 67 in UniFi-only mode).";
|
||||
};
|
||||
|
||||
chainTokenFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/run/agenix/every-channel-netboot-chain-token";
|
||||
description = "Optional file containing netboot chain token passed to stage/serve scripts.";
|
||||
};
|
||||
|
||||
httpAllowedCIDRs = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
example = [ "10.20.30.0/24" ];
|
||||
description = "Optional CIDR allowlist for HTTP artifact serving.";
|
||||
};
|
||||
|
||||
openFirewall = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Open firewall ports for TFTP/HTTP (and ProxyDHCP ports when enabled).";
|
||||
};
|
||||
|
||||
stageOnBoot = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Start the stage oneshot service during boot before serving.";
|
||||
};
|
||||
|
||||
release = {
|
||||
host = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "https://git.every.channel";
|
||||
description = "Forge host used to fetch netboot release assets.";
|
||||
};
|
||||
|
||||
repo = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "every-channel/every.channel";
|
||||
description = "Forge repository containing netboot release assets.";
|
||||
};
|
||||
|
||||
tag = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional release tag to stage; defaults to latest release.";
|
||||
};
|
||||
|
||||
localTarball = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional local netboot tarball path; bypasses release API download when set.";
|
||||
};
|
||||
|
||||
tokenFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional file containing Forge API token for private release access.";
|
||||
};
|
||||
|
||||
verifyChecksums = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Verify staged release tarball using SHA256SUMS.txt when available.";
|
||||
};
|
||||
};
|
||||
|
||||
ipxe = {
|
||||
buildEmbedded = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Build embedded iPXE (ec-ipxe.efi) before staging artifacts.";
|
||||
};
|
||||
|
||||
path = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional prebuilt iPXE EFI binary path; used when buildEmbedded is false.";
|
||||
};
|
||||
|
||||
allowRemoteDownload = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Allow remote iPXE URL download during staging. Disabled by default.";
|
||||
};
|
||||
|
||||
sha256 = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional SHA256 digest expected for selected iPXE EFI binary.";
|
||||
};
|
||||
|
||||
repo = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "https://github.com/ipxe/ipxe.git";
|
||||
description = "iPXE source repository used for embedded EFI builds.";
|
||||
};
|
||||
|
||||
ref = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional git ref/commit for iPXE build reproducibility.";
|
||||
};
|
||||
|
||||
useDocker = lib.mkOption {
|
||||
type = lib.types.enum [ "auto" "true" "false" ];
|
||||
default = "auto";
|
||||
description = "Container fallback mode for iPXE builds (auto/true/false).";
|
||||
};
|
||||
|
||||
dockerImage = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "ubuntu:24.04";
|
||||
description = "Docker image used when iPXE build runs in container mode.";
|
||||
};
|
||||
|
||||
dockerPlatform = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "linux/amd64";
|
||||
description = "Docker platform used for containerized iPXE build.";
|
||||
};
|
||||
};
|
||||
|
||||
proxyDhcp = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Enable dnsmasq ProxyDHCP chainloading mode.";
|
||||
};
|
||||
|
||||
subnet = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "10.20.30.0/24";
|
||||
description = "ProxyDHCP subnet (required when proxyDhcp.enable is true).";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.proxyDhcp.enable -> cfg.proxyDhcp.subnet != null;
|
||||
message = "services.every-channel.netboot.proxyDhcp.subnet must be set when proxyDhcp.enable is true";
|
||||
}
|
||||
{
|
||||
assertion = cfg.ipxe.buildEmbedded || cfg.ipxe.path != null || cfg.ipxe.allowRemoteDownload;
|
||||
message = "Set ipxe.buildEmbedded=true, or provide ipxe.path, or allow ipxe.allowRemoteDownload=true";
|
||||
}
|
||||
];
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${cfg.rootDir} 0750 root root -"
|
||||
"d ${cfg.rootDir}/http 0750 root root -"
|
||||
"d ${cfg.rootDir}/tftp 0750 root root -"
|
||||
];
|
||||
|
||||
systemd.services.every-channel-netboot-ipxe = lib.mkIf cfg.ipxe.buildEmbedded {
|
||||
description = "every.channel netboot embedded iPXE build";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
wantedBy = lib.mkIf cfg.stageOnBoot [ "multi-user.target" ];
|
||||
path = stageToolchain ++ ipxeToolchain;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_ROOT" cfg.rootDir}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_HOSTNAME" cfg.hostname}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_HTTP_PORT" (toString cfg.httpPort)}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_IPXE_FILENAME" cfg.tftpBootFilename}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_IPXE_REPO" cfg.ipxe.repo}
|
||||
${optionalExport "EVERY_CHANNEL_NETBOOT_IPXE_REF" cfg.ipxe.ref}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_IPXE_USE_DOCKER" cfg.ipxe.useDocker}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_IPXE_DOCKER_IMAGE" cfg.ipxe.dockerImage}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_IPXE_DOCKER_PLATFORM" cfg.ipxe.dockerPlatform}
|
||||
${optionalFileExport "EVERY_CHANNEL_NETBOOT_CHAIN_TOKEN" cfg.chainTokenFile}
|
||||
|
||||
${scriptsRoot}/scripts/netboot-build-ipxe.sh
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.services.every-channel-netboot-stage = {
|
||||
description = "every.channel netboot artifact stage";
|
||||
requires = lib.optionals cfg.ipxe.buildEmbedded [ "every-channel-netboot-ipxe.service" ];
|
||||
after = [ "network-online.target" ] ++ lib.optionals cfg.ipxe.buildEmbedded [ "every-channel-netboot-ipxe.service" ];
|
||||
wants = [ "network-online.target" ] ++ lib.optionals cfg.ipxe.buildEmbedded [ "every-channel-netboot-ipxe.service" ];
|
||||
wantedBy = lib.mkIf cfg.stageOnBoot [ "multi-user.target" ];
|
||||
path = stageToolchain;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_ROOT" cfg.rootDir}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_HOSTNAME" cfg.hostname}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_HTTP_PORT" (toString cfg.httpPort)}
|
||||
${mkExport "EVERY_CHANNEL_FORGE_HOST" cfg.release.host}
|
||||
${mkExport "EVERY_CHANNEL_FORGE_REPO" cfg.release.repo}
|
||||
${optionalExport "EVERY_CHANNEL_NETBOOT_RELEASE_TAG" cfg.release.tag}
|
||||
${optionalExport "EVERY_CHANNEL_NETBOOT_TARBALL" cfg.release.localTarball}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_VERIFY_RELEASE_CHECKSUMS" (boolString cfg.release.verifyChecksums)}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_ALLOW_REMOTE_IPXE" (boolString cfg.ipxe.allowRemoteDownload)}
|
||||
${mkExport "EVERY_CHANNEL_IPXE_EFI_FILENAME" cfg.tftpBootFilename}
|
||||
${optionalExport "EVERY_CHANNEL_IPXE_EFI_SHA256" cfg.ipxe.sha256}
|
||||
${optionalFileExport "EVERY_CHANNEL_FORGE_TOKEN" cfg.release.tokenFile}
|
||||
${optionalFileExport "EVERY_CHANNEL_NETBOOT_CHAIN_TOKEN" cfg.chainTokenFile}
|
||||
|
||||
${lib.optionalString cfg.ipxe.buildEmbedded (mkExport "EVERY_CHANNEL_IPXE_EFI_PATH" "${cfg.rootDir}/tftp/${cfg.tftpBootFilename}")}
|
||||
${lib.optionalString (!cfg.ipxe.buildEmbedded && cfg.ipxe.path != null) (mkExport "EVERY_CHANNEL_IPXE_EFI_PATH" cfg.ipxe.path)}
|
||||
|
||||
${scriptsRoot}/scripts/netboot-stage.sh
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.services.every-channel-netboot = {
|
||||
description = "every.channel netboot HTTP + TFTP service";
|
||||
requires = [ "every-channel-netboot-stage.service" ];
|
||||
after = [ "network-online.target" "every-channel-netboot-stage.service" ];
|
||||
wants = [ "network-online.target" "every-channel-netboot-stage.service" ];
|
||||
wantedBy = lib.mkIf cfg.stageOnBoot [ "multi-user.target" ];
|
||||
path = stageToolchain ++ [ pkgs.dnsmasq ];
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
Restart = "always";
|
||||
RestartSec = "5s";
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_ROOT" cfg.rootDir}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_LISTEN_IP" cfg.listenIP}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_INTERFACE" cfg.interface}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_HOSTNAME" cfg.hostname}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_HTTP_PORT" (toString cfg.httpPort)}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_PROXY_DHCP" (boolString cfg.proxyDhcp.enable)}
|
||||
${optionalExport "EVERY_CHANNEL_NETBOOT_PROXY_SUBNET" cfg.proxyDhcp.subnet}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_TFTP_BOOT_FILENAME" cfg.tftpBootFilename}
|
||||
${mkExport "EVERY_CHANNEL_NETBOOT_HTTP_ALLOWED_CIDRS" (lib.concatStringsSep "," cfg.httpAllowedCIDRs)}
|
||||
${optionalFileExport "EVERY_CHANNEL_NETBOOT_CHAIN_TOKEN" cfg.chainTokenFile}
|
||||
|
||||
${scriptsRoot}/scripts/netboot-serve.sh
|
||||
'';
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.httpPort ];
|
||||
networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall (
|
||||
[ 69 ] ++ lib.optionals cfg.proxyDhcp.enable [ 67 4011 ]
|
||||
);
|
||||
};
|
||||
}
|
||||
387
nix/modules/ec-op-stack.nix
Normal file
387
nix/modules/ec-op-stack.nix
Normal file
|
|
@ -0,0 +1,387 @@
|
|||
{ lib, config, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.services.every-channel.op-stack;
|
||||
scriptRoot = ../..;
|
||||
bootstrapScript = "${scriptRoot}/scripts/op-stack/setup-rollup.sh";
|
||||
downloadScript = "${scriptRoot}/scripts/op-stack/download-op-deployer.sh";
|
||||
|
||||
containerName = name: "every-channel-op-${name}";
|
||||
in
|
||||
{
|
||||
options.services.every-channel.op-stack = {
|
||||
enable = lib.mkEnableOption "every.channel OP Stack Sepolia testnet services";
|
||||
|
||||
rootDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/var/lib/every-channel/op-stack";
|
||||
description = "Persistent root directory for OP Stack bootstrap outputs and container state.";
|
||||
};
|
||||
|
||||
privateKeyFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "File containing the Sepolia private key used for op-deployer and operator services.";
|
||||
};
|
||||
|
||||
l1RpcUrl = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "https://ethereum-sepolia-rpc.publicnode.com";
|
||||
description = "Sepolia L1 RPC URL.";
|
||||
};
|
||||
|
||||
l1BeaconUrl = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "https://ethereum-sepolia-beacon-api.publicnode.com";
|
||||
description = "Sepolia L1 beacon API URL.";
|
||||
};
|
||||
|
||||
chainId = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 245245;
|
||||
description = "L2 chain ID for the every.channel OP Stack testnet.";
|
||||
};
|
||||
|
||||
p2pAdvertiseIp = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "127.0.0.1";
|
||||
description = "Public IP advertised by op-node for P2P.";
|
||||
};
|
||||
|
||||
p2pListenPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 9222;
|
||||
description = "P2P listen port for op-node.";
|
||||
};
|
||||
|
||||
openFirewall = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Open the op-node P2P TCP/UDP port.";
|
||||
};
|
||||
|
||||
disputeMonEnable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Run op-dispute-mon alongside the core OP Stack services.";
|
||||
};
|
||||
|
||||
challengerEnable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Run op-challenger for the rollup.";
|
||||
};
|
||||
|
||||
challengerPrestateFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Path to the Cannon absolute prestate .bin.gz file used by op-challenger.";
|
||||
};
|
||||
|
||||
opDeployerTag = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "op-deployer/v0.6.0-rc.3";
|
||||
description = "Pinned op-deployer release tag used for bootstrap.";
|
||||
};
|
||||
|
||||
images = {
|
||||
opNode = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:v1.13.5";
|
||||
description = "Container image for op-node.";
|
||||
};
|
||||
opGeth = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-geth:v1.101511.1";
|
||||
description = "Container image for op-geth.";
|
||||
};
|
||||
batcher = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-batcher:v1.14.0";
|
||||
description = "Container image for op-batcher.";
|
||||
};
|
||||
proposer = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-proposer:v1.10.0";
|
||||
description = "Container image for op-proposer.";
|
||||
};
|
||||
challenger = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-challenger:v1.5.1";
|
||||
description = "Container image for op-challenger.";
|
||||
};
|
||||
disputeMon = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-dispute-mon:v1.4.2-rc.1";
|
||||
description = "Container image for op-dispute-mon.";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.privateKeyFile != null;
|
||||
message = "services.every-channel.op-stack.privateKeyFile must be set when the OP Stack is enabled";
|
||||
}
|
||||
{
|
||||
assertion = builtins.pathExists bootstrapScript;
|
||||
message = "missing bootstrap script at scripts/op-stack/setup-rollup.sh";
|
||||
}
|
||||
{
|
||||
assertion = builtins.pathExists downloadScript;
|
||||
message = "missing download helper at scripts/op-stack/download-op-deployer.sh";
|
||||
}
|
||||
{
|
||||
assertion = (!cfg.challengerEnable) || cfg.challengerPrestateFile != null;
|
||||
message = "services.every-channel.op-stack.challengerPrestateFile must be set when challengerEnable = true";
|
||||
}
|
||||
{
|
||||
assertion = (!cfg.disputeMonEnable) || cfg.challengerEnable;
|
||||
message = "services.every-channel.op-stack.disputeMonEnable requires challengerEnable = true";
|
||||
}
|
||||
];
|
||||
|
||||
networking.firewall = lib.mkIf cfg.openFirewall {
|
||||
allowedTCPPorts = [ cfg.p2pListenPort ];
|
||||
allowedUDPPorts = [ cfg.p2pListenPort ];
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${cfg.rootDir} 0750 root root - -"
|
||||
"d ${cfg.rootDir}/bin 0750 root root - -"
|
||||
"d ${cfg.rootDir}/deployer 0750 root root - -"
|
||||
"d ${cfg.rootDir}/sequencer 0750 root root - -"
|
||||
"d ${cfg.rootDir}/batcher 0750 root root - -"
|
||||
"d ${cfg.rootDir}/proposer 0750 root root - -"
|
||||
"d ${cfg.rootDir}/challenger 0750 root root - -"
|
||||
"d ${cfg.rootDir}/challenger/data 0750 root root - -"
|
||||
"d ${cfg.rootDir}/dispute-mon 0750 root root - -"
|
||||
"d ${cfg.rootDir}/op-geth-data 0750 root root - -"
|
||||
];
|
||||
|
||||
virtualisation.oci-containers.containers = {
|
||||
"${containerName "geth"}" = {
|
||||
image = cfg.images.opGeth;
|
||||
autoStart = true;
|
||||
volumes = [
|
||||
"${cfg.rootDir}/sequencer:/workspace"
|
||||
"${cfg.rootDir}/op-geth-data:/workspace/op-geth-data"
|
||||
];
|
||||
extraOptions = [ "--network=host" ];
|
||||
entrypoint = "/bin/sh";
|
||||
cmd = [
|
||||
"-lc"
|
||||
''
|
||||
set -e
|
||||
if [ ! -d /workspace/op-geth-data/geth/chaindata ]; then
|
||||
geth init --datadir=/workspace/op-geth-data --state.scheme=hash /workspace/genesis.json
|
||||
fi
|
||||
exec geth --datadir=/workspace/op-geth-data --http --http.addr=127.0.0.1 --http.port=8545 --ws --ws.addr=127.0.0.1 --ws.port=8546 --authrpc.addr=127.0.0.1 --authrpc.port=8551 --authrpc.jwtsecret=/workspace/jwt.txt --syncmode=full --gcmode=archive --rollup.disabletxpoolgossip=true --http.vhosts=* --http.corsdomain=* --http.api=eth,net,web3,debug,txpool,admin --ws.origins=* --ws.api=eth,net,web3,debug,txpool,admin --authrpc.vhosts=*
|
||||
''
|
||||
];
|
||||
};
|
||||
|
||||
"${containerName "node"}" = {
|
||||
image = cfg.images.opNode;
|
||||
autoStart = true;
|
||||
volumes = [
|
||||
"${cfg.rootDir}/sequencer:/workspace"
|
||||
"${cfg.rootDir}/op-geth-data:/workspace/op-geth-data"
|
||||
];
|
||||
environmentFiles = [ "${cfg.rootDir}/sequencer/.env" ];
|
||||
extraOptions = [ "--network=host" ];
|
||||
entrypoint = "/bin/sh";
|
||||
cmd = [
|
||||
"-lc"
|
||||
''
|
||||
exec op-node \
|
||||
--l1="$L1_RPC_URL" \
|
||||
--l1.beacon="$L1_BEACON_URL" \
|
||||
--l2=http://127.0.0.1:8551 \
|
||||
--l2.jwt-secret=/workspace/jwt.txt \
|
||||
--rollup.config=/workspace/rollup.json \
|
||||
--sequencer.enabled=true \
|
||||
--sequencer.stopped=false \
|
||||
--sequencer.max-safe-lag=3600 \
|
||||
--verifier.l1-confs=4 \
|
||||
--p2p.listen.ip=0.0.0.0 \
|
||||
--p2p.listen.tcp=${toString cfg.p2pListenPort} \
|
||||
--p2p.listen.udp=${toString cfg.p2pListenPort} \
|
||||
--p2p.advertise.ip="$P2P_ADVERTISE_IP" \
|
||||
--p2p.advertise.tcp=${toString cfg.p2pListenPort} \
|
||||
--p2p.advertise.udp=${toString cfg.p2pListenPort} \
|
||||
--p2p.sequencer.key="$PRIVATE_KEY" \
|
||||
--rpc.addr=127.0.0.1 \
|
||||
--rpc.port=8547 \
|
||||
--rpc.enable-admin \
|
||||
--log.level=info \
|
||||
--log.format=json
|
||||
''
|
||||
];
|
||||
};
|
||||
|
||||
"${containerName "batcher"}" = {
|
||||
image = cfg.images.batcher;
|
||||
autoStart = true;
|
||||
volumes = [ "${cfg.rootDir}/batcher:/workspace" ];
|
||||
environmentFiles = [ "${cfg.rootDir}/batcher/.env" ];
|
||||
extraOptions = [ "--network=host" ];
|
||||
entrypoint = "/bin/sh";
|
||||
cmd = [
|
||||
"-lc"
|
||||
''
|
||||
exec op-batcher \
|
||||
--l1-eth-rpc="$L1_RPC_URL" \
|
||||
--l2-eth-rpc="$L2_RPC_URL" \
|
||||
--rollup-rpc="$ROLLUP_RPC_URL" \
|
||||
--private-key="$PRIVATE_KEY" \
|
||||
--batch-inbox-address="$BATCH_INBOX_ADDRESS" \
|
||||
--rpc.addr=127.0.0.1 \
|
||||
--rpc.port=8548 \
|
||||
--rpc.enable-admin \
|
||||
--max-channel-duration=1 \
|
||||
--data-availability-type=calldata \
|
||||
--resubmission-timeout=30s \
|
||||
--log.level=info \
|
||||
--log.format=json
|
||||
''
|
||||
];
|
||||
};
|
||||
|
||||
"${containerName "proposer"}" = {
|
||||
image = cfg.images.proposer;
|
||||
autoStart = true;
|
||||
volumes = [ "${cfg.rootDir}/proposer:/workspace" ];
|
||||
environmentFiles = [ "${cfg.rootDir}/proposer/.env" ];
|
||||
extraOptions = [ "--network=host" ];
|
||||
entrypoint = "/bin/sh";
|
||||
cmd = [
|
||||
"-lc"
|
||||
''
|
||||
exec op-proposer \
|
||||
--rpc.port=8560 \
|
||||
--rollup-rpc="$ROLLUP_RPC_URL" \
|
||||
--l1-eth-rpc="$L1_RPC_URL" \
|
||||
--private-key="$PRIVATE_KEY" \
|
||||
--game-factory-address="$GAME_FACTORY_ADDRESS" \
|
||||
--proposal-interval="$PROPOSAL_INTERVAL" \
|
||||
--allow-non-finalized=true \
|
||||
--wait-node-sync=true \
|
||||
--log.level=info \
|
||||
--log.format=json
|
||||
''
|
||||
];
|
||||
};
|
||||
} // lib.optionalAttrs cfg.challengerEnable {
|
||||
"${containerName "challenger"}" = {
|
||||
image = cfg.images.challenger;
|
||||
autoStart = true;
|
||||
volumes = [ "${cfg.rootDir}/challenger:/workspace" ];
|
||||
environmentFiles = [ "${cfg.rootDir}/challenger/.env" ];
|
||||
extraOptions = [ "--network=host" ];
|
||||
entrypoint = "/bin/sh";
|
||||
cmd = [
|
||||
"-lc"
|
||||
''
|
||||
exec op-challenger run-trace \
|
||||
--trace-type=cannon \
|
||||
--l1-eth-rpc="$L1_RPC_URL" \
|
||||
--l1-beacon="$L1_BEACON_URL" \
|
||||
--private-key="$PRIVATE_KEY" \
|
||||
--game-factory-address="$GAME_FACTORY_ADDRESS" \
|
||||
--cannon-l2-genesis=/workspace/genesis.json \
|
||||
--cannon-rollup-config=/workspace/rollup.json \
|
||||
--cannon-prestate="$CANNON_PRESTATE" \
|
||||
--l2-eth-rpc="$L2_RPC_URL" \
|
||||
--rollup-rpc="$ROLLUP_RPC_URL" \
|
||||
--datadir=/workspace/data \
|
||||
--log.level=info \
|
||||
--log.format=json
|
||||
''
|
||||
];
|
||||
};
|
||||
} // lib.optionalAttrs cfg.disputeMonEnable {
|
||||
"${containerName "dispute-mon"}" = {
|
||||
image = cfg.images.disputeMon;
|
||||
autoStart = true;
|
||||
volumes = [ "${cfg.rootDir}/dispute-mon:/workspace" ];
|
||||
environmentFiles = [ "${cfg.rootDir}/dispute-mon/.env" ];
|
||||
extraOptions = [ "--network=host" ];
|
||||
entrypoint = "/bin/sh";
|
||||
cmd = [
|
||||
"-lc"
|
||||
''
|
||||
exec op-dispute-mon
|
||||
''
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services = {
|
||||
every-channel-op-stack-bootstrap = {
|
||||
description = "every.channel OP Stack bootstrap";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = with pkgs; [
|
||||
bash
|
||||
coreutils
|
||||
curl
|
||||
gnutar
|
||||
gzip
|
||||
jq
|
||||
openssl
|
||||
foundry
|
||||
python3
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
export EVERY_CHANNEL_OP_STACK_ROOT=${lib.escapeShellArg cfg.rootDir}
|
||||
export EVERY_CHANNEL_OP_STACK_PRIVATE_KEY_FILE=${lib.escapeShellArg cfg.privateKeyFile}
|
||||
export EVERY_CHANNEL_OP_STACK_L1_RPC_URL=${lib.escapeShellArg cfg.l1RpcUrl}
|
||||
export EVERY_CHANNEL_OP_STACK_L1_BEACON_URL=${lib.escapeShellArg cfg.l1BeaconUrl}
|
||||
export EVERY_CHANNEL_OP_STACK_CHAIN_ID=${toString cfg.chainId}
|
||||
export EVERY_CHANNEL_OP_STACK_P2P_ADVERTISE_IP=${lib.escapeShellArg cfg.p2pAdvertiseIp}
|
||||
export EVERY_CHANNEL_OP_DEPLOYER_BIN=${lib.escapeShellArg "${cfg.rootDir}/bin/op-deployer"}
|
||||
export EVERY_CHANNEL_OP_DEPLOYER_TAG=${lib.escapeShellArg cfg.opDeployerTag}
|
||||
export EVERY_CHANNEL_OP_DEPLOYER_DOWNLOAD_SCRIPT=${lib.escapeShellArg downloadScript}
|
||||
export EVERY_CHANNEL_OP_STACK_CHALLENGER_PRESTATE_FILE=${lib.escapeShellArg (if cfg.challengerPrestateFile == null then "" else cfg.challengerPrestateFile)}
|
||||
${lib.escapeShellArg bootstrapScript}
|
||||
'';
|
||||
};
|
||||
"podman-${containerName "geth"}" = {
|
||||
after = [ "every-channel-op-stack-bootstrap.service" ];
|
||||
wants = [ "every-channel-op-stack-bootstrap.service" ];
|
||||
requires = [ "every-channel-op-stack-bootstrap.service" ];
|
||||
};
|
||||
"podman-${containerName "node"}" = {
|
||||
after = [ "every-channel-op-stack-bootstrap.service" "podman-${containerName "geth"}.service" ];
|
||||
wants = [ "every-channel-op-stack-bootstrap.service" "podman-${containerName "geth"}.service" ];
|
||||
requires = [ "every-channel-op-stack-bootstrap.service" ];
|
||||
};
|
||||
"podman-${containerName "batcher"}" = {
|
||||
after = [ "podman-${containerName "node"}.service" ];
|
||||
wants = [ "podman-${containerName "node"}.service" ];
|
||||
};
|
||||
"podman-${containerName "proposer"}" = {
|
||||
after = [ "podman-${containerName "node"}.service" ];
|
||||
wants = [ "podman-${containerName "node"}.service" ];
|
||||
};
|
||||
} // lib.optionalAttrs cfg.challengerEnable {
|
||||
"podman-${containerName "challenger"}" = {
|
||||
after = [ "podman-${containerName "node"}.service" ];
|
||||
wants = [ "podman-${containerName "node"}.service" ];
|
||||
};
|
||||
} // lib.optionalAttrs cfg.disputeMonEnable {
|
||||
"podman-${containerName "dispute-mon"}" = {
|
||||
after = [ "podman-${containerName "node"}.service" ];
|
||||
wants = [ "podman-${containerName "node"}.service" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
23
nix/modules/ec-publisher-guest.nix
Normal file
23
nix/modules/ec-publisher-guest.nix
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
{ lib, pkgs, ... }:
|
||||
|
||||
{
|
||||
networking.hostName = lib.mkForce "ec-publisher";
|
||||
|
||||
services.every-channel.ec-node = {
|
||||
relayUrl = lib.mkDefault "https://cdn.moq.dev/anon";
|
||||
passthrough = lib.mkDefault false;
|
||||
|
||||
hdhomerun.autoDiscover = lib.mkDefault true;
|
||||
|
||||
control = {
|
||||
enable = lib.mkDefault true;
|
||||
discovery = lib.mkDefault "dht,mdns,dns";
|
||||
};
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
curl
|
||||
ffmpeg
|
||||
jq
|
||||
];
|
||||
}
|
||||
30
nix/nixos/ecp-forge-hardware.nix
Normal file
30
nix/nixos/ecp-forge-hardware.nix
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
{ lib, ... }:
|
||||
|
||||
{
|
||||
boot.initrd.availableKernelModules = [
|
||||
"ahci"
|
||||
"xhci_pci"
|
||||
"nvme"
|
||||
"sd_mod"
|
||||
"sr_mod"
|
||||
];
|
||||
boot.initrd.kernelModules = [ ];
|
||||
boot.kernelModules = [ "kvm-intel" ];
|
||||
boot.extraModulePackages = [ ];
|
||||
|
||||
fileSystems."/" = {
|
||||
device = "/dev/disk/by-label/nixos";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
fileSystems."/boot" = {
|
||||
device = "/dev/disk/by-label/boot";
|
||||
fsType = "vfat";
|
||||
options = [ "fmask=0077" "dmask=0077" ];
|
||||
};
|
||||
|
||||
swapDevices = [ ];
|
||||
|
||||
networking.useDHCP = lib.mkDefault true;
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
}
|
||||
288
nix/nixos/ecp-forge.nix
Normal file
288
nix/nixos/ecp-forge.nix
Normal file
|
|
@ -0,0 +1,288 @@
|
|||
{ config, pkgs, lib, ... }:
|
||||
|
||||
let
|
||||
hasForgejoApiToken = builtins.pathExists ../../secrets/forgejo-api-token.age;
|
||||
hasNetbootChainToken = builtins.pathExists ../../secrets/netboot-chain-token.age;
|
||||
hasOpStackSepoliaKey = builtins.pathExists ../../secrets/op-stack-sepolia-private-key.age;
|
||||
hasOpStackChallengerPrestate = builtins.pathExists ../../secrets/op-stack-challenger-prestate.bin.gz.age;
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
./ecp-forge-hardware.nix
|
||||
];
|
||||
|
||||
networking = {
|
||||
hostName = "ecp-forge";
|
||||
hostId = "007f0200";
|
||||
useDHCP = lib.mkForce true;
|
||||
networkmanager.enable = lib.mkForce false;
|
||||
nameservers = [ "1.1.1.1" "8.8.8.8" ];
|
||||
firewall = {
|
||||
trustedInterfaces = [ "tailscale0" ];
|
||||
allowedTCPPorts = [
|
||||
80
|
||||
443
|
||||
2222
|
||||
69
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
67
|
||||
69
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
services.openssh = {
|
||||
enable = true;
|
||||
startWhenNeeded = true;
|
||||
openFirewall = true;
|
||||
settings = {
|
||||
PasswordAuthentication = false;
|
||||
PermitRootLogin = lib.mkForce "prohibit-password";
|
||||
KbdInteractiveAuthentication = false;
|
||||
};
|
||||
};
|
||||
|
||||
users.users.root.openssh.authorizedKeys.keys = [
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBueQxNbP2246pxr/m7au4zNVm+ShC96xuOcfEcpIjWZ"
|
||||
];
|
||||
|
||||
security.sudo = {
|
||||
execWheelOnly = true;
|
||||
wheelNeedsPassword = false;
|
||||
};
|
||||
|
||||
users = {
|
||||
mutableUsers = false;
|
||||
defaultUserShell = pkgs.bash;
|
||||
users.conradev = {
|
||||
uid = 1000;
|
||||
isNormalUser = true;
|
||||
password = "password";
|
||||
group = "conradev";
|
||||
extraGroups = [ "wheel" "docker" "libvirtd" "kvm" ];
|
||||
openssh.authorizedKeys.keys = [
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBueQxNbP2246pxr/m7au4zNVm+ShC96xuOcfEcpIjWZ"
|
||||
];
|
||||
};
|
||||
groups.conradev = { };
|
||||
};
|
||||
|
||||
boot.loader = {
|
||||
grub = {
|
||||
enable = true;
|
||||
device = "/dev/disk/by-id/nvme-KIOXIA_KCD81RUG7T68_25R0A0KZTTEJ";
|
||||
};
|
||||
efi.canTouchEfiVariables = false;
|
||||
};
|
||||
|
||||
boot.supportedFilesystems = [ "zfs" ];
|
||||
boot.zfs.extraPools = [ "tank" ];
|
||||
boot.kernel.sysctl = {
|
||||
"fs.inotify.max_user_watches" = "204800";
|
||||
};
|
||||
|
||||
hardware.nvidia-container-toolkit.enable = false;
|
||||
|
||||
virtualisation = {
|
||||
containers.enable = true;
|
||||
oci-containers.backend = "podman";
|
||||
podman = {
|
||||
enable = true;
|
||||
dockerCompat = true;
|
||||
autoPrune.enable = true;
|
||||
defaultNetwork.settings.dns_enabled = true;
|
||||
};
|
||||
};
|
||||
|
||||
services.tailscale = {
|
||||
enable = true;
|
||||
extraUpFlags = "--accept-routes";
|
||||
};
|
||||
|
||||
age.identityPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||||
age.secrets = lib.mkMerge [
|
||||
(lib.mkIf hasForgejoApiToken {
|
||||
"forgejo-api-token" = {
|
||||
file = ../../secrets/forgejo-api-token.age;
|
||||
owner = "root";
|
||||
group = "root";
|
||||
mode = "0400";
|
||||
};
|
||||
})
|
||||
(lib.mkIf hasNetbootChainToken {
|
||||
"every-channel-netboot-chain-token" = {
|
||||
file = ../../secrets/netboot-chain-token.age;
|
||||
owner = "root";
|
||||
group = "root";
|
||||
mode = "0400";
|
||||
};
|
||||
})
|
||||
(lib.mkIf hasOpStackSepoliaKey {
|
||||
"every-channel-op-stack-sepolia-private-key" = {
|
||||
file = ../../secrets/op-stack-sepolia-private-key.age;
|
||||
owner = "root";
|
||||
group = "root";
|
||||
mode = "0400";
|
||||
};
|
||||
})
|
||||
(lib.mkIf hasOpStackChallengerPrestate {
|
||||
"every-channel-op-stack-challenger-prestate" = {
|
||||
file = ../../secrets/op-stack-challenger-prestate.bin.gz.age;
|
||||
owner = "root";
|
||||
group = "root";
|
||||
mode = "0400";
|
||||
};
|
||||
})
|
||||
];
|
||||
|
||||
services.zfs.autoScrub = {
|
||||
enable = true;
|
||||
pools = [ "tank" ];
|
||||
interval = "weekly";
|
||||
};
|
||||
|
||||
services.nfs.server = {
|
||||
enable = true;
|
||||
# Keep NFS on trusted/private paths only; public Hetzner exposure triggers
|
||||
# rpcbind/portmapper enumeration and is not needed for forge access.
|
||||
exports = ''
|
||||
/tank 10.0.0.0/8(rw,fsid=0,crossmnt,no_subtree_check,sync) 100.64.0.0/10(rw,fsid=0,crossmnt,no_subtree_check,sync) 192.168.0.0/16(rw,fsid=0,crossmnt,no_subtree_check,sync)
|
||||
'';
|
||||
};
|
||||
|
||||
nix.gc = {
|
||||
automatic = true;
|
||||
dates = "weekly";
|
||||
options = "--delete-older-than 30d";
|
||||
};
|
||||
|
||||
nix.settings = {
|
||||
experimental-features = [ "nix-command" "flakes" ];
|
||||
substituters = lib.mkForce [ "https://cache.nixos.org" ];
|
||||
trusted-substituters = lib.mkForce [ "https://cache.nixos.org" ];
|
||||
extra-substituters = lib.mkForce [ ];
|
||||
trusted-public-keys = lib.mkForce [
|
||||
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
|
||||
];
|
||||
extra-trusted-public-keys = lib.mkForce [ ];
|
||||
};
|
||||
|
||||
time.timeZone = "America/New_York";
|
||||
i18n.defaultLocale = "en_US.UTF-8";
|
||||
xdg.mime.enable = true;
|
||||
|
||||
services.forgejo = {
|
||||
enable = true;
|
||||
database.type = "sqlite3";
|
||||
lfs.enable = true;
|
||||
settings = {
|
||||
server = {
|
||||
DOMAIN = "git.every.channel";
|
||||
ROOT_URL = "https://git.every.channel/";
|
||||
HTTP_ADDR = "127.0.0.1";
|
||||
HTTP_PORT = 3000;
|
||||
SSH_DOMAIN = "git.every.channel";
|
||||
SSH_PORT = 2222;
|
||||
SSH_LISTEN_PORT = 2222;
|
||||
START_SSH_SERVER = true;
|
||||
};
|
||||
service = {
|
||||
DISABLE_REGISTRATION = true;
|
||||
REQUIRE_SIGNIN_VIEW = false;
|
||||
};
|
||||
repository = {
|
||||
DEFAULT_BRANCH = "main";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.caddy = {
|
||||
enable = true;
|
||||
email = "infra@every.channel";
|
||||
acmeCA = "https://acme-v02.api.letsencrypt.org/directory";
|
||||
virtualHosts = {
|
||||
"git.every.channel".extraConfig = ''
|
||||
reverse_proxy http://127.0.0.1:3000
|
||||
'';
|
||||
"forge.every.channel".extraConfig = ''
|
||||
redir https://git.every.channel{uri} permanent
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
services.every-channel.netboot = {
|
||||
enable = true;
|
||||
listenIP = "95.216.114.54";
|
||||
interface = "enp5s0f3u2u2c2";
|
||||
hostname = "boot.every.channel";
|
||||
httpPort = 8080;
|
||||
tftpBootFilename = "ec-ipxe.efi";
|
||||
httpAllowedCIDRs = [
|
||||
"10.0.0.0/8"
|
||||
"172.16.0.0/12"
|
||||
"192.168.0.0/16"
|
||||
"100.64.0.0/10"
|
||||
];
|
||||
chainTokenFile =
|
||||
if hasNetbootChainToken
|
||||
then config.age.secrets."every-channel-netboot-chain-token".path
|
||||
else null;
|
||||
proxyDhcp.enable = false;
|
||||
release.host = "https://git.every.channel";
|
||||
release.repo = "every-channel/every.channel";
|
||||
release.localTarball = "/var/lib/every-channel-netboot/sources/ec-runner-x86_64-netboot-local.tar.gz";
|
||||
release.tokenFile =
|
||||
if hasForgejoApiToken
|
||||
then config.age.secrets."forgejo-api-token".path
|
||||
else null;
|
||||
};
|
||||
|
||||
services.every-channel.ipxe-qemu = {
|
||||
enable = true;
|
||||
name = "ecp-forge-ipxe";
|
||||
boot.userNet.hostname = "ecp-forge-ipxe";
|
||||
boot.http.root = "${config.services.every-channel.netboot.rootDir}/http";
|
||||
};
|
||||
|
||||
systemd.services.every-channel-ipxe-qemu = {
|
||||
after = [ "every-channel-netboot-stage.service" ];
|
||||
wants = [ "every-channel-netboot-stage.service" ];
|
||||
};
|
||||
|
||||
services.every-channel.ec-node = {
|
||||
enable = true;
|
||||
archive = {
|
||||
enable = true;
|
||||
outputDir = "/tank/every-channel/archive";
|
||||
manifestDir = "/var/lib/every-channel/manifests";
|
||||
# Keep forge archival as an ingest worker only; replay serving is separate.
|
||||
serve.enable = false;
|
||||
};
|
||||
};
|
||||
|
||||
services.every-channel.op-stack = {
|
||||
enable = hasOpStackSepoliaKey;
|
||||
challengerEnable = hasOpStackChallengerPrestate;
|
||||
disputeMonEnable = hasOpStackChallengerPrestate;
|
||||
privateKeyFile =
|
||||
if hasOpStackSepoliaKey
|
||||
then config.age.secrets."every-channel-op-stack-sepolia-private-key".path
|
||||
else null;
|
||||
challengerPrestateFile =
|
||||
if hasOpStackChallengerPrestate
|
||||
then config.age.secrets."every-channel-op-stack-challenger-prestate".path
|
||||
else null;
|
||||
p2pAdvertiseIp = "95.216.114.54";
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
git
|
||||
htop
|
||||
jq
|
||||
tmux
|
||||
zfs
|
||||
];
|
||||
|
||||
system.stateVersion = "22.11";
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue