{ lib, config, pkgs, ... }: let cfg = config.services.every-channel.ethereum; mkNetworkSubmodule = name: defaults: { ... }: { options = { enable = lib.mkOption { type = lib.types.bool; default = true; description = "Whether to run the ${name} Ethereum execution and consensus pair."; }; rootDir = lib.mkOption { type = lib.types.str; default = "${cfg.rootDir}/${name}"; description = "Persistent root directory for the ${name} node state."; }; reth = { httpPort = lib.mkOption { type = lib.types.port; default = defaults.rethHttpPort; description = "Local HTTP JSON-RPC port for the ${name} Reth node."; }; wsPort = lib.mkOption { type = lib.types.port; default = defaults.rethWsPort; description = "Local WebSocket JSON-RPC port for the ${name} Reth node."; }; authPort = lib.mkOption { type = lib.types.port; default = defaults.rethAuthPort; description = "Local Engine API port for the ${name} Reth node."; }; p2pPort = lib.mkOption { type = lib.types.port; default = defaults.rethP2pPort; description = "RLPx/P2P TCP port for the ${name} Reth node."; }; discoveryPort = lib.mkOption { type = lib.types.port; default = defaults.rethDiscoveryPort; description = "Discovery UDP port for the ${name} Reth node."; }; metricsPort = lib.mkOption { type = lib.types.port; default = defaults.rethMetricsPort; description = "Prometheus port for the ${name} Reth node."; }; }; lighthouse = { httpPort = lib.mkOption { type = lib.types.port; default = defaults.lighthouseHttpPort; description = "Local Beacon API port for the ${name} Lighthouse node."; }; p2pPort = lib.mkOption { type = lib.types.port; default = defaults.lighthouseP2pPort; description = "TCP libp2p port for the ${name} Lighthouse node."; }; discoveryPort = lib.mkOption { type = lib.types.port; default = defaults.lighthouseDiscoveryPort; description = "UDP discovery port for the ${name} Lighthouse node."; }; quicPort = lib.mkOption { type = lib.types.port; default = defaults.lighthouseQuicPort; description = "UDP QUIC port for the ${name} Lighthouse node."; }; metricsPort = lib.mkOption { type = lib.types.port; default = defaults.lighthouseMetricsPort; description = "Prometheus port for the ${name} Lighthouse node."; }; }; }; }; networks = { mainnet = cfg.mainnet; sepolia = cfg.sepolia; }; enabledNetworks = lib.filterAttrs (_: networkCfg: networkCfg.enable) networks; rethContainerName = network: "every-channel-ethereum-${network}-reth"; lighthouseContainerName = network: "every-channel-ethereum-${network}-lighthouse"; networkDatasetLines = lib.concatStringsSep "\n" ( lib.mapAttrsToList (network: networkCfg: '' ensure_dataset ${lib.escapeShellArg "${cfg.poolName}/${network}"} ensure_dataset ${lib.escapeShellArg "${cfg.poolName}/${network}/reth"} ensure_dataset ${lib.escapeShellArg "${cfg.poolName}/${network}/lighthouse"} ensure_jwt ${lib.escapeShellArg "${networkCfg.rootDir}/jwt.hex"} '') enabledNetworks ); mkNatArgs = lib.optionals (cfg.publicIp != null) [ "--nat" "extip:${cfg.publicIp}" ]; mkEnrArgs = lib.optionals (cfg.publicIp != null) [ "--enr-address" cfg.publicIp ]; mkRethContainer = network: networkCfg: { image = cfg.images.reth; autoStart = true; extraOptions = [ "--network=host" ]; volumes = [ "${networkCfg.rootDir}:/state" ]; cmd = [ "node" "--chain" network "--datadir" "/state/reth" "--full" "--http" "--http.addr" "127.0.0.1" "--http.port" (toString networkCfg.reth.httpPort) "--http.api" "eth,net,web3,rpc" "--ws" "--ws.addr" "127.0.0.1" "--ws.port" (toString networkCfg.reth.wsPort) "--ws.api" "eth,net,web3,rpc" "--authrpc.addr" "127.0.0.1" "--authrpc.port" (toString networkCfg.reth.authPort) "--authrpc.jwtsecret" "/state/jwt.hex" "--port" (toString networkCfg.reth.p2pPort) "--discovery.port" (toString networkCfg.reth.discoveryPort) "--metrics" "127.0.0.1:${toString networkCfg.reth.metricsPort}" "--log.stdout.format" "json" ] ++ mkNatArgs; }; mkLighthouseContainer = network: networkCfg: { image = cfg.images.lighthouse; autoStart = true; extraOptions = [ "--network=host" ]; volumes = [ "${networkCfg.rootDir}:/state" ]; entrypoint = "/usr/local/bin/lighthouse"; cmd = [ "beacon_node" "--network" network "--datadir" "/state/lighthouse" "--http" "--http-address" "127.0.0.1" "--http-port" (toString networkCfg.lighthouse.httpPort) "--execution-endpoint" "http://127.0.0.1:${toString networkCfg.reth.authPort}" "--execution-jwt" "/state/jwt.hex" "--allow-insecure-genesis-sync" "--port" (toString networkCfg.lighthouse.p2pPort) "--discovery-port" (toString networkCfg.lighthouse.discoveryPort) "--quic-port" (toString networkCfg.lighthouse.quicPort) "--metrics" "--metrics-address" "127.0.0.1" "--metrics-port" (toString networkCfg.lighthouse.metricsPort) ] ++ mkEnrArgs; }; caddyRootBody = '' every.channel ethereum nodes mainnet sync: /mainnet/sync mainnet finality: /mainnet/finality sepolia sync: /sepolia/sync sepolia finality: /sepolia/finality raw execution and beacon RPC remain local-only on ecp-forge for now. ''; in { options.services.every-channel.ethereum = { enable = lib.mkEnableOption "every.channel dual-network Ethereum full nodes"; poolName = lib.mkOption { type = lib.types.str; default = "eth"; description = "Dedicated ZFS pool name used for Ethereum node state."; }; poolDevice = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; description = "Block device used to create the dedicated Ethereum ZFS pool if it does not already exist."; }; rootDir = lib.mkOption { type = lib.types.str; default = "/eth"; description = "Mountpoint for the dedicated Ethereum ZFS pool."; }; publicIp = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; description = "Public IP to advertise in Ethereum P2P metadata."; }; publicHost = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; description = "Optional HTTPS host that publishes node sync and finality surfaces."; }; images = { reth = lib.mkOption { type = lib.types.str; default = "ghcr.io/paradigmxyz/reth:v1.9.3"; description = "Pinned Reth OCI image."; }; lighthouse = lib.mkOption { type = lib.types.str; default = "docker.io/sigp/lighthouse:v8.1.1"; description = "Pinned Lighthouse OCI image."; }; }; mainnet = lib.mkOption { type = lib.types.submodule (mkNetworkSubmodule "mainnet" { rethHttpPort = 8545; rethWsPort = 8546; rethAuthPort = 8551; rethP2pPort = 30303; rethDiscoveryPort = 30303; rethMetricsPort = 19001; lighthouseHttpPort = 5052; lighthouseP2pPort = 9000; lighthouseDiscoveryPort = 9000; lighthouseQuicPort = 9001; lighthouseMetricsPort = 5054; }); default = { }; description = "Mainnet Ethereum node configuration."; }; sepolia = lib.mkOption { type = lib.types.submodule (mkNetworkSubmodule "sepolia" { rethHttpPort = 18545; rethWsPort = 18546; rethAuthPort = 18551; rethP2pPort = 31303; rethDiscoveryPort = 31303; rethMetricsPort = 29001; lighthouseHttpPort = 15052; lighthouseP2pPort = 19000; lighthouseDiscoveryPort = 19000; lighthouseQuicPort = 19001; lighthouseMetricsPort = 15054; }); default = { }; description = "Sepolia Ethereum node configuration."; }; }; config = lib.mkIf cfg.enable { assertions = [ { assertion = cfg.poolDevice != null; message = "services.every-channel.ethereum.poolDevice must be set when the Ethereum node is enabled"; } { assertion = enabledNetworks != { }; message = "At least one Ethereum network must be enabled"; } ]; boot.zfs.extraPools = [ cfg.poolName ]; networking.firewall = { allowedTCPPorts = lib.flatten ( lib.mapAttrsToList (_: networkCfg: [ networkCfg.reth.p2pPort networkCfg.lighthouse.p2pPort ]) enabledNetworks ); allowedUDPPorts = lib.flatten ( lib.mapAttrsToList (_: networkCfg: [ networkCfg.reth.discoveryPort networkCfg.lighthouse.discoveryPort networkCfg.lighthouse.quicPort ]) enabledNetworks ); }; virtualisation.oci-containers.containers = (lib.mapAttrs' (network: networkCfg: lib.nameValuePair (rethContainerName network) (mkRethContainer network networkCfg)) enabledNetworks) // (lib.mapAttrs' (network: networkCfg: lib.nameValuePair (lighthouseContainerName network) (mkLighthouseContainer network networkCfg)) enabledNetworks); systemd.services = { every-channel-ethereum-storage = { description = "every.channel Ethereum NVMe ZFS pool and dataset bootstrap"; wantedBy = [ "multi-user.target" ]; after = [ "local-fs.target" "zfs.target" ]; wants = [ "zfs.target" ]; before = lib.flatten ( lib.mapAttrsToList (network: _: [ "podman-${rethContainerName network}.service" "podman-${lighthouseContainerName network}.service" ]) enabledNetworks ); path = with pkgs; [ coreutils openssl util-linux zfs ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; script = '' set -euo pipefail pool=${lib.escapeShellArg cfg.poolName} root_dir=${lib.escapeShellArg cfg.rootDir} device=${lib.escapeShellArg cfg.poolDevice} ensure_dataset() { local dataset="$1" if ! zfs list -H "$dataset" >/dev/null 2>&1; then zfs create -p "$dataset" fi zfs set atime=off compression=lz4 xattr=sa "$dataset" >/dev/null } ensure_jwt() { local path="$1" if [[ ! -s "$path" ]]; then umask 077 openssl rand -hex 32 | tr -d '\n' > "$path" printf '\n' >> "$path" fi chmod 0400 "$path" } if ! zpool list -H "$pool" >/dev/null 2>&1; then if [[ -z "$device" ]]; then echo "every-channel-ethereum-storage: missing poolDevice for pool $pool" >&2 exit 1 fi if [[ ! -b "$device" ]]; then echo "every-channel-ethereum-storage: device $device not present" >&2 exit 1 fi if blkid "$device" >/dev/null 2>&1; then echo "every-channel-ethereum-storage: device $device already has signatures; refusing to overwrite automatically" >&2 exit 1 fi zpool create -f \ -o ashift=12 \ -O mountpoint="$root_dir" \ -O atime=off \ -O compression=lz4 \ -O xattr=sa \ "$pool" "$device" else zfs set mountpoint="$root_dir" "$pool" >/dev/null fi ${networkDatasetLines} ''; }; } // (lib.mapAttrs' (network: networkCfg: lib.nameValuePair "podman-${rethContainerName network}" { after = [ "network-online.target" "every-channel-ethereum-storage.service" ]; wants = [ "network-online.target" "every-channel-ethereum-storage.service" ]; requires = [ "every-channel-ethereum-storage.service" ]; unitConfig.RequiresMountsFor = [ networkCfg.rootDir ]; }) enabledNetworks) // (lib.mapAttrs' (network: networkCfg: lib.nameValuePair "podman-${lighthouseContainerName network}" { after = [ "network-online.target" "every-channel-ethereum-storage.service" "podman-${rethContainerName network}.service" ]; wants = [ "network-online.target" "every-channel-ethereum-storage.service" "podman-${rethContainerName network}.service" ]; requires = [ "every-channel-ethereum-storage.service" "podman-${rethContainerName network}.service" ]; unitConfig.RequiresMountsFor = [ networkCfg.rootDir ]; }) enabledNetworks); services.caddy.virtualHosts = lib.mkIf (cfg.publicHost != null) { "${cfg.publicHost}".extraConfig = '' encode zstd gzip handle /mainnet/sync { uri replace /mainnet/sync /eth/v1/node/syncing reverse_proxy http://127.0.0.1:${toString cfg.mainnet.lighthouse.httpPort} } handle /mainnet/finality { uri replace /mainnet/finality /eth/v1/beacon/states/head/finality_checkpoints reverse_proxy http://127.0.0.1:${toString cfg.mainnet.lighthouse.httpPort} } handle /sepolia/sync { uri replace /sepolia/sync /eth/v1/node/syncing reverse_proxy http://127.0.0.1:${toString cfg.sepolia.lighthouse.httpPort} } handle /sepolia/finality { uri replace /sepolia/finality /eth/v1/beacon/states/head/finality_checkpoints reverse_proxy http://127.0.0.1:${toString cfg.sepolia.lighthouse.httpPort} } handle { header Content-Type text/plain respond "${caddyRootBody}" 200 } ''; }; }; }