every.channel/nix/modules/ec-ethereum.nix

{ lib, config, pkgs, ... }:

let
  cfg = config.services.every-channel.ethereum;

  mkNetworkSubmodule =
    name: defaults:
    { ... }:
    {
      options = {
        enable = lib.mkOption {
          type = lib.types.bool;
          default = true;
          description = "Whether to run the ${name} Ethereum execution and consensus pair.";
        };

        rootDir = lib.mkOption {
          type = lib.types.str;
          default = "${cfg.rootDir}/${name}";
          description = "Persistent root directory for the ${name} node state.";
        };

        reth = {
          httpPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.rethHttpPort;
            description = "Local HTTP JSON-RPC port for the ${name} Reth node.";
          };

          wsPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.rethWsPort;
            description = "Local WebSocket JSON-RPC port for the ${name} Reth node.";
          };

          authPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.rethAuthPort;
            description = "Local Engine API port for the ${name} Reth node.";
          };

          p2pPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.rethP2pPort;
            description = "RLPx/P2P TCP port for the ${name} Reth node.";
          };

          discoveryPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.rethDiscoveryPort;
            description = "Discovery UDP port for the ${name} Reth node.";
          };

          metricsPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.rethMetricsPort;
            description = "Prometheus port for the ${name} Reth node.";
          };
        };

        lighthouse = {
          httpPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.lighthouseHttpPort;
            description = "Local Beacon API port for the ${name} Lighthouse node.";
          };

          p2pPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.lighthouseP2pPort;
            description = "TCP libp2p port for the ${name} Lighthouse node.";
          };

          discoveryPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.lighthouseDiscoveryPort;
            description = "UDP discovery port for the ${name} Lighthouse node.";
          };

          quicPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.lighthouseQuicPort;
            description = "UDP QUIC port for the ${name} Lighthouse node.";
          };

          metricsPort = lib.mkOption {
            type = lib.types.port;
            default = defaults.lighthouseMetricsPort;
            description = "Prometheus port for the ${name} Lighthouse node.";
          };
        };
      };
    };

  networks = {
    mainnet = cfg.mainnet;
    sepolia = cfg.sepolia;
  };

  enabledNetworks = lib.filterAttrs (_: networkCfg: networkCfg.enable) networks;

  rethContainerName = network: "every-channel-ethereum-${network}-reth";
  lighthouseContainerName = network: "every-channel-ethereum-${network}-lighthouse";

  networkDatasetLines = lib.concatStringsSep "\n" (
    lib.mapAttrsToList
      (network: networkCfg: ''
        ensure_dataset ${lib.escapeShellArg "${cfg.poolName}/${network}"}
        ensure_dataset ${lib.escapeShellArg "${cfg.poolName}/${network}/reth"}
        ensure_dataset ${lib.escapeShellArg "${cfg.poolName}/${network}/lighthouse"}
        ensure_jwt ${lib.escapeShellArg "${networkCfg.rootDir}/jwt.hex"}
      '')
      enabledNetworks
  );

  mkNatArgs = lib.optionals (cfg.publicIp != null) [ "--nat" "extip:${cfg.publicIp}" ];
  mkEnrArgs = lib.optionals (cfg.publicIp != null) [ "--enr-address" cfg.publicIp ];

  mkRethContainer =
    network: networkCfg: {
      image = cfg.images.reth;
      autoStart = true;
      extraOptions = [ "--network=host" ];
      volumes = [ "${networkCfg.rootDir}:/state" ];
      cmd =
        [
          "node"
          "--chain"
          network
          "--datadir"
          "/state/reth"
          "--full"
          "--http"
          "--http.addr"
          "127.0.0.1"
          "--http.port"
          (toString networkCfg.reth.httpPort)
          "--http.api"
          "eth,net,web3,rpc"
          "--ws"
          "--ws.addr"
          "127.0.0.1"
          "--ws.port"
          (toString networkCfg.reth.wsPort)
          "--ws.api"
          "eth,net,web3,rpc"
          "--authrpc.addr"
          "127.0.0.1"
          "--authrpc.port"
          (toString networkCfg.reth.authPort)
          "--authrpc.jwtsecret"
          "/state/jwt.hex"
          "--port"
          (toString networkCfg.reth.p2pPort)
          "--discovery.port"
          (toString networkCfg.reth.discoveryPort)
          "--metrics"
          "127.0.0.1:${toString networkCfg.reth.metricsPort}"
          "--log.stdout.format"
          "json"
        ]
        ++ mkNatArgs;
    };

  mkLighthouseContainer =
    network: networkCfg: {
      image = cfg.images.lighthouse;
      autoStart = true;
      extraOptions = [ "--network=host" ];
      volumes = [ "${networkCfg.rootDir}:/state" ];
      entrypoint = "/usr/local/bin/lighthouse";
      cmd =
        [
          "beacon_node"
          "--network"
          network
          "--datadir"
          "/state/lighthouse"
          "--http"
          "--http-address"
          "127.0.0.1"
          "--http-port"
          (toString networkCfg.lighthouse.httpPort)
          "--execution-endpoint"
          "http://127.0.0.1:${toString networkCfg.reth.authPort}"
          "--execution-jwt"
          "/state/jwt.hex"
          "--allow-insecure-genesis-sync"
          "--port"
          (toString networkCfg.lighthouse.p2pPort)
          "--discovery-port"
          (toString networkCfg.lighthouse.discoveryPort)
          "--quic-port"
          (toString networkCfg.lighthouse.quicPort)
          "--metrics"
          "--metrics-address"
          "127.0.0.1"
          "--metrics-port"
          (toString networkCfg.lighthouse.metricsPort)
        ]
        ++ mkEnrArgs;
    };

  caddyRootBody = ''
    every.channel ethereum nodes
    mainnet sync: /mainnet/sync
    mainnet finality: /mainnet/finality
    sepolia sync: /sepolia/sync
    sepolia finality: /sepolia/finality
    raw execution and beacon RPC remain local-only on ecp-forge for now.
  '';
in
{
  options.services.every-channel.ethereum = {
    enable = lib.mkEnableOption "every.channel dual-network Ethereum full nodes";

    poolName = lib.mkOption {
      type = lib.types.str;
      default = "eth";
      description = "Dedicated ZFS pool name used for Ethereum node state.";
    };

    poolDevice = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "Block device used to create the dedicated Ethereum ZFS pool if it does not already exist.";
    };

    rootDir = lib.mkOption {
      type = lib.types.str;
      default = "/eth";
      description = "Mountpoint for the dedicated Ethereum ZFS pool.";
    };

    publicIp = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "Public IP to advertise in Ethereum P2P metadata.";
    };

    publicHost = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      description = "Optional HTTPS host that publishes node sync and finality surfaces.";
    };

    images = {
      reth = lib.mkOption {
        type = lib.types.str;
        default = "ghcr.io/paradigmxyz/reth:v1.9.3";
        description = "Pinned Reth OCI image.";
      };

      lighthouse = lib.mkOption {
        type = lib.types.str;
        default = "docker.io/sigp/lighthouse:v8.1.1";
        description = "Pinned Lighthouse OCI image.";
      };
    };

    mainnet = lib.mkOption {
      type = lib.types.submodule (mkNetworkSubmodule "mainnet" {
        rethHttpPort = 8545;
        rethWsPort = 8546;
        rethAuthPort = 8551;
        rethP2pPort = 30303;
        rethDiscoveryPort = 30303;
        rethMetricsPort = 19001;
        lighthouseHttpPort = 5052;
        lighthouseP2pPort = 9000;
        lighthouseDiscoveryPort = 9000;
        lighthouseQuicPort = 9001;
        lighthouseMetricsPort = 5054;
      });
      default = { };
      description = "Mainnet Ethereum node configuration.";
    };

    sepolia = lib.mkOption {
      type = lib.types.submodule (mkNetworkSubmodule "sepolia" {
        rethHttpPort = 18545;
        rethWsPort = 18546;
        rethAuthPort = 18551;
        rethP2pPort = 31303;
        rethDiscoveryPort = 31303;
        rethMetricsPort = 29001;
        lighthouseHttpPort = 15052;
        lighthouseP2pPort = 19000;
        lighthouseDiscoveryPort = 19000;
        lighthouseQuicPort = 19001;
        lighthouseMetricsPort = 15054;
      });
      default = { };
      description = "Sepolia Ethereum node configuration.";
    };
  };

  config = lib.mkIf cfg.enable {
    assertions = [
      {
        assertion = cfg.poolDevice != null;
        message = "services.every-channel.ethereum.poolDevice must be set when the Ethereum node is enabled";
      }
      {
        assertion = enabledNetworks != { };
        message = "At least one Ethereum network must be enabled";
      }
    ];

    boot.zfs.extraPools = [ cfg.poolName ];

    networking.firewall = {
      allowedTCPPorts =
        lib.flatten (
          lib.mapAttrsToList
            (_: networkCfg: [
              networkCfg.reth.p2pPort
              networkCfg.lighthouse.p2pPort
            ])
            enabledNetworks
        );
      allowedUDPPorts =
        lib.flatten (
          lib.mapAttrsToList
            (_: networkCfg: [
              networkCfg.reth.discoveryPort
              networkCfg.lighthouse.discoveryPort
              networkCfg.lighthouse.quicPort
            ])
            enabledNetworks
        );
    };

    virtualisation.oci-containers.containers =
      (lib.mapAttrs'
        (network: networkCfg:
          lib.nameValuePair (rethContainerName network) (mkRethContainer network networkCfg))
        enabledNetworks)
      // (lib.mapAttrs'
        (network: networkCfg:
          lib.nameValuePair (lighthouseContainerName network) (mkLighthouseContainer network networkCfg))
        enabledNetworks);

    systemd.services =
      {
        every-channel-ethereum-storage = {
          description = "every.channel Ethereum NVMe ZFS pool and dataset bootstrap";
          wantedBy = [ "multi-user.target" ];
          after = [ "local-fs.target" "zfs.target" ];
          wants = [ "zfs.target" ];
          before =
            lib.flatten (
              lib.mapAttrsToList
                (network: _: [
                  "podman-${rethContainerName network}.service"
                  "podman-${lighthouseContainerName network}.service"
                ])
                enabledNetworks
            );
          path = with pkgs; [
            coreutils
            openssl
            util-linux
            zfs
          ];
          serviceConfig = {
            Type = "oneshot";
            RemainAfterExit = true;
          };
          script = ''
            set -euo pipefail

            pool=${lib.escapeShellArg cfg.poolName}
            root_dir=${lib.escapeShellArg cfg.rootDir}
            device=${lib.escapeShellArg cfg.poolDevice}

            ensure_dataset() {
              local dataset="$1"
              if ! zfs list -H "$dataset" >/dev/null 2>&1; then
                zfs create -p "$dataset"
              fi
              zfs set atime=off compression=lz4 xattr=sa "$dataset" >/dev/null
            }

            ensure_jwt() {
              local path="$1"
              if [[ ! -s "$path" ]]; then
                umask 077
                openssl rand -hex 32 | tr -d '\n' > "$path"
                printf '\n' >> "$path"
              fi
              chmod 0400 "$path"
            }

            if ! zpool list -H "$pool" >/dev/null 2>&1; then
              if [[ -z "$device" ]]; then
                echo "every-channel-ethereum-storage: missing poolDevice for pool $pool" >&2
                exit 1
              fi
              if [[ ! -b "$device" ]]; then
                echo "every-channel-ethereum-storage: device $device not present" >&2
                exit 1
              fi
              if blkid "$device" >/dev/null 2>&1; then
                echo "every-channel-ethereum-storage: device $device already has signatures; refusing to overwrite automatically" >&2
                exit 1
              fi

              zpool create -f \
                -o ashift=12 \
                -O mountpoint="$root_dir" \
                -O atime=off \
                -O compression=lz4 \
                -O xattr=sa \
                "$pool" "$device"
            else
              zfs set mountpoint="$root_dir" "$pool" >/dev/null
            fi

            ${networkDatasetLines}
          '';
        };
      }
      // (lib.mapAttrs'
        (network: networkCfg:
          lib.nameValuePair "podman-${rethContainerName network}" {
            after = [ "network-online.target" "every-channel-ethereum-storage.service" ];
            wants = [ "network-online.target" "every-channel-ethereum-storage.service" ];
            requires = [ "every-channel-ethereum-storage.service" ];
            unitConfig.RequiresMountsFor = [ networkCfg.rootDir ];
          })
        enabledNetworks)
      // (lib.mapAttrs'
        (network: networkCfg:
          lib.nameValuePair "podman-${lighthouseContainerName network}" {
            after = [
              "network-online.target"
              "every-channel-ethereum-storage.service"
              "podman-${rethContainerName network}.service"
            ];
            wants = [
              "network-online.target"
              "every-channel-ethereum-storage.service"
              "podman-${rethContainerName network}.service"
            ];
            requires = [
              "every-channel-ethereum-storage.service"
              "podman-${rethContainerName network}.service"
            ];
            unitConfig.RequiresMountsFor = [ networkCfg.rootDir ];
          })
        enabledNetworks);

    services.caddy.virtualHosts = lib.mkIf (cfg.publicHost != null) {
      "${cfg.publicHost}".extraConfig = ''
        encode zstd gzip

        handle /mainnet/sync {
          uri replace /mainnet/sync /eth/v1/node/syncing
          reverse_proxy http://127.0.0.1:${toString cfg.mainnet.lighthouse.httpPort}
        }

        handle /mainnet/finality {
          uri replace /mainnet/finality /eth/v1/beacon/states/head/finality_checkpoints
          reverse_proxy http://127.0.0.1:${toString cfg.mainnet.lighthouse.httpPort}
        }

        handle /sepolia/sync {
          uri replace /sepolia/sync /eth/v1/node/syncing
          reverse_proxy http://127.0.0.1:${toString cfg.sepolia.lighthouse.httpPort}
        }

        handle /sepolia/finality {
          uri replace /sepolia/finality /eth/v1/beacon/states/head/finality_checkpoints
          reverse_proxy http://127.0.0.1:${toString cfg.sepolia.lighthouse.httpPort}
        }

        handle {
          header Content-Type text/plain
          respond "${caddyRootBody}" 200
        }
      '';
    };
  };
}