Advance forge NBC worker and Ethereum full nodes
This commit is contained in:
every.channel
parent
7d84510eac
commit
3402f7dab2
17 changed files with 3066 additions and 414 deletions
485
nix/modules/ec-ethereum.nix
Normal file
485
nix/modules/ec-ethereum.nix
Normal file
|
|
@ -0,0 +1,485 @@
|
|||
{ lib, config, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.services.every-channel.ethereum;
|
||||
|
||||
mkNetworkSubmodule =
|
||||
name: defaults:
|
||||
{ ... }:
|
||||
{
|
||||
options = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Whether to run the ${name} Ethereum execution and consensus pair.";
|
||||
};
|
||||
|
||||
rootDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "${cfg.rootDir}/${name}";
|
||||
description = "Persistent root directory for the ${name} node state.";
|
||||
};
|
||||
|
||||
reth = {
|
||||
httpPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.rethHttpPort;
|
||||
description = "Local HTTP JSON-RPC port for the ${name} Reth node.";
|
||||
};
|
||||
|
||||
wsPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.rethWsPort;
|
||||
description = "Local WebSocket JSON-RPC port for the ${name} Reth node.";
|
||||
};
|
||||
|
||||
authPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.rethAuthPort;
|
||||
description = "Local Engine API port for the ${name} Reth node.";
|
||||
};
|
||||
|
||||
p2pPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.rethP2pPort;
|
||||
description = "RLPx/P2P TCP port for the ${name} Reth node.";
|
||||
};
|
||||
|
||||
discoveryPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.rethDiscoveryPort;
|
||||
description = "Discovery UDP port for the ${name} Reth node.";
|
||||
};
|
||||
|
||||
metricsPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.rethMetricsPort;
|
||||
description = "Prometheus port for the ${name} Reth node.";
|
||||
};
|
||||
};
|
||||
|
||||
lighthouse = {
|
||||
httpPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.lighthouseHttpPort;
|
||||
description = "Local Beacon API port for the ${name} Lighthouse node.";
|
||||
};
|
||||
|
||||
p2pPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.lighthouseP2pPort;
|
||||
description = "TCP libp2p port for the ${name} Lighthouse node.";
|
||||
};
|
||||
|
||||
discoveryPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.lighthouseDiscoveryPort;
|
||||
description = "UDP discovery port for the ${name} Lighthouse node.";
|
||||
};
|
||||
|
||||
quicPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.lighthouseQuicPort;
|
||||
description = "UDP QUIC port for the ${name} Lighthouse node.";
|
||||
};
|
||||
|
||||
metricsPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = defaults.lighthouseMetricsPort;
|
||||
description = "Prometheus port for the ${name} Lighthouse node.";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
networks = {
|
||||
mainnet = cfg.mainnet;
|
||||
sepolia = cfg.sepolia;
|
||||
};
|
||||
|
||||
enabledNetworks = lib.filterAttrs (_: networkCfg: networkCfg.enable) networks;
|
||||
|
||||
rethContainerName = network: "every-channel-ethereum-${network}-reth";
|
||||
lighthouseContainerName = network: "every-channel-ethereum-${network}-lighthouse";
|
||||
|
||||
networkDatasetLines = lib.concatStringsSep "\n" (
|
||||
lib.mapAttrsToList
|
||||
(network: networkCfg: ''
|
||||
ensure_dataset ${lib.escapeShellArg "${cfg.poolName}/${network}"}
|
||||
ensure_dataset ${lib.escapeShellArg "${cfg.poolName}/${network}/reth"}
|
||||
ensure_dataset ${lib.escapeShellArg "${cfg.poolName}/${network}/lighthouse"}
|
||||
ensure_jwt ${lib.escapeShellArg "${networkCfg.rootDir}/jwt.hex"}
|
||||
'')
|
||||
enabledNetworks
|
||||
);
|
||||
|
||||
mkNatArgs = lib.optionals (cfg.publicIp != null) [ "--nat" "extip:${cfg.publicIp}" ];
|
||||
mkEnrArgs = lib.optionals (cfg.publicIp != null) [ "--enr-address" cfg.publicIp ];
|
||||
|
||||
mkRethContainer =
|
||||
network: networkCfg: {
|
||||
image = cfg.images.reth;
|
||||
autoStart = true;
|
||||
extraOptions = [ "--network=host" ];
|
||||
volumes = [ "${networkCfg.rootDir}:/state" ];
|
||||
cmd =
|
||||
[
|
||||
"node"
|
||||
"--chain"
|
||||
network
|
||||
"--datadir"
|
||||
"/state/reth"
|
||||
"--full"
|
||||
"--http"
|
||||
"--http.addr"
|
||||
"127.0.0.1"
|
||||
"--http.port"
|
||||
(toString networkCfg.reth.httpPort)
|
||||
"--http.api"
|
||||
"eth,net,web3,rpc"
|
||||
"--ws"
|
||||
"--ws.addr"
|
||||
"127.0.0.1"
|
||||
"--ws.port"
|
||||
(toString networkCfg.reth.wsPort)
|
||||
"--ws.api"
|
||||
"eth,net,web3,rpc"
|
||||
"--authrpc.addr"
|
||||
"127.0.0.1"
|
||||
"--authrpc.port"
|
||||
(toString networkCfg.reth.authPort)
|
||||
"--authrpc.jwtsecret"
|
||||
"/state/jwt.hex"
|
||||
"--port"
|
||||
(toString networkCfg.reth.p2pPort)
|
||||
"--discovery.port"
|
||||
(toString networkCfg.reth.discoveryPort)
|
||||
"--metrics"
|
||||
"127.0.0.1:${toString networkCfg.reth.metricsPort}"
|
||||
"--log.stdout.format"
|
||||
"json"
|
||||
]
|
||||
++ mkNatArgs;
|
||||
};
|
||||
|
||||
mkLighthouseContainer =
|
||||
network: networkCfg: {
|
||||
image = cfg.images.lighthouse;
|
||||
autoStart = true;
|
||||
extraOptions = [ "--network=host" ];
|
||||
volumes = [ "${networkCfg.rootDir}:/state" ];
|
||||
entrypoint = "/usr/local/bin/lighthouse";
|
||||
cmd =
|
||||
[
|
||||
"beacon_node"
|
||||
"--network"
|
||||
network
|
||||
"--datadir"
|
||||
"/state/lighthouse"
|
||||
"--http"
|
||||
"--http-address"
|
||||
"127.0.0.1"
|
||||
"--http-port"
|
||||
(toString networkCfg.lighthouse.httpPort)
|
||||
"--execution-endpoint"
|
||||
"http://127.0.0.1:${toString networkCfg.reth.authPort}"
|
||||
"--execution-jwt"
|
||||
"/state/jwt.hex"
|
||||
"--allow-insecure-genesis-sync"
|
||||
"--port"
|
||||
(toString networkCfg.lighthouse.p2pPort)
|
||||
"--discovery-port"
|
||||
(toString networkCfg.lighthouse.discoveryPort)
|
||||
"--quic-port"
|
||||
(toString networkCfg.lighthouse.quicPort)
|
||||
"--metrics"
|
||||
"--metrics-address"
|
||||
"127.0.0.1"
|
||||
"--metrics-port"
|
||||
(toString networkCfg.lighthouse.metricsPort)
|
||||
]
|
||||
++ mkEnrArgs;
|
||||
};
|
||||
|
||||
caddyRootBody = ''
|
||||
every.channel ethereum nodes
|
||||
mainnet sync: /mainnet/sync
|
||||
mainnet finality: /mainnet/finality
|
||||
sepolia sync: /sepolia/sync
|
||||
sepolia finality: /sepolia/finality
|
||||
raw execution and beacon RPC remain local-only on ecp-forge for now.
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.services.every-channel.ethereum = {
|
||||
enable = lib.mkEnableOption "every.channel dual-network Ethereum full nodes";
|
||||
|
||||
poolName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "eth";
|
||||
description = "Dedicated ZFS pool name used for Ethereum node state.";
|
||||
};
|
||||
|
||||
poolDevice = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Block device used to create the dedicated Ethereum ZFS pool if it does not already exist.";
|
||||
};
|
||||
|
||||
rootDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/eth";
|
||||
description = "Mountpoint for the dedicated Ethereum ZFS pool.";
|
||||
};
|
||||
|
||||
publicIp = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Public IP to advertise in Ethereum P2P metadata.";
|
||||
};
|
||||
|
||||
publicHost = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Optional HTTPS host that publishes node sync and finality surfaces.";
|
||||
};
|
||||
|
||||
images = {
|
||||
reth = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "ghcr.io/paradigmxyz/reth:v1.9.3";
|
||||
description = "Pinned Reth OCI image.";
|
||||
};
|
||||
|
||||
lighthouse = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "docker.io/sigp/lighthouse:v8.1.1";
|
||||
description = "Pinned Lighthouse OCI image.";
|
||||
};
|
||||
};
|
||||
|
||||
mainnet = lib.mkOption {
|
||||
type = lib.types.submodule (mkNetworkSubmodule "mainnet" {
|
||||
rethHttpPort = 8545;
|
||||
rethWsPort = 8546;
|
||||
rethAuthPort = 8551;
|
||||
rethP2pPort = 30303;
|
||||
rethDiscoveryPort = 30303;
|
||||
rethMetricsPort = 19001;
|
||||
lighthouseHttpPort = 5052;
|
||||
lighthouseP2pPort = 9000;
|
||||
lighthouseDiscoveryPort = 9000;
|
||||
lighthouseQuicPort = 9001;
|
||||
lighthouseMetricsPort = 5054;
|
||||
});
|
||||
default = { };
|
||||
description = "Mainnet Ethereum node configuration.";
|
||||
};
|
||||
|
||||
sepolia = lib.mkOption {
|
||||
type = lib.types.submodule (mkNetworkSubmodule "sepolia" {
|
||||
rethHttpPort = 18545;
|
||||
rethWsPort = 18546;
|
||||
rethAuthPort = 18551;
|
||||
rethP2pPort = 31303;
|
||||
rethDiscoveryPort = 31303;
|
||||
rethMetricsPort = 29001;
|
||||
lighthouseHttpPort = 15052;
|
||||
lighthouseP2pPort = 19000;
|
||||
lighthouseDiscoveryPort = 19000;
|
||||
lighthouseQuicPort = 19001;
|
||||
lighthouseMetricsPort = 15054;
|
||||
});
|
||||
default = { };
|
||||
description = "Sepolia Ethereum node configuration.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.poolDevice != null;
|
||||
message = "services.every-channel.ethereum.poolDevice must be set when the Ethereum node is enabled";
|
||||
}
|
||||
{
|
||||
assertion = enabledNetworks != { };
|
||||
message = "At least one Ethereum network must be enabled";
|
||||
}
|
||||
];
|
||||
|
||||
boot.zfs.extraPools = [ cfg.poolName ];
|
||||
|
||||
networking.firewall = {
|
||||
allowedTCPPorts =
|
||||
lib.flatten (
|
||||
lib.mapAttrsToList
|
||||
(_: networkCfg: [
|
||||
networkCfg.reth.p2pPort
|
||||
networkCfg.lighthouse.p2pPort
|
||||
])
|
||||
enabledNetworks
|
||||
);
|
||||
allowedUDPPorts =
|
||||
lib.flatten (
|
||||
lib.mapAttrsToList
|
||||
(_: networkCfg: [
|
||||
networkCfg.reth.discoveryPort
|
||||
networkCfg.lighthouse.discoveryPort
|
||||
networkCfg.lighthouse.quicPort
|
||||
])
|
||||
enabledNetworks
|
||||
);
|
||||
};
|
||||
|
||||
virtualisation.oci-containers.containers =
|
||||
(lib.mapAttrs'
|
||||
(network: networkCfg:
|
||||
lib.nameValuePair (rethContainerName network) (mkRethContainer network networkCfg))
|
||||
enabledNetworks)
|
||||
// (lib.mapAttrs'
|
||||
(network: networkCfg:
|
||||
lib.nameValuePair (lighthouseContainerName network) (mkLighthouseContainer network networkCfg))
|
||||
enabledNetworks);
|
||||
|
||||
systemd.services =
|
||||
{
|
||||
every-channel-ethereum-storage = {
|
||||
description = "every.channel Ethereum NVMe ZFS pool and dataset bootstrap";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "local-fs.target" "zfs.target" ];
|
||||
wants = [ "zfs.target" ];
|
||||
before =
|
||||
lib.flatten (
|
||||
lib.mapAttrsToList
|
||||
(network: _: [
|
||||
"podman-${rethContainerName network}.service"
|
||||
"podman-${lighthouseContainerName network}.service"
|
||||
])
|
||||
enabledNetworks
|
||||
);
|
||||
path = with pkgs; [
|
||||
coreutils
|
||||
openssl
|
||||
util-linux
|
||||
zfs
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
pool=${lib.escapeShellArg cfg.poolName}
|
||||
root_dir=${lib.escapeShellArg cfg.rootDir}
|
||||
device=${lib.escapeShellArg cfg.poolDevice}
|
||||
|
||||
ensure_dataset() {
|
||||
local dataset="$1"
|
||||
if ! zfs list -H "$dataset" >/dev/null 2>&1; then
|
||||
zfs create -p "$dataset"
|
||||
fi
|
||||
zfs set atime=off compression=lz4 xattr=sa "$dataset" >/dev/null
|
||||
}
|
||||
|
||||
ensure_jwt() {
|
||||
local path="$1"
|
||||
if [[ ! -s "$path" ]]; then
|
||||
umask 077
|
||||
openssl rand -hex 32 | tr -d '\n' > "$path"
|
||||
printf '\n' >> "$path"
|
||||
fi
|
||||
chmod 0400 "$path"
|
||||
}
|
||||
|
||||
if ! zpool list -H "$pool" >/dev/null 2>&1; then
|
||||
if [[ -z "$device" ]]; then
|
||||
echo "every-channel-ethereum-storage: missing poolDevice for pool $pool" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -b "$device" ]]; then
|
||||
echo "every-channel-ethereum-storage: device $device not present" >&2
|
||||
exit 1
|
||||
fi
|
||||
if blkid "$device" >/dev/null 2>&1; then
|
||||
echo "every-channel-ethereum-storage: device $device already has signatures; refusing to overwrite automatically" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
zpool create -f \
|
||||
-o ashift=12 \
|
||||
-O mountpoint="$root_dir" \
|
||||
-O atime=off \
|
||||
-O compression=lz4 \
|
||||
-O xattr=sa \
|
||||
"$pool" "$device"
|
||||
else
|
||||
zfs set mountpoint="$root_dir" "$pool" >/dev/null
|
||||
fi
|
||||
|
||||
${networkDatasetLines}
|
||||
'';
|
||||
};
|
||||
}
|
||||
// (lib.mapAttrs'
|
||||
(network: networkCfg:
|
||||
lib.nameValuePair "podman-${rethContainerName network}" {
|
||||
after = [ "network-online.target" "every-channel-ethereum-storage.service" ];
|
||||
wants = [ "network-online.target" "every-channel-ethereum-storage.service" ];
|
||||
requires = [ "every-channel-ethereum-storage.service" ];
|
||||
unitConfig.RequiresMountsFor = [ networkCfg.rootDir ];
|
||||
})
|
||||
enabledNetworks)
|
||||
// (lib.mapAttrs'
|
||||
(network: networkCfg:
|
||||
lib.nameValuePair "podman-${lighthouseContainerName network}" {
|
||||
after = [
|
||||
"network-online.target"
|
||||
"every-channel-ethereum-storage.service"
|
||||
"podman-${rethContainerName network}.service"
|
||||
];
|
||||
wants = [
|
||||
"network-online.target"
|
||||
"every-channel-ethereum-storage.service"
|
||||
"podman-${rethContainerName network}.service"
|
||||
];
|
||||
requires = [
|
||||
"every-channel-ethereum-storage.service"
|
||||
"podman-${rethContainerName network}.service"
|
||||
];
|
||||
unitConfig.RequiresMountsFor = [ networkCfg.rootDir ];
|
||||
})
|
||||
enabledNetworks);
|
||||
|
||||
services.caddy.virtualHosts = lib.mkIf (cfg.publicHost != null) {
|
||||
"${cfg.publicHost}".extraConfig = ''
|
||||
encode zstd gzip
|
||||
|
||||
handle /mainnet/sync {
|
||||
uri replace /mainnet/sync /eth/v1/node/syncing
|
||||
reverse_proxy http://127.0.0.1:${toString cfg.mainnet.lighthouse.httpPort}
|
||||
}
|
||||
|
||||
handle /mainnet/finality {
|
||||
uri replace /mainnet/finality /eth/v1/beacon/states/head/finality_checkpoints
|
||||
reverse_proxy http://127.0.0.1:${toString cfg.mainnet.lighthouse.httpPort}
|
||||
}
|
||||
|
||||
handle /sepolia/sync {
|
||||
uri replace /sepolia/sync /eth/v1/node/syncing
|
||||
reverse_proxy http://127.0.0.1:${toString cfg.sepolia.lighthouse.httpPort}
|
||||
}
|
||||
|
||||
handle /sepolia/finality {
|
||||
uri replace /sepolia/finality /eth/v1/beacon/states/head/finality_checkpoints
|
||||
reverse_proxy http://127.0.0.1:${toString cfg.sepolia.lighthouse.httpPort}
|
||||
}
|
||||
|
||||
handle {
|
||||
header Content-Type text/plain
|
||||
respond "${caddyRootBody}" 200
|
||||
}
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue