every-channel/every.channel
1
0
Fork 0
Code Releases Activity Actions 2
every.channel/nix/nixos/ecp-forge.nix
every.channel 3402f7dab2
Advance forge NBC worker and Ethereum full nodes
2026-04-03 02:01:34 -07:00

338 lines
8.8 KiB
Nix
Raw Blame History

{ config, pkgs, lib, ... }:
let
hasForgejoApiToken = builtins.pathExists ../../secrets/forgejo-api-token.age;
hasNetbootChainToken = builtins.pathExists ../../secrets/netboot-chain-token.age;
hasOpStackSepoliaKey = builtins.pathExists ../../secrets/op-stack-sepolia-private-key.age;
hasOpStackChallengerPrestate = builtins.pathExists ../../secrets/op-stack-challenger-prestate.bin.gz.age;
in
{
imports = [
./ecp-forge-hardware.nix
];
nixpkgs.config.allowUnfreePredicate = pkg:
builtins.elem (lib.getName pkg) [
"google-chrome"
"google-chrome-stable"
];
networking = {
hostName = "ecp-forge";
hostId = "007f0200";
useDHCP = lib.mkForce true;
networkmanager.enable = lib.mkForce false;
nameservers = [ "1.1.1.1" "8.8.8.8" ];
firewall = {
trustedInterfaces = [ "tailscale0" ];
allowedTCPPorts = [
80
443
2222
69
];
allowedUDPPorts = [
67
69
];
};
};
services.openssh = {
enable = true;
startWhenNeeded = true;
openFirewall = true;
settings = {
PasswordAuthentication = false;
PermitRootLogin = lib.mkForce "prohibit-password";
KbdInteractiveAuthentication = false;
};
};
users.users.root.openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBueQxNbP2246pxr/m7au4zNVm+ShC96xuOcfEcpIjWZ"
];
security.sudo = {
execWheelOnly = true;
wheelNeedsPassword = false;
};
users = {
mutableUsers = false;
defaultUserShell = pkgs.bash;
users.conradev = {
uid = 1000;
isNormalUser = true;
password = "password";
group = "conradev";
extraGroups = [ "wheel" "docker" "libvirtd" "kvm" ];
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBueQxNbP2246pxr/m7au4zNVm+ShC96xuOcfEcpIjWZ"
];
};
groups.conradev = { };
};
boot.loader = {
grub = {
enable = true;
device = "/dev/disk/by-id/nvme-KIOXIA_KCD81RUG7T68_25R0A0KZTTEJ";
};
efi.canTouchEfiVariables = false;
};
boot.supportedFilesystems = [ "zfs" ];
boot.zfs.extraPools = [ "tank" ];
boot.kernel.sysctl = {
"fs.inotify.max_user_watches" = "204800";
};
hardware.nvidia-container-toolkit.enable = false;
virtualisation = {
containers.enable = true;
oci-containers.backend = "podman";
podman = {
enable = true;
dockerCompat = true;
autoPrune.enable = true;
defaultNetwork.settings.dns_enabled = true;
};
};
services.tailscale = {
enable = true;
extraUpFlags = "--accept-routes";
};
age.identityPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
age.secrets = lib.mkMerge [
(lib.mkIf hasForgejoApiToken {
"forgejo-api-token" = {
file = ../../secrets/forgejo-api-token.age;
owner = "root";
group = "root";
mode = "0400";
};
})
(lib.mkIf hasNetbootChainToken {
"every-channel-netboot-chain-token" = {
file = ../../secrets/netboot-chain-token.age;
owner = "root";
group = "root";
mode = "0400";
};
})
(lib.mkIf hasOpStackSepoliaKey {
"every-channel-op-stack-sepolia-private-key" = {
file = ../../secrets/op-stack-sepolia-private-key.age;
owner = "root";
group = "root";
mode = "0400";
};
})
(lib.mkIf hasOpStackChallengerPrestate {
"every-channel-op-stack-challenger-prestate" = {
file = ../../secrets/op-stack-challenger-prestate.bin.gz.age;
owner = "root";
group = "root";
mode = "0400";
};
})
];
services.zfs.autoScrub = {
enable = true;
pools = [ "tank" ];
interval = "weekly";
};
services.nfs.server = {
enable = true;
# Keep NFS on trusted/private paths only; public Hetzner exposure triggers
# rpcbind/portmapper enumeration and is not needed for forge access.
exports = ''
/tank 10.0.0.0/8(rw,fsid=0,crossmnt,no_subtree_check,sync) 100.64.0.0/10(rw,fsid=0,crossmnt,no_subtree_check,sync) 192.168.0.0/16(rw,fsid=0,crossmnt,no_subtree_check,sync)
'';
};
nix.gc = {
automatic = true;
dates = "weekly";
options = "--delete-older-than 30d";
};
nix.settings = {
experimental-features = [ "nix-command" "flakes" ];
substituters = lib.mkForce [ "https://cache.nixos.org" ];
trusted-substituters = lib.mkForce [ "https://cache.nixos.org" ];
extra-substituters = lib.mkForce [ ];
trusted-public-keys = lib.mkForce [
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
];
extra-trusted-public-keys = lib.mkForce [ ];
};
time.timeZone = "America/New_York";
i18n.defaultLocale = "en_US.UTF-8";
xdg.mime.enable = true;
services.forgejo = {
enable = true;
database.type = "sqlite3";
lfs.enable = true;
settings = {
server = {
DOMAIN = "git.every.channel";
ROOT_URL = "https://git.every.channel/";
HTTP_ADDR = "127.0.0.1";
HTTP_PORT = 3000;
SSH_DOMAIN = "git.every.channel";
SSH_PORT = 2222;
SSH_LISTEN_PORT = 2222;
START_SSH_SERVER = true;
};
service = {
DISABLE_REGISTRATION = true;
REQUIRE_SIGNIN_VIEW = false;
};
repository = {
DEFAULT_BRANCH = "main";
};
};
};
services.caddy = {
enable = true;
email = "infra@every.channel";
acmeCA = "https://acme-v02.api.letsencrypt.org/directory";
virtualHosts = {
"git.every.channel".extraConfig = ''
reverse_proxy http://127.0.0.1:3000
'';
"forge.every.channel".extraConfig = ''
redir https://git.every.channel{uri} permanent
'';
};
};
services.every-channel.netboot = {
enable = true;
listenIP = "95.216.114.54";
interface = "enp5s0f3u2u2c2";
hostname = "boot.every.channel";
httpPort = 8080;
tftpBootFilename = "ec-ipxe.efi";
httpAllowedCIDRs = [
"10.0.0.0/8"
"172.16.0.0/12"
"192.168.0.0/16"
"100.64.0.0/10"
];
chainTokenFile =
if hasNetbootChainToken
then config.age.secrets."every-channel-netboot-chain-token".path
else null;
proxyDhcp.enable = false;
release.host = "https://git.every.channel";
release.repo = "every-channel/every.channel";
release.localTarball = "/var/lib/every-channel-netboot/sources/ec-runner-x86_64-netboot-local.tar.gz";
release.tokenFile =
if hasForgejoApiToken
then config.age.secrets."forgejo-api-token".path
else null;
};
services.every-channel.ipxe-qemu = {
enable = true;
name = "ecp-forge-ipxe";
boot.userNet.hostname = "ecp-forge-ipxe";
boot.http.root = "${config.services.every-channel.netboot.rootDir}/http";
};
systemd.services.every-channel-ipxe-qemu = {
after = [ "every-channel-netboot-stage.service" ];
wants = [ "every-channel-netboot-stage.service" ];
};
services.every-channel.ec-node = {
enable = true;
nbc = {
enable = true;
chromeBinary = "${pkgs.google-chrome}/bin/google-chrome-stable";
display = ":120";
screen = "1920x1080x24";
noSandbox = true;
vnc = {
enable = true;
listen = "127.0.0.1";
port = 5900;
};
};
broadcasts = [
{
name = "forge-nbc-sports-philly";
nbcUrl = "https://www.nbc.com/live?brand=nbc-sports-philadelphia";
}
];
archive = {
enable = true;
outputDir = "/tank/every-channel/archive";
manifestDir = "/var/lib/every-channel/manifests";
# Keep forge archival as an ingest worker only; replay serving is separate.
serve.enable = false;
};
};
services.every-channel.ethereum = {
enable = true;
poolName = "eth";
poolDevice = "/dev/disk/by-id/nvme-eui.01000000000000008ce38ee307de5c01";
rootDir = "/eth";
publicIp = "95.216.114.54";
publicHost = "eth.every.channel";
};
services.mullvad-vpn = {
enable = true;
enableExcludeWrapper = true;
};
systemd.services.every-channel-wt-publish-forge-nbc-sports-philly = {
after = [ "mullvad-daemon.service" ];
wants = [ "mullvad-daemon.service" ];
};
services.every-channel.op-stack = {
enable = hasOpStackSepoliaKey;
challengerEnable = hasOpStackChallengerPrestate;
disputeMonEnable = hasOpStackChallengerPrestate;
privateKeyFile =
if hasOpStackSepoliaKey
then config.age.secrets."every-channel-op-stack-sepolia-private-key".path
else null;
challengerPrestateFile =
if hasOpStackChallengerPrestate
then config.age.secrets."every-channel-op-stack-challenger-prestate".path
else null;
p2pAdvertiseIp = "95.216.114.54";
};
environment.systemPackages =
(with pkgs; [
git
google-chrome
htop
jq
mullvad-vpn
tmux
x11vnc
zfs
])
++ [
config.services.every-channel.ec-node.package
];
system.stateVersion = "22.11";
}
View git blame Copy permalink