every.channel/nix/nixos/ecp-forge.nix

{ config, pkgs, lib, ... }:

let
  hasForgejoApiToken = builtins.pathExists ../../secrets/forgejo-api-token.age;
  hasNetbootChainToken = builtins.pathExists ../../secrets/netboot-chain-token.age;
  hasOpStackSepoliaKey = builtins.pathExists ../../secrets/op-stack-sepolia-private-key.age;
  hasOpStackChallengerPrestate = builtins.pathExists ../../secrets/op-stack-challenger-prestate.bin.gz.age;
in
{
  imports = [
    ./ecp-forge-hardware.nix
  ];

  networking = {
    hostName = "ecp-forge";
    hostId = "007f0200";
    useDHCP = lib.mkForce true;
    networkmanager.enable = lib.mkForce false;
    nameservers = [ "1.1.1.1" "8.8.8.8" ];
    firewall = {
      trustedInterfaces = [ "tailscale0" ];
      allowedTCPPorts = [
        80
        443
        2222
        69
      ];
      allowedUDPPorts = [
        67
        69
      ];
    };
  };

  services.openssh = {
    enable = true;
    startWhenNeeded = true;
    openFirewall = true;
    settings = {
      PasswordAuthentication = false;
      PermitRootLogin = lib.mkForce "prohibit-password";
      KbdInteractiveAuthentication = false;
    };
  };

  users.users.root.openssh.authorizedKeys.keys = [
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBueQxNbP2246pxr/m7au4zNVm+ShC96xuOcfEcpIjWZ"
  ];

  security.sudo = {
    execWheelOnly = true;
    wheelNeedsPassword = false;
  };

  users = {
    mutableUsers = false;
    defaultUserShell = pkgs.bash;
    users.conradev = {
      uid = 1000;
      isNormalUser = true;
      password = "password";
      group = "conradev";
      extraGroups = [ "wheel" "docker" "libvirtd" "kvm" ];
      openssh.authorizedKeys.keys = [
        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBueQxNbP2246pxr/m7au4zNVm+ShC96xuOcfEcpIjWZ"
      ];
    };
    groups.conradev = { };
  };

  boot.loader = {
    grub = {
      enable = true;
      device = "/dev/disk/by-id/nvme-KIOXIA_KCD81RUG7T68_25R0A0KZTTEJ";
    };
    efi.canTouchEfiVariables = false;
  };

  boot.supportedFilesystems = [ "zfs" ];
  boot.zfs.extraPools = [ "tank" ];
  boot.kernel.sysctl = {
    "fs.inotify.max_user_watches" = "204800";
  };

  hardware.nvidia-container-toolkit.enable = false;

  virtualisation = {
    containers.enable = true;
    oci-containers.backend = "podman";
    podman = {
      enable = true;
      dockerCompat = true;
      autoPrune.enable = true;
      defaultNetwork.settings.dns_enabled = true;
    };
  };

  services.tailscale = {
    enable = true;
    extraUpFlags = "--accept-routes";
  };

  age.identityPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
  age.secrets = lib.mkMerge [
    (lib.mkIf hasForgejoApiToken {
      "forgejo-api-token" = {
        file = ../../secrets/forgejo-api-token.age;
        owner = "root";
        group = "root";
        mode = "0400";
      };
    })
    (lib.mkIf hasNetbootChainToken {
      "every-channel-netboot-chain-token" = {
        file = ../../secrets/netboot-chain-token.age;
        owner = "root";
        group = "root";
        mode = "0400";
      };
    })
    (lib.mkIf hasOpStackSepoliaKey {
      "every-channel-op-stack-sepolia-private-key" = {
        file = ../../secrets/op-stack-sepolia-private-key.age;
        owner = "root";
        group = "root";
        mode = "0400";
      };
    })
    (lib.mkIf hasOpStackChallengerPrestate {
      "every-channel-op-stack-challenger-prestate" = {
        file = ../../secrets/op-stack-challenger-prestate.bin.gz.age;
        owner = "root";
        group = "root";
        mode = "0400";
      };
    })
  ];

  services.zfs.autoScrub = {
    enable = true;
    pools = [ "tank" ];
    interval = "weekly";
  };

  services.nfs.server = {
    enable = true;
    # Keep NFS on trusted/private paths only; public Hetzner exposure triggers
    # rpcbind/portmapper enumeration and is not needed for forge access.
    exports = ''
      /tank 10.0.0.0/8(rw,fsid=0,crossmnt,no_subtree_check,sync) 100.64.0.0/10(rw,fsid=0,crossmnt,no_subtree_check,sync) 192.168.0.0/16(rw,fsid=0,crossmnt,no_subtree_check,sync)
    '';
  };

  nix.gc = {
    automatic = true;
    dates = "weekly";
    options = "--delete-older-than 30d";
  };

  nix.settings = {
    experimental-features = [ "nix-command" "flakes" ];
    substituters = lib.mkForce [ "https://cache.nixos.org" ];
    trusted-substituters = lib.mkForce [ "https://cache.nixos.org" ];
    extra-substituters = lib.mkForce [ ];
    trusted-public-keys = lib.mkForce [
      "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
    ];
    extra-trusted-public-keys = lib.mkForce [ ];
  };

  time.timeZone = "America/New_York";
  i18n.defaultLocale = "en_US.UTF-8";
  xdg.mime.enable = true;

  services.forgejo = {
    enable = true;
    database.type = "sqlite3";
    lfs.enable = true;
    settings = {
      server = {
        DOMAIN = "git.every.channel";
        ROOT_URL = "https://git.every.channel/";
        HTTP_ADDR = "127.0.0.1";
        HTTP_PORT = 3000;
        SSH_DOMAIN = "git.every.channel";
        SSH_PORT = 2222;
        SSH_LISTEN_PORT = 2222;
        START_SSH_SERVER = true;
      };
      service = {
        DISABLE_REGISTRATION = true;
        REQUIRE_SIGNIN_VIEW = false;
      };
      repository = {
        DEFAULT_BRANCH = "main";
      };
    };
  };

  services.caddy = {
    enable = true;
    email = "infra@every.channel";
    acmeCA = "https://acme-v02.api.letsencrypt.org/directory";
    virtualHosts = {
      "git.every.channel".extraConfig = ''
        reverse_proxy http://127.0.0.1:3000
      '';
      "forge.every.channel".extraConfig = ''
        redir https://git.every.channel{uri} permanent
      '';
    };
  };

  services.every-channel.netboot = {
    enable = true;
    listenIP = "95.216.114.54";
    interface = "enp5s0f3u2u2c2";
    hostname = "boot.every.channel";
    httpPort = 8080;
    tftpBootFilename = "ec-ipxe.efi";
    httpAllowedCIDRs = [
      "10.0.0.0/8"
      "172.16.0.0/12"
      "192.168.0.0/16"
      "100.64.0.0/10"
    ];
    chainTokenFile =
      if hasNetbootChainToken
      then config.age.secrets."every-channel-netboot-chain-token".path
      else null;
    proxyDhcp.enable = false;
    release.host = "https://git.every.channel";
    release.repo = "every-channel/every.channel";
    release.localTarball = "/var/lib/every-channel-netboot/sources/ec-runner-x86_64-netboot-local.tar.gz";
    release.tokenFile =
      if hasForgejoApiToken
      then config.age.secrets."forgejo-api-token".path
      else null;
  };

  services.every-channel.ipxe-qemu = {
    enable = true;
    name = "ecp-forge-ipxe";
    boot.userNet.hostname = "ecp-forge-ipxe";
    boot.http.root = "${config.services.every-channel.netboot.rootDir}/http";
  };

  systemd.services.every-channel-ipxe-qemu = {
    after = [ "every-channel-netboot-stage.service" ];
    wants = [ "every-channel-netboot-stage.service" ];
  };

  services.every-channel.ec-node = {
    enable = true;
    archive = {
      enable = true;
      outputDir = "/tank/every-channel/archive";
      manifestDir = "/var/lib/every-channel/manifests";
      # Keep forge archival as an ingest worker only; replay serving is separate.
      serve.enable = false;
    };
  };

  services.every-channel.op-stack = {
    enable = hasOpStackSepoliaKey;
    challengerEnable = hasOpStackChallengerPrestate;
    disputeMonEnable = hasOpStackChallengerPrestate;
    privateKeyFile =
      if hasOpStackSepoliaKey
      then config.age.secrets."every-channel-op-stack-sepolia-private-key".path
      else null;
    challengerPrestateFile =
      if hasOpStackChallengerPrestate
      then config.age.secrets."every-channel-op-stack-challenger-prestate".path
      else null;
    p2pAdvertiseIp = "95.216.114.54";
  };

  environment.systemPackages = with pkgs; [
    git
    htop
    jq
    tmux
    zfs
  ];

  system.stateVersion = "22.11";
}