control: add transport resolver and nix control announce wiring
This commit is contained in:
parent
f77fab378b
commit
faec62f9ae
4 changed files with 260 additions and 30 deletions
10
README.md
10
README.md
|
|
@ -68,14 +68,20 @@ Control protocol (iroh gossip, relay + direct transport discovery):
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
# Listener (on node A)
|
# Listener (on node A)
|
||||||
cargo run -p ec-node -- control-listen --gossip-peer <node-b-endpoint-id>
|
cargo run -p ec-node -- control-listen --gossip-peer <node-b-endpoint-addr>
|
||||||
|
|
||||||
# Announcer (on node B)
|
# Announcer (on node B)
|
||||||
cargo run -p ec-node -- control-announce \
|
cargo run -p ec-node -- control-announce \
|
||||||
--stream-id la-nbc \
|
--stream-id la-nbc \
|
||||||
--relay-url https://cdn.moq.dev/anon \
|
--relay-url https://cdn.moq.dev/anon \
|
||||||
--relay-broadcast la-nbc \
|
--relay-broadcast la-nbc \
|
||||||
--gossip-peer <node-a-endpoint-id>
|
--gossip-peer <node-a-endpoint-addr>
|
||||||
|
|
||||||
|
# Resolver (consumer picks best announced path)
|
||||||
|
cargo run -p ec-node -- control-resolve \
|
||||||
|
--stream-id la-nbc \
|
||||||
|
--prefer direct-first \
|
||||||
|
--gossip-peer <node-a-endpoint-addr>
|
||||||
```
|
```
|
||||||
|
|
||||||
Coverage:
|
Coverage:
|
||||||
|
|
|
||||||
|
|
@ -84,6 +84,8 @@ enum Commands {
|
||||||
ControlAnnounce(ControlAnnounceArgs),
|
ControlAnnounce(ControlAnnounceArgs),
|
||||||
/// Listen for stream transport announcements from iroh gossip control topic.
|
/// Listen for stream transport announcements from iroh gossip control topic.
|
||||||
ControlListen(ControlListenArgs),
|
ControlListen(ControlListenArgs),
|
||||||
|
/// Resolve a stream id to the best currently-announced transport.
|
||||||
|
ControlResolve(ControlResolveArgs),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||||
|
|
@ -508,6 +510,46 @@ struct ControlListenArgs {
|
||||||
once: bool,
|
once: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||||
|
enum ControlTransportPreference {
|
||||||
|
/// Prefer iroh direct transport when both are available.
|
||||||
|
DirectFirst,
|
||||||
|
/// Prefer relay transport when both are available.
|
||||||
|
RelayFirst,
|
||||||
|
/// Accept only iroh direct transport.
|
||||||
|
DirectOnly,
|
||||||
|
/// Accept only relay transport.
|
||||||
|
RelayOnly,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Parser, Debug)]
|
||||||
|
struct ControlResolveArgs {
|
||||||
|
/// Stable stream id to resolve from control announcements.
|
||||||
|
#[arg(long)]
|
||||||
|
stream_id: String,
|
||||||
|
/// Transport selection preference.
|
||||||
|
#[arg(long, value_enum, default_value_t = ControlTransportPreference::DirectFirst)]
|
||||||
|
prefer: ControlTransportPreference,
|
||||||
|
/// Maximum wall-clock wait for a matching announcement (ms).
|
||||||
|
#[arg(long, default_value_t = 30000)]
|
||||||
|
timeout_ms: u64,
|
||||||
|
/// Maximum accepted staleness from updated_unix_ms (ms).
|
||||||
|
#[arg(long, default_value_t = 30000)]
|
||||||
|
max_age_ms: u64,
|
||||||
|
/// Include the full raw announcement in output JSON.
|
||||||
|
#[arg(long, default_value_t = false)]
|
||||||
|
include_announcement: bool,
|
||||||
|
/// Optional iroh secret key (hex) for control gossip endpoint identity.
|
||||||
|
#[arg(long)]
|
||||||
|
iroh_secret: Option<String>,
|
||||||
|
/// Discovery modes to enable (comma-separated: dht, mdns, dns).
|
||||||
|
#[arg(long)]
|
||||||
|
discovery: Option<String>,
|
||||||
|
/// Gossip peers to connect to (repeatable).
|
||||||
|
#[arg(long)]
|
||||||
|
gossip_peer: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Subcommand, Debug)]
|
#[derive(Subcommand, Debug)]
|
||||||
enum IngestSource {
|
enum IngestSource {
|
||||||
/// Ingest from an HDHomeRun device.
|
/// Ingest from an HDHomeRun device.
|
||||||
|
|
@ -587,6 +629,7 @@ fn main() -> Result<()> {
|
||||||
Commands::WtPublish(args) => run_async(wt_publish(args))?,
|
Commands::WtPublish(args) => run_async(wt_publish(args))?,
|
||||||
Commands::ControlAnnounce(args) => run_async(control_announce(args))?,
|
Commands::ControlAnnounce(args) => run_async(control_announce(args))?,
|
||||||
Commands::ControlListen(args) => run_async(control_listen(args))?,
|
Commands::ControlListen(args) => run_async(control_listen(args))?,
|
||||||
|
Commands::ControlResolve(args) => run_async(control_resolve(args))?,
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
@ -4361,12 +4404,6 @@ async fn spawn_control_announcer_task(
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn control_announce(args: ControlAnnounceArgs) -> Result<()> {
|
async fn control_announce(args: ControlAnnounceArgs) -> Result<()> {
|
||||||
if args.gossip_peer.is_empty() {
|
|
||||||
return Err(anyhow!(
|
|
||||||
"control announce requires at least one --gossip-peer currently"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
let secret = parse_iroh_secret(args.iroh_secret)?;
|
let secret = parse_iroh_secret(args.iroh_secret)?;
|
||||||
let discovery = parse_discovery(args.discovery.as_deref())?;
|
let discovery = parse_discovery(args.discovery.as_deref())?;
|
||||||
let endpoint = ec_iroh::build_endpoint(secret, discovery).await?;
|
let endpoint = ec_iroh::build_endpoint(secret, discovery).await?;
|
||||||
|
|
@ -4459,6 +4496,111 @@ async fn control_listen(args: ControlListenArgs) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn control_announcement_is_fresh(
|
||||||
|
announcement: &StreamControlAnnouncement,
|
||||||
|
max_age_ms: u64,
|
||||||
|
) -> bool {
|
||||||
|
let now = now_unix_ms();
|
||||||
|
let freshness_ms = announcement.ttl_ms.min(max_age_ms.max(1));
|
||||||
|
now <= announcement.updated_unix_ms.saturating_add(freshness_ms)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn select_control_transport(
|
||||||
|
transports: &[StreamTransportDescriptor],
|
||||||
|
prefer: ControlTransportPreference,
|
||||||
|
) -> Option<StreamTransportDescriptor> {
|
||||||
|
let is_direct =
|
||||||
|
|t: &StreamTransportDescriptor| matches!(t, StreamTransportDescriptor::IrohDirect { .. });
|
||||||
|
let is_relay =
|
||||||
|
|t: &StreamTransportDescriptor| matches!(t, StreamTransportDescriptor::RelayMoq { .. });
|
||||||
|
|
||||||
|
match prefer {
|
||||||
|
ControlTransportPreference::DirectOnly => transports.iter().find(|t| is_direct(t)).cloned(),
|
||||||
|
ControlTransportPreference::RelayOnly => transports.iter().find(|t| is_relay(t)).cloned(),
|
||||||
|
ControlTransportPreference::DirectFirst => transports
|
||||||
|
.iter()
|
||||||
|
.find(|t| is_direct(t))
|
||||||
|
.cloned()
|
||||||
|
.or_else(|| transports.iter().find(|t| is_relay(t)).cloned()),
|
||||||
|
ControlTransportPreference::RelayFirst => transports
|
||||||
|
.iter()
|
||||||
|
.find(|t| is_relay(t))
|
||||||
|
.cloned()
|
||||||
|
.or_else(|| transports.iter().find(|t| is_direct(t)).cloned()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn control_resolve(args: ControlResolveArgs) -> Result<()> {
|
||||||
|
let secret = parse_iroh_secret(args.iroh_secret)?;
|
||||||
|
let discovery = parse_discovery(args.discovery.as_deref())?;
|
||||||
|
let endpoint = ec_iroh::build_endpoint(secret, discovery).await?;
|
||||||
|
eprintln!("control endpoint id: {}", endpoint.id());
|
||||||
|
|
||||||
|
let mut gossip = tokio::time::timeout(
|
||||||
|
Duration::from_secs(30),
|
||||||
|
ec_iroh::ControlGossip::join(endpoint.clone(), &args.gossip_peer),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.context("timed out joining control gossip topic")??;
|
||||||
|
|
||||||
|
let timeout = Duration::from_millis(args.timeout_ms.max(1000));
|
||||||
|
let deadline = Instant::now() + timeout;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let now = Instant::now();
|
||||||
|
if now >= deadline {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let remaining = deadline - now;
|
||||||
|
|
||||||
|
let maybe = tokio::time::timeout(remaining, gossip.next_announcement())
|
||||||
|
.await
|
||||||
|
.context("timed out waiting for control announcement")??;
|
||||||
|
let Some(announcement) = maybe else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if announcement.stream.id.0.as_str() != args.stream_id {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !control_announcement_is_fresh(&announcement, args.max_age_ms) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(transport) = select_control_transport(&announcement.transports, args.prefer)
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let stream_id = announcement.stream.id.0.clone();
|
||||||
|
let title = announcement.stream.title.clone();
|
||||||
|
|
||||||
|
let output = if args.include_announcement {
|
||||||
|
serde_json::json!({
|
||||||
|
"stream_id": stream_id,
|
||||||
|
"title": title,
|
||||||
|
"transport": transport,
|
||||||
|
"updated_unix_ms": announcement.updated_unix_ms,
|
||||||
|
"ttl_ms": announcement.ttl_ms,
|
||||||
|
"announcement": announcement,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
serde_json::json!({
|
||||||
|
"stream_id": stream_id,
|
||||||
|
"title": title,
|
||||||
|
"transport": transport,
|
||||||
|
"updated_unix_ms": announcement.updated_unix_ms,
|
||||||
|
"ttl_ms": announcement.ttl_ms,
|
||||||
|
})
|
||||||
|
};
|
||||||
|
println!("{}", serde_json::to_string(&output)?);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(anyhow!(
|
||||||
|
"timed out resolving stream_id '{}' from control topic",
|
||||||
|
args.stream_id
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
fn wait_for_stable_file(path: &Path, timeout: Duration) -> Result<()> {
|
fn wait_for_stable_file(path: &Path, timeout: Duration) -> Result<()> {
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let mut last_len: Option<u64> = None;
|
let mut last_len: Option<u64> = None;
|
||||||
|
|
@ -4513,28 +4655,24 @@ async fn wt_publish(args: WtPublishArgs) -> Result<()> {
|
||||||
args.control_ttl_ms,
|
args.control_ttl_ms,
|
||||||
);
|
);
|
||||||
|
|
||||||
if args.gossip_peer.is_empty() {
|
match spawn_control_announcer_task(
|
||||||
tracing::warn!("control announce requested but no gossip peers configured; skipping");
|
endpoint.clone(),
|
||||||
} else {
|
args.gossip_peer.clone(),
|
||||||
match spawn_control_announcer_task(
|
announcement,
|
||||||
endpoint.clone(),
|
Duration::from_millis(args.control_interval_ms.max(1000)),
|
||||||
args.gossip_peer.clone(),
|
)
|
||||||
announcement,
|
.await
|
||||||
Duration::from_millis(args.control_interval_ms.max(1000)),
|
{
|
||||||
)
|
Ok(stop_tx) => {
|
||||||
.await
|
tracing::info!(
|
||||||
{
|
endpoint = %endpoint.id(),
|
||||||
Ok(stop_tx) => {
|
stream = %args.name,
|
||||||
tracing::info!(
|
"control announce enabled"
|
||||||
endpoint = %endpoint.id(),
|
);
|
||||||
stream = %args.name,
|
control_stop = Some(stop_tx);
|
||||||
"control announce enabled"
|
}
|
||||||
);
|
Err(err) => {
|
||||||
control_stop = Some(stop_tx);
|
tracing::warn!("failed to start control announce task: {err:#}");
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
tracing::warn!("failed to start control announce task: {err:#}");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
# ECP-0067: Control Transport Resolution And NixOS Control Wiring
|
||||||
|
|
||||||
|
Status: Draft
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Add two pieces on top of ECP-0066:
|
||||||
|
|
||||||
|
1. `ec-node control-resolve`:
|
||||||
|
- resolve a `stream_id` from iroh-gossip control announcements,
|
||||||
|
- enforce freshness (`updated_unix_ms` + TTL / max age),
|
||||||
|
- choose transport by policy (`direct-first`, `relay-first`, direct-only, relay-only),
|
||||||
|
- emit machine-readable JSON for automation.
|
||||||
|
|
||||||
|
2. Extend the `services.every-channel.ec-node` NixOS module with `control.*` options that map directly to `wt-publish --control-announce` flags.
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
We already announce relay/direct transport availability, but consumers and deployment automation still need ad-hoc logic to pick a path. `control-resolve` makes this deterministic and scriptable.
|
||||||
|
|
||||||
|
For ops, control announcements should be configured as immutable host state in Nix, not hand-managed CLI flags on each machine.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
In scope:
|
||||||
|
- New `control-resolve` command in `ec-node`.
|
||||||
|
- Freshness + transport-preference policy in resolver.
|
||||||
|
- NixOS module options for control announce enable/ttl/interval/discovery/identity/peers.
|
||||||
|
|
||||||
|
Out of scope:
|
||||||
|
- Browser-native iroh direct transport.
|
||||||
|
- End-to-end automatic failover execution (resolve + launch subscribe) in one command.
|
||||||
|
- Cryptographic policy hardening beyond current control-topic trust model.
|
||||||
|
|
||||||
|
## Rollout / Reversibility
|
||||||
|
|
||||||
|
- Additive only: existing relay and direct publish/subscribe paths remain unchanged.
|
||||||
|
- If needed, disable by not using `control-resolve` and leaving `services.every-channel.ec-node.control.enable = false`.
|
||||||
|
|
@ -107,6 +107,45 @@ in
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
control = {
|
||||||
|
enable = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = false;
|
||||||
|
description = "Enable iroh-gossip control announcements from each wt-publish service.";
|
||||||
|
};
|
||||||
|
|
||||||
|
ttlMs = lib.mkOption {
|
||||||
|
type = lib.types.ints.positive;
|
||||||
|
default = 15000;
|
||||||
|
description = "Control announcement TTL passed to `ec-node wt-publish --control-ttl-ms`.";
|
||||||
|
};
|
||||||
|
|
||||||
|
intervalMs = lib.mkOption {
|
||||||
|
type = lib.types.ints.positive;
|
||||||
|
default = 5000;
|
||||||
|
description = "Control announcement interval passed to `ec-node wt-publish --control-interval-ms`.";
|
||||||
|
};
|
||||||
|
|
||||||
|
discovery = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = null;
|
||||||
|
example = "dht,mdns,dns";
|
||||||
|
description = "Optional iroh discovery mode list for control announcements.";
|
||||||
|
};
|
||||||
|
|
||||||
|
irohSecret = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = null;
|
||||||
|
description = "Optional iroh secret key (hex) for control announcement identity.";
|
||||||
|
};
|
||||||
|
|
||||||
|
gossipPeers = lib.mkOption {
|
||||||
|
type = lib.types.listOf lib.types.str;
|
||||||
|
default = [ ];
|
||||||
|
description = "Optional iroh endpoint addresses to seed control gossip joins.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
broadcasts = lib.mkOption {
|
broadcasts = lib.mkOption {
|
||||||
type = lib.types.listOf (lib.types.submodule {
|
type = lib.types.listOf (lib.types.submodule {
|
||||||
options = {
|
options = {
|
||||||
|
|
@ -188,6 +227,7 @@ in
|
||||||
"cmd+=(${lib.concatStringsSep " " (map lib.escapeShellArg cfg.extraArgs)})";
|
"cmd+=(${lib.concatStringsSep " " (map lib.escapeShellArg cfg.extraArgs)})";
|
||||||
explicitInputStr = if b.input == null then "" else b.input;
|
explicitInputStr = if b.input == null then "" else b.input;
|
||||||
channelStr = if b.channel == null then "" else b.channel;
|
channelStr = if b.channel == null then "" else b.channel;
|
||||||
|
controlGossipPeerLines = lib.concatMapStrings (peer: "cmd+=(--gossip-peer ${lib.escapeShellArg peer})\n") cfg.control.gossipPeers;
|
||||||
in
|
in
|
||||||
''
|
''
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
@ -302,6 +342,14 @@ in
|
||||||
${lib.optionalString (!cfg.transcode) "cmd+=(--transcode=false)"}
|
${lib.optionalString (!cfg.transcode) "cmd+=(--transcode=false)"}
|
||||||
${lib.optionalString (!cfg.passthrough) "cmd+=(--passthrough=false)"}
|
${lib.optionalString (!cfg.passthrough) "cmd+=(--passthrough=false)"}
|
||||||
${lib.optionalString cfg.tlsDisableVerify "cmd+=(--tls-disable-verify)"}
|
${lib.optionalString cfg.tlsDisableVerify "cmd+=(--tls-disable-verify)"}
|
||||||
|
${lib.optionalString cfg.control.enable ''
|
||||||
|
cmd+=(--control-announce)
|
||||||
|
cmd+=(--control-ttl-ms ${toString cfg.control.ttlMs})
|
||||||
|
cmd+=(--control-interval-ms ${toString cfg.control.intervalMs})
|
||||||
|
${lib.optionalString (cfg.control.discovery != null) "cmd+=(--discovery ${lib.escapeShellArg cfg.control.discovery})"}
|
||||||
|
${lib.optionalString (cfg.control.irohSecret != null) "cmd+=(--iroh-secret ${lib.escapeShellArg cfg.control.irohSecret})"}
|
||||||
|
${controlGossipPeerLines}
|
||||||
|
''}
|
||||||
${extraArgsLine}
|
${extraArgsLine}
|
||||||
|
|
||||||
# Keep the unit alive even if the relay is temporarily unreachable.
|
# Keep the unit alive even if the relay is temporarily unreachable.
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue