Add duplicate publisher determinism proof

crates/ec-chopper/src/lib.rs

@@ -12,6 +12,7 @@ use ec_core::{
 };
 use ec_ts::{SectionAssembler, TimeSyncEngine, TimeSyncUpdate, TsReader};
 use serde::{Deserialize, Serialize};
+use std::collections::VecDeque;
 use std::fs;
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
@@ -299,12 +300,55 @@ pub fn chunk_ts_stream<T: Read>(
     })
 }
 
+pub fn chunk_ts_stream_with_preroll<T: Read>(
+    stream: T,
+    output_dir: &Path,
+    chunk_duration_ms: u64,
+    max_chunks: Option<usize>,
+    preroll_packets: usize,
+) -> Result<TsChunkManifest> {
+    let mut chunks = Vec::new();
+    chunk_ts_stream_live_with_preroll(
+        stream,
+        output_dir,
+        chunk_duration_ms,
+        max_chunks,
+        preroll_packets,
+        |chunk| {
+            chunks.push(chunk);
+            Ok(())
+        },
+    )?;
+    Ok(TsChunkManifest {
+        output_dir: output_dir.to_path_buf(),
+        chunks,
+    })
+}
+
 pub fn chunk_ts_stream_live<T: Read, F: FnMut(TsChunk) -> Result<()>>(
     stream: T,
     output_dir: &Path,
     chunk_duration_ms: u64,
     max_chunks: Option<usize>,
     mut on_chunk: F,
+) -> Result<()> {
+    chunk_ts_stream_live_with_preroll(
+        stream,
+        output_dir,
+        chunk_duration_ms,
+        max_chunks,
+        0,
+        |chunk| on_chunk(chunk),
+    )
+}
+
+pub fn chunk_ts_stream_live_with_preroll<T: Read, F: FnMut(TsChunk) -> Result<()>>(
+    stream: T,
+    output_dir: &Path,
+    chunk_duration_ms: u64,
+    max_chunks: Option<usize>,
+    preroll_packets: usize,
+    mut on_chunk: F,
 ) -> Result<()> {
     fs::create_dir_all(output_dir)
         .with_context(|| format!("failed to create {}", output_dir.display()))?;
@@ -317,6 +361,7 @@ pub fn chunk_ts_stream_live<T: Read, F: FnMut(TsChunk) -> Result<()>>(
     let mut current_file: Option<std::fs::File> = None;
     let mut current_timing: Option<ChunkTiming> = None;
     let mut emitted = 0usize;
+    let mut preroll = VecDeque::<[u8; ec_ts::TS_PACKET_SIZE]>::with_capacity(preroll_packets);
 
     let mut close_and_emit =
         |index: u64, timing: ChunkTiming, file: std::fs::File| -> Result<bool> {
@@ -332,6 +377,7 @@ pub fn chunk_ts_stream_live<T: Read, F: FnMut(TsChunk) -> Result<()>>(
         };
 
     while let Some(packet) = reader.read_packet()? {
+        let packet_bytes = *packet.as_bytes();
         let updates = engine.ingest_packet(&packet, &mut assembler);
         for update in updates {
             if update.discontinuity {
@@ -344,6 +390,7 @@ pub fn chunk_ts_stream_live<T: Read, F: FnMut(TsChunk) -> Result<()>>(
                         return Ok(());
                     }
                 }
+                preroll.clear();
             }
 
             if let Some(index) = update.chunk_index {
@@ -359,8 +406,11 @@ pub fn chunk_ts_stream_live<T: Read, F: FnMut(TsChunk) -> Result<()>>(
                     }
 
                     let path = chunk_path(output_dir, index);
-                    let file = std::fs::File::create(&path)
+                    let mut file = std::fs::File::create(&path)
                         .with_context(|| format!("failed to create {}", path.display()))?;
+                    for bytes in &preroll {
+                        file.write_all(bytes)?;
+                    }
                     current_file = Some(file);
                     current_index = Some(index);
                     current_timing = Some(ChunkTiming {
@@ -381,6 +431,13 @@ pub fn chunk_ts_stream_live<T: Read, F: FnMut(TsChunk) -> Result<()>>(
         if let Some(file) = current_file.as_mut() {
             file.write_all(packet.as_bytes())?;
         }
+
+        if preroll_packets > 0 {
+            preroll.push_back(packet_bytes);
+            while preroll.len() > preroll_packets {
+                preroll.pop_front();
+            }
+        }
     }
 
     if let (Some(index), Some(timing), Some(file)) = (
@@ -388,7 +445,7 @@ pub fn chunk_ts_stream_live<T: Read, F: FnMut(TsChunk) -> Result<()>>(
         current_timing.take(),
         current_file.take(),
     ) {
-        let _ = close_and_emit(index, timing, file);
+        close_and_emit(index, timing, file)?;
     }
 
     Ok(())
@@ -929,6 +986,43 @@ mod tests {
         let _ = fs::remove_dir_all(&dir);
     }
 
+    #[test]
+    fn chunk_ts_stream_with_preroll_prepends_previous_packets() {
+        let chunk_ms = 1000u64;
+        let dir =
+            std::env::temp_dir().join(format!("ec-chopper-chunks-preroll-{}", std::process::id()));
+        let _ = fs::remove_dir_all(&dir);
+        fs::create_dir_all(&dir).unwrap();
+
+        let packet0 = ts_packet_with_pcr(0x0100, 0, 0);
+        let packet1 = ts_packet_with_pcr(0x0100, 1, 27_000_000);
+        let packet2 = ts_packet_with_pcr(0x0100, 2, 54_000_000);
+        let mut bytes = Vec::new();
+        bytes.extend_from_slice(&packet0);
+        bytes.extend_from_slice(&packet1);
+        bytes.extend_from_slice(&packet2);
+
+        let manifest =
+            chunk_ts_stream_with_preroll(Cursor::new(bytes), &dir, chunk_ms, None, 1).unwrap();
+        let indices = manifest.chunks.iter().map(|c| c.index).collect::<Vec<_>>();
+        assert_eq!(indices, vec![0, 1, 2]);
+
+        assert_eq!(
+            fs::read(&manifest.chunks[0].path).unwrap(),
+            packet0.to_vec()
+        );
+        assert_eq!(
+            fs::read(&manifest.chunks[1].path).unwrap(),
+            [packet0, packet1].concat()
+        );
+        assert_eq!(
+            fs::read(&manifest.chunks[2].path).unwrap(),
+            [packet1, packet2].concat()
+        );
+
+        let _ = fs::remove_dir_all(&dir);
+    }
+
     #[test]
     fn hashed_manifest_merkle_root_matches_core() {
         let dir = std::env::temp_dir().join(format!("ec-chopper-merkle-{}", std::process::id()));

crates/ec-core/tests/simulation.rs

@@ -0,0 +1,986 @@
+use ec_core::sim::{
+    check_control_plane_propagation_invariants, check_duplicate_publisher_invariants,
+    check_system_duplicate_publisher_invariants, run_control_plane_propagation_campaign,
+    run_control_plane_propagation_simulation, run_duplicate_publisher_campaign,
+    run_duplicate_publisher_simulation, run_seeded_simulation_campaign,
+    run_system_duplicate_publisher_campaign, run_system_duplicate_publisher_simulation,
+    shrink_duplicate_publisher_failure, simulated_media_hash,
+    ControlPlanePropagationInvariantConfig, ControlPlanePropagationScenario,
+    ControlPlaneTraceEvent, DeterministicSimulation, DuplicatePublisherInvariantConfig,
+    DuplicatePublisherScenario, EncoderDriftFault, FoundationStyleSystemScenarioConfig,
+    PublisherSequenceClock, SimulationOutage, SimulationPartition, SimulationSeed,
+    SystemDuplicatePublisherInvariantConfig, SystemDuplicatePublisherScenario,
+};
+
+const STREAM: &str = "la-kcop";
+const RENDITION: &str = "720p";
+const TRACK: &str = "0.m4s";
+const PROFILE: &str = "x264-hd3-v1";
+
+fn schedule_publisher_window(
+    sim: &mut DeterministicSimulation,
+    node: &str,
+    start_sequence: u64,
+    end_sequence: u64,
+    first_delivery_ms: u64,
+    step_ms: u64,
+    profile: &str,
+) {
+    for sequence in start_sequence..end_sequence {
+        let hash = simulated_media_hash(STREAM, RENDITION, TRACK, sequence, profile);
+        sim.schedule_observation(
+            first_delivery_ms + (sequence - start_sequence) * step_ms,
+            node,
+            STREAM,
+            RENDITION,
+            TRACK,
+            sequence,
+            &hash,
+        );
+    }
+}
+
+#[test]
+fn duplicate_publishers_converge_after_delayed_backfill() {
+    let mut sim = DeterministicSimulation::new();
+
+    schedule_publisher_window(&mut sim, "nuc-a", 0, 12, 0, 10, PROFILE);
+    schedule_publisher_window(&mut sim, "nuc-b", 0, 4, 30, 10, PROFILE);
+    schedule_publisher_window(&mut sim, "nuc-b", 4, 12, 500, 10, PROFILE);
+
+    sim.run_until(250);
+    let before_backfill = sim.convergence().summarize(STREAM, RENDITION, TRACK, 0, 12);
+    assert_eq!(before_backfill.expected_sequences, 12);
+    assert_eq!(before_backfill.missing_sequences, Vec::<u64>::new());
+    assert_eq!(
+        before_backfill.matching_duplicate_sequences,
+        vec![0, 1, 2, 3]
+    );
+    assert!(before_backfill.ok());
+
+    sim.run_to_idle();
+    let after_backfill = sim.convergence().summarize(STREAM, RENDITION, TRACK, 0, 12);
+    let duplicate_complete_at_ms = sim
+        .convergence()
+        .duplicate_complete_at_ms(STREAM, RENDITION, TRACK, 0, 12);
+    assert_eq!(after_backfill.missing_sequences, Vec::<u64>::new());
+    assert_eq!(after_backfill.divergent_sequences, Vec::<u64>::new());
+    assert_eq!(
+        after_backfill.matching_duplicate_sequences,
+        (0_u64..12).collect::<Vec<_>>()
+    );
+    assert_eq!(after_backfill.duplicate_source_records, 24);
+    assert_eq!(duplicate_complete_at_ms, Some(570));
+    assert_eq!(sim.trace().len(), 24);
+    assert!(
+        sim.trace()
+            .windows(2)
+            .all(|pair| (pair[0].at_ms, pair[0].order) <= (pair[1].at_ms, pair[1].order)),
+        "trace should preserve deterministic event order"
+    );
+    assert!(after_backfill.ok());
+}
+
+#[test]
+fn media_convergence_can_summarize_sparse_observed_sequences() {
+    let mut sim = DeterministicSimulation::new();
+    for sequence in [7_287_381_184_512, 7_287_381_188_608] {
+        let hash = simulated_media_hash(STREAM, RENDITION, TRACK, sequence, PROFILE);
+        sim.schedule_observation(0, "nuc-a", STREAM, RENDITION, TRACK, sequence, &hash);
+        sim.schedule_observation(1, "nuc-b", STREAM, RENDITION, TRACK, sequence, &hash);
+    }
+
+    sim.run_to_idle();
+    let dense = sim.convergence().summarize(
+        STREAM,
+        RENDITION,
+        TRACK,
+        7_287_381_184_512,
+        7_287_381_188_609,
+    );
+    let sparse = sim.convergence().summarize_observed_sequences(
+        STREAM,
+        RENDITION,
+        TRACK,
+        7_287_381_184_512,
+        7_287_381_188_609,
+    );
+
+    assert!(!dense.missing_sequences.is_empty());
+    assert_eq!(sparse.expected_sequences, 2);
+    assert_eq!(sparse.missing_sequences, Vec::<u64>::new());
+    assert_eq!(
+        sparse.matching_duplicate_sequences,
+        vec![7_287_381_184_512, 7_287_381_188_608]
+    );
+    assert!(sparse.ok());
+}
+
+#[test]
+fn duplicate_publisher_simulation_detects_encoder_drift() {
+    let mut sim = DeterministicSimulation::new();
+
+    schedule_publisher_window(&mut sim, "nuc-a", 0, 8, 0, 10, PROFILE);
+    schedule_publisher_window(&mut sim, "nuc-b", 0, 8, 5, 10, PROFILE);
+
+    let drift_hash = simulated_media_hash(STREAM, RENDITION, TRACK, 4, "x264-hd3-drift");
+    sim.schedule_observation(90, "nuc-b", STREAM, RENDITION, TRACK, 4, &drift_hash);
+
+    sim.run_to_idle();
+    let summary = sim.convergence().summarize(STREAM, RENDITION, TRACK, 0, 8);
+
+    assert_eq!(summary.missing_sequences, Vec::<u64>::new());
+    assert_eq!(summary.divergent_sequences, vec![4]);
+    assert!(!summary.ok());
+}
+
+#[test]
+fn duplicate_publisher_fault_schedule_replays_from_seed() {
+    let scenario = faulted_duplicate_scenario(SimulationSeed::new(0x6d6f_712d_6475_7021));
+
+    let first = run_duplicate_publisher_simulation(&scenario);
+    let second = run_duplicate_publisher_simulation(&scenario);
+
+    assert_eq!(first, second);
+    assert!(first.duplicate_complete(), "replay {}", first.replay_hint);
+    assert_eq!(first.summary.matching_duplicate_sequences.len(), 48);
+    assert_eq!(
+        first.trace, second.trace,
+        "replayed reports should carry the same event history"
+    );
+}
+
+#[test]
+fn duplicate_publisher_many_seed_fault_schedules_converge() {
+    let mut saw_transient_drop = false;
+    let mut saw_partition_delay = false;
+    let mut saw_publisher_outage = false;
+
+    for seed in 1..=96 {
+        let scenario = faulted_duplicate_scenario(SimulationSeed::new(seed));
+        let report = run_duplicate_publisher_simulation(&scenario);
+
+        saw_transient_drop |= report.fault_stats.transient_dropped_observations > 0;
+        saw_partition_delay |= report.fault_stats.partition_delayed_observations > 0;
+        saw_publisher_outage |= report.fault_stats.publisher_outage_observations > 0;
+
+        assert!(
+            report.duplicate_complete(),
+            "duplicate publisher convergence failed for {}: {:?}",
+            report.replay_hint,
+            report.summary
+        );
+        assert_eq!(report.summary.missing_sequences, Vec::<u64>::new());
+        assert_eq!(report.summary.divergent_sequences, Vec::<u64>::new());
+        assert_eq!(report.summary.duplicate_source_records, 96);
+    }
+
+    assert!(
+        saw_transient_drop,
+        "fault suite did not exercise transient drops"
+    );
+    assert!(
+        saw_partition_delay,
+        "fault suite did not exercise partitions"
+    );
+    assert!(
+        saw_publisher_outage,
+        "fault suite did not exercise publisher outages"
+    );
+}
+
+#[test]
+fn seeded_fault_scenario_detects_encoder_drift() {
+    let mut scenario = faulted_duplicate_scenario(SimulationSeed::new(0x6472_6966_7421));
+    scenario
+        .encoder_drifts
+        .push(EncoderDriftFault::new("nuc-b", 17, "x264-hd3-drift"));
+
+    let report = run_duplicate_publisher_simulation(&scenario);
+
+    assert!(!report.duplicate_complete());
+    assert_eq!(report.summary.divergent_sequences, vec![17]);
+    assert_eq!(report.duplicate_complete_at_ms, None);
+    assert_eq!(report.fault_stats.encoder_drift_observations, 1);
+}
+
+#[test]
+fn duplicate_publisher_simulation_detects_unaligned_publisher_phase() {
+    let mut scenario = DuplicatePublisherScenario::new(
+        SimulationSeed::new(0x7068_6173_652d_6275),
+        vec!["nuc-a".to_string(), "nuc-b".to_string()],
+        STREAM,
+        RENDITION,
+        TRACK,
+        PROFILE,
+        0,
+        8,
+    );
+    scenario.base_network_delay_ms = 0;
+    scenario.max_jitter_ms = 0;
+    scenario
+        .publisher_sequence_offsets
+        .insert("nuc-b".to_string(), 3);
+
+    let report = run_duplicate_publisher_simulation(&scenario);
+    let invariant = check_duplicate_publisher_invariants(
+        &report,
+        &DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(1_000),
+    );
+
+    assert!(!report.duplicate_complete());
+    assert_eq!(report.summary.missing_sequences, Vec::<u64>::new());
+    assert_eq!(
+        report.summary.matching_duplicate_sequences,
+        Vec::<u64>::new()
+    );
+    assert_eq!(
+        report.summary.divergent_sequences,
+        (0_u64..8).collect::<Vec<_>>()
+    );
+    assert_eq!(report.fault_stats.publisher_phase_offset_observations, 8);
+    assert_eq!(
+        invariant.failures,
+        vec![
+            "divergent_sequences".to_string(),
+            "media_timing_conflict_sequences".to_string(),
+            "duplicate_incomplete".to_string(),
+            "duplicate_complete_deadline_unreached".to_string(),
+        ]
+    );
+}
+
+#[test]
+fn duplicate_publisher_simulation_rejects_missing_media_timing() {
+    let mut scenario = DuplicatePublisherScenario::new(
+        SimulationSeed::new(0x7469_6d65_2d6d_6973),
+        vec!["nuc-a".to_string(), "nuc-b".to_string()],
+        STREAM,
+        RENDITION,
+        TRACK,
+        PROFILE,
+        0,
+        6,
+    );
+    scenario.base_network_delay_ms = 0;
+    scenario.max_jitter_ms = 0;
+    scenario
+        .missing_media_timing_publishers
+        .insert("nuc-b".to_string());
+
+    let report = run_duplicate_publisher_simulation(&scenario);
+    let invariant = check_duplicate_publisher_invariants(
+        &report,
+        &DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(1_000),
+    );
+
+    assert_eq!(report.summary.divergent_sequences, Vec::<u64>::new());
+    assert_eq!(report.summary.media_timing_missing_records, 6);
+    assert_eq!(
+        invariant.failures,
+        vec![
+            "media_timing_missing_records".to_string(),
+            "duplicate_incomplete".to_string(),
+            "duplicate_complete_deadline_unreached".to_string(),
+        ]
+    );
+}
+
+#[test]
+fn duplicate_publisher_simulation_rejects_conflicting_media_timing() {
+    let mut scenario = DuplicatePublisherScenario::new(
+        SimulationSeed::new(0x7469_6d65_2d73_6b65),
+        vec!["nuc-a".to_string(), "nuc-b".to_string()],
+        STREAM,
+        RENDITION,
+        TRACK,
+        PROFILE,
+        0,
+        6,
+    );
+    scenario.base_network_delay_ms = 0;
+    scenario.max_jitter_ms = 0;
+    scenario
+        .publisher_media_time_offsets_ms
+        .insert("nuc-b".to_string(), 17);
+
+    let report = run_duplicate_publisher_simulation(&scenario);
+    let invariant = check_duplicate_publisher_invariants(
+        &report,
+        &DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(1_000),
+    );
+
+    assert_eq!(report.summary.divergent_sequences, Vec::<u64>::new());
+    assert_eq!(
+        report.summary.media_timing_conflict_sequences,
+        (0_u64..6).collect::<Vec<_>>()
+    );
+    assert_eq!(
+        invariant.failures,
+        vec![
+            "media_timing_conflict_sequences".to_string(),
+            "duplicate_incomplete".to_string(),
+            "duplicate_complete_deadline_unreached".to_string(),
+        ]
+    );
+}
+
+#[test]
+fn duplicate_publisher_simulation_rejects_independent_source_material() {
+    let mut scenario = DuplicatePublisherScenario::new(
+        SimulationSeed::new(0x736f_7572_6365_6d61),
+        vec!["nuc-a".to_string(), "nuc-b".to_string()],
+        STREAM,
+        RENDITION,
+        TRACK,
+        PROFILE,
+        0,
+        6,
+    );
+    scenario.base_network_delay_ms = 0;
+    scenario.max_jitter_ms = 0;
+    scenario
+        .publisher_source_material
+        .insert("nuc-b".to_string(), "independent-rf-window".to_string());
+
+    let report = run_duplicate_publisher_simulation(&scenario);
+    let invariant = check_duplicate_publisher_invariants(
+        &report,
+        &DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(1_000),
+    );
+
+    assert_eq!(
+        report.summary.divergent_sequences,
+        (0_u64..6).collect::<Vec<_>>()
+    );
+    assert_eq!(
+        report.summary.media_timing_conflict_sequences,
+        Vec::<u64>::new()
+    );
+    assert_eq!(report.fault_stats.source_material_mismatch_observations, 12);
+    assert_eq!(
+        invariant.failures,
+        vec![
+            "divergent_sequences".to_string(),
+            "source_material_mismatch_observations".to_string(),
+            "duplicate_incomplete".to_string(),
+            "duplicate_complete_deadline_unreached".to_string(),
+        ]
+    );
+}
+
+#[test]
+fn duplicate_publisher_outage_backfills_after_restart() {
+    let mut scenario = faulted_duplicate_scenario(SimulationSeed::new(0x6f75_7461_6765));
+    scenario.partitions.clear();
+    scenario.transient_drop_per_million = 0;
+    scenario.publisher_outages = vec![SimulationOutage::new("nuc-b", 320, 760, 180)];
+
+    let report = run_duplicate_publisher_simulation(&scenario);
+
+    assert!(
+        report.duplicate_complete(),
+        "{} {:?}",
+        report.replay_hint,
+        report.summary
+    );
+    assert!(report.fault_stats.publisher_outage_observations > 0);
+    assert_eq!(
+        report.fault_stats.backfill_observations,
+        report.fault_stats.publisher_outage_observations
+    );
+    assert!(
+        report.duplicate_complete_at_ms.unwrap() >= 940,
+        "outage restart should move convergence later than the live path"
+    );
+    assert!(report.duplicate_complete_at_ms.unwrap() <= 3_000);
+}
+
+#[test]
+fn duplicate_publisher_simulation_checks_convergence_deadline() {
+    let report = run_duplicate_publisher_simulation(&faulted_duplicate_scenario(
+        SimulationSeed::new(0x6465_6164_6c69_6e65),
+    ));
+    let invariant = check_duplicate_publisher_invariants(
+        &report,
+        &DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(3_000),
+    );
+
+    assert!(
+        invariant.ok,
+        "{} {:?}",
+        invariant.replay_hint, invariant.failures
+    );
+    assert!(invariant.duplicate_complete_at_ms.is_some());
+    assert!(
+        invariant.duplicate_complete_at_ms.unwrap() <= 3_000,
+        "{} completed too late: {:?}",
+        invariant.replay_hint,
+        invariant.duplicate_complete_at_ms
+    );
+}
+
+#[test]
+fn seeded_simulation_campaign_preserves_first_failure() {
+    let campaign = run_seeded_simulation_campaign(
+        "generic-seeded-campaign",
+        SimulationSeed::new(40),
+        8,
+        |seed| (seed.0 == 44).then_some(seed.replay_hint()),
+    );
+
+    assert!(!campaign.all_passed());
+    assert_eq!(campaign.passed, 7);
+    assert_eq!(campaign.failed, 1);
+    assert_eq!(
+        campaign.first_failure.as_deref(),
+        Some("EC_SIM_SEED=000000000000002c")
+    );
+}
+
+#[test]
+fn control_plane_propagation_replays_from_seed() {
+    let scenario = faulted_control_plane_scenario(SimulationSeed::new(0x6374_726c_7265_706c));
+
+    let first = run_control_plane_propagation_simulation(&scenario);
+    let second = run_control_plane_propagation_simulation(&scenario);
+
+    assert_eq!(first, second);
+    assert!(
+        first.propagation_complete(),
+        "control propagation failed for {}: {:?}",
+        first.replay_hint,
+        first.missing_nodes
+    );
+    assert_eq!(first.known_count, scenario.nodes.len() as u64);
+    assert_eq!(
+        first.trace, second.trace,
+        "replayed control-plane schedules should carry identical traces"
+    );
+    assert!(first
+        .trace
+        .iter()
+        .any(|entry| matches!(entry.event, ControlPlaneTraceEvent::MessageScheduled { .. })));
+    assert!(first
+        .trace
+        .iter()
+        .any(|entry| matches!(entry.event, ControlPlaneTraceEvent::NodeLearned { .. })));
+}
+
+#[test]
+fn control_plane_campaign_runs_many_fault_schedules() {
+    let invariant = ControlPlanePropagationInvariantConfig::complete_with_deadline(7, 900);
+    let campaign = run_control_plane_propagation_campaign(
+        "control-plane-gossip-fault-campaign",
+        SimulationSeed::new(1),
+        512,
+        &invariant,
+        faulted_control_plane_scenario,
+    );
+
+    assert!(
+        campaign.all_passed(),
+        "campaign failed: {:?}",
+        campaign.first_failure
+    );
+    assert_eq!(campaign.passed, 512);
+    assert_eq!(campaign.failed, 0);
+    assert!(campaign.total_transient_dropped_messages > 0);
+    assert!(campaign.total_partition_delayed_messages > 0);
+    assert!(campaign.total_node_outage_delayed_messages > 0);
+    assert!(campaign.total_duplicate_messages > 0);
+    assert!(campaign.max_propagation_complete_ms_observed <= 900);
+}
+
+#[test]
+fn control_plane_simulation_detects_dead_fanout() {
+    let mut scenario = faulted_control_plane_scenario(SimulationSeed::new(0x6661_6e6f_7574));
+    scenario.fanout = 0;
+    scenario.transient_drop_per_million = 0;
+    scenario.partitions.clear();
+    scenario.node_outages.clear();
+
+    let report = run_control_plane_propagation_simulation(&scenario);
+    let invariant = check_control_plane_propagation_invariants(
+        &report,
+        &ControlPlanePropagationInvariantConfig::complete_with_deadline(7, 900),
+    );
+
+    assert!(!report.propagation_complete());
+    assert_eq!(report.known_nodes, vec!["nuc-a".to_string()]);
+    assert_eq!(report.missing_nodes.len(), 6);
+    assert_eq!(
+        invariant.failures,
+        vec![
+            "propagation_incomplete".to_string(),
+            "propagation_deadline_unreached".to_string(),
+        ]
+    );
+}
+
+#[test]
+fn duplicate_publisher_campaign_runs_many_seed_schedules() {
+    let invariant = DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(3_000);
+    let campaign = run_duplicate_publisher_campaign(
+        "duplicate-publisher-fault-campaign",
+        SimulationSeed::new(1),
+        512,
+        &invariant,
+        faulted_duplicate_scenario,
+    );
+
+    assert!(
+        campaign.all_passed(),
+        "campaign failed: {:?}",
+        campaign.first_failure
+    );
+    assert_eq!(campaign.passed, 512);
+    assert_eq!(campaign.failed, 0);
+    assert!(campaign.total_transient_dropped_observations > 0);
+    assert!(campaign.total_partition_delayed_observations > 0);
+    assert!(campaign.total_publisher_outage_observations > 0);
+    assert!(campaign.total_backfill_observations > 0);
+    assert!(campaign.max_duplicate_complete_ms_observed <= 3_000);
+}
+
+#[test]
+fn duplicate_publisher_shrinker_minimizes_noisy_drift_failure() {
+    let invariant = DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(3_000);
+    let mut scenario = faulted_duplicate_scenario(SimulationSeed::new(19));
+    scenario
+        .encoder_drifts
+        .push(EncoderDriftFault::new("nuc-b", 17, "x264-hd3-drift"));
+
+    let shrunk = shrink_duplicate_publisher_failure(&scenario, &invariant)
+        .expect("drift should fail and be shrinkable");
+
+    assert_eq!(shrunk.seed, SimulationSeed::new(19));
+    assert_eq!(shrunk.scenario.expected_sequences(), 18);
+    assert_eq!(shrunk.scenario.partitions.len(), 0);
+    assert_eq!(shrunk.scenario.publisher_outages.len(), 0);
+    assert_eq!(shrunk.scenario.transient_drop_per_million, 0);
+    assert_eq!(shrunk.scenario.max_jitter_ms, 0);
+    assert_eq!(shrunk.scenario.base_network_delay_ms, 0);
+    assert_eq!(shrunk.report.summary.divergent_sequences, vec![17]);
+    assert_eq!(
+        shrunk.invariant.failures,
+        vec![
+            "divergent_sequences".to_string(),
+            "duplicate_incomplete".to_string(),
+            "duplicate_complete_deadline_unreached".to_string(),
+        ]
+    );
+    assert!(
+        shrunk
+            .steps
+            .iter()
+            .any(|step| step.dimension == "sequence_count" && step.after == "18"),
+        "shrink steps should record the minimized failing media window"
+    );
+}
+
+#[test]
+fn duplicate_publisher_campaign_keeps_first_replayable_failure() {
+    let invariant = DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(3_000);
+    let campaign = run_duplicate_publisher_campaign(
+        "duplicate-publisher-replayable-failure",
+        SimulationSeed::new(10),
+        32,
+        &invariant,
+        |seed| {
+            let mut scenario = faulted_duplicate_scenario(seed);
+            if seed.0 == 19 {
+                scenario
+                    .encoder_drifts
+                    .push(EncoderDriftFault::new("nuc-b", 17, "x264-hd3-drift"));
+            }
+            scenario
+        },
+    );
+
+    let failure = campaign
+        .first_failure
+        .as_ref()
+        .expect("campaign should preserve first failure");
+    let shrunk = failure
+        .shrunk_failure
+        .as_ref()
+        .expect("campaign should preserve a shrunk replay");
+    assert_eq!(failure.seed, SimulationSeed::new(19));
+    assert_eq!(
+        failure.invariant.failures,
+        vec![
+            "divergent_sequences".to_string(),
+            "duplicate_incomplete".to_string(),
+            "duplicate_complete_deadline_unreached".to_string(),
+        ]
+    );
+
+    let mut replay = faulted_duplicate_scenario(failure.seed);
+    replay
+        .encoder_drifts
+        .push(EncoderDriftFault::new("nuc-b", 17, "x264-hd3-drift"));
+    let replayed_report = run_duplicate_publisher_simulation(&replay);
+    assert_eq!(replayed_report, failure.report);
+    assert_eq!(shrunk.scenario.expected_sequences(), 18);
+    assert_eq!(shrunk.report.summary.divergent_sequences, vec![17]);
+}
+
+#[test]
+fn system_duplicate_publishers_converge_with_global_sequence_clock() {
+    let scenario = system_duplicate_scenario(
+        SimulationSeed::new(0x7379_7374_656d_676c),
+        PublisherSequenceClock::Global,
+    );
+
+    let report = run_system_duplicate_publisher_simulation(&scenario);
+    let invariant = check_system_duplicate_publisher_invariants(
+        &report,
+        &SystemDuplicatePublisherInvariantConfig::complete_with_deadline(3_500),
+    );
+
+    assert!(
+        report.system_complete(),
+        "{} control={:?} media={:?}",
+        report.replay_hint,
+        report.control.missing_nodes,
+        report.media.summary
+    );
+    assert!(
+        invariant.ok,
+        "{} {:?}",
+        invariant.replay_hint, invariant.failures
+    );
+    assert_eq!(report.media.summary.divergent_sequences, Vec::<u64>::new());
+    assert_eq!(
+        report.media.summary.matching_duplicate_sequences.len() as u64,
+        scenario.media.expected_sequences()
+    );
+    assert!(
+        report
+            .publisher_activation_ms
+            .get("nuc-b")
+            .copied()
+            .unwrap_or_default()
+            > report
+                .publisher_activation_ms
+                .get("nuc-a")
+                .copied()
+                .unwrap_or_default(),
+        "faulted control plane should activate nuc-b later than nuc-a"
+    );
+}
+
+#[test]
+fn system_duplicate_publishers_reject_local_activation_sequence_clock() {
+    let scenario = system_duplicate_scenario(
+        SimulationSeed::new(0x7379_7374_656d_6c6f),
+        PublisherSequenceClock::LocalActivation,
+    );
+
+    let report = run_system_duplicate_publisher_simulation(&scenario);
+    let invariant = check_system_duplicate_publisher_invariants(
+        &report,
+        &SystemDuplicatePublisherInvariantConfig::complete_with_deadline(3_500),
+    );
+
+    assert!(report.control.propagation_complete());
+    assert!(!report.media.duplicate_complete());
+    assert!(
+        !report.media.summary.divergent_sequences.is_empty(),
+        "local activation clock should cause same advertised sequence to hash differently"
+    );
+    assert_eq!(
+        invariant.failures,
+        vec![
+            "media_divergent_sequences".to_string(),
+            "media_timing_conflict_sequences".to_string(),
+            "media_duplicate_incomplete".to_string(),
+            "system_complete_deadline_unreached".to_string(),
+        ]
+    );
+}
+
+#[test]
+fn system_duplicate_publisher_campaign_runs_many_seed_schedules() {
+    let invariant = SystemDuplicatePublisherInvariantConfig::complete_with_deadline(3_500);
+    let campaign = run_system_duplicate_publisher_campaign(
+        "system-duplicate-publisher-fault-campaign",
+        SimulationSeed::new(1),
+        256,
+        &invariant,
+        |seed| system_duplicate_scenario(seed, PublisherSequenceClock::Global),
+    );
+
+    assert!(
+        campaign.all_passed(),
+        "campaign failed: {:?}",
+        campaign.first_failure
+    );
+    assert_eq!(campaign.passed, 256);
+    assert_eq!(campaign.failed, 0);
+    assert!(campaign.max_control_propagation_ms_observed > 0);
+    assert!(campaign.max_media_duplicate_complete_ms_observed > 0);
+    assert!(campaign.max_system_complete_ms_observed <= 3_500);
+    assert!(campaign.total_system_complete_ms_observed > 0);
+    assert!(campaign.total_control_trace_events > 0);
+    assert!(campaign.total_media_trace_events > 0);
+    assert_eq!(
+        campaign.total_trace_events,
+        campaign.total_control_trace_events + campaign.total_media_trace_events
+    );
+    assert!(campaign.total_control_transient_drops > 0);
+    assert!(campaign.total_media_transient_drops > 0);
+    assert!(campaign.total_media_backfill_observations > 0);
+    assert!(campaign.seeds_with_system_convergence_time > 0);
+    assert!(campaign.seeds_with_control_transient_drops > 0);
+    assert!(campaign.seeds_with_media_transient_drops > 0);
+    assert!(campaign.seeds_with_media_backfill_observations > 0);
+    assert!(!campaign.slowest_system_runs.is_empty());
+    assert!(campaign.slowest_system_runs.len() <= 16);
+    assert!(campaign
+        .slowest_system_runs
+        .windows(2)
+        .all(|pair| pair[0].system_complete_at_ms.unwrap_or(u64::MAX)
+            >= pair[1].system_complete_at_ms.unwrap_or(u64::MAX)));
+    assert_eq!(campaign.total_media_publisher_phase_offsets, 0);
+}
+
+#[test]
+fn foundation_style_system_campaign_runs_replayable_fault_schedules() {
+    let invariant = SystemDuplicatePublisherInvariantConfig::complete_with_deadline(6_000);
+    let config = FoundationStyleSystemScenarioConfig::default();
+    let campaign = run_system_duplicate_publisher_campaign(
+        "foundation-style-system-campaign",
+        SimulationSeed::new(1),
+        512,
+        &invariant,
+        |seed| ec_core::sim::foundation_style_system_duplicate_publisher_scenario(seed, &config),
+    );
+
+    assert!(
+        campaign.all_passed(),
+        "campaign failed: {:?}",
+        campaign.first_failure
+    );
+    assert_eq!(campaign.passed, 512);
+    assert_eq!(campaign.failed, 0);
+    assert!(campaign.max_system_complete_ms_observed <= 6_000);
+    assert!(campaign.total_system_complete_ms_observed > 0);
+    assert!(campaign.total_control_trace_events > 0);
+    assert!(campaign.total_media_trace_events > 0);
+    assert_eq!(
+        campaign.total_trace_events,
+        campaign.total_control_trace_events + campaign.total_media_trace_events
+    );
+    assert!(campaign.total_control_transient_drops > 0);
+    assert!(campaign.total_control_partition_delays > 0);
+    assert!(campaign.total_control_node_outage_delays > 0);
+    assert!(campaign.total_control_duplicate_messages > 0);
+    assert!(campaign.total_media_transient_drops > 0);
+    assert!(campaign.total_media_partition_delays > 0);
+    assert!(campaign.total_media_publisher_outages > 0);
+    assert!(campaign.total_media_backfill_observations > 0);
+    assert!(campaign.seeds_with_system_convergence_time > 0);
+    assert!(campaign.seeds_with_control_propagation_time > 0);
+    assert!(campaign.seeds_with_media_duplicate_convergence_time > 0);
+    assert!(campaign.seeds_with_control_transient_drops > 0);
+    assert!(campaign.seeds_with_control_partition_delays > 0);
+    assert!(campaign.seeds_with_control_node_outage_delays > 0);
+    assert!(campaign.seeds_with_control_duplicate_messages > 0);
+    assert!(campaign.seeds_with_media_transient_drops > 0);
+    assert!(campaign.seeds_with_media_partition_delays > 0);
+    assert!(campaign.seeds_with_media_publisher_outages > 0);
+    assert!(campaign.seeds_with_media_backfill_observations > 0);
+    assert!(campaign.fault_coverage_ok());
+    assert!(!campaign.slowest_system_runs.is_empty());
+    assert!(campaign.slowest_system_runs.len() <= 16);
+    assert_eq!(campaign.total_media_publisher_phase_offsets, 0);
+}
+
+#[test]
+fn foundation_style_system_campaign_rejects_local_activation_sequence_clock() {
+    let invariant = SystemDuplicatePublisherInvariantConfig::complete_with_deadline(6_000);
+    let mut config = FoundationStyleSystemScenarioConfig::default();
+    config.sequence_clock = PublisherSequenceClock::LocalActivation;
+    let campaign = run_system_duplicate_publisher_campaign(
+        "foundation-style-local-activation-failure",
+        SimulationSeed::new(1),
+        32,
+        &invariant,
+        |seed| ec_core::sim::foundation_style_system_duplicate_publisher_scenario(seed, &config),
+    );
+
+    let failure = campaign
+        .first_failure
+        .as_ref()
+        .expect("local activation clock should fail under foundation-style faults");
+
+    assert!(!campaign.all_passed());
+    assert!(failure
+        .invariant
+        .failures
+        .contains(&"media_divergent_sequences".to_string()));
+    assert!(!failure.report.media.summary.divergent_sequences.is_empty());
+    assert!(
+        failure
+            .report
+            .media
+            .fault_stats
+            .publisher_phase_offset_observations
+            > 0
+    );
+    assert!(campaign.total_media_publisher_phase_offsets > 0);
+    assert!(campaign.seeds_with_media_publisher_phase_offsets > 0);
+}
+
+#[test]
+fn system_duplicate_publisher_campaign_classifies_source_material_mismatch() {
+    let invariant = SystemDuplicatePublisherInvariantConfig::complete_with_deadline(3_500);
+    let campaign = run_system_duplicate_publisher_campaign(
+        "system-source-material-failure",
+        SimulationSeed::new(1),
+        1,
+        &invariant,
+        |seed| {
+            let mut scenario = system_duplicate_scenario(seed, PublisherSequenceClock::Global);
+            scenario
+                .media
+                .publisher_source_material
+                .insert("nuc-b".to_string(), "independent-rf-window".to_string());
+            scenario
+        },
+    );
+
+    let failure = campaign
+        .first_failure
+        .as_ref()
+        .expect("source material mismatch should fail");
+
+    assert!(!campaign.all_passed());
+    assert!(failure
+        .invariant
+        .failures
+        .contains(&"media_source_material_mismatch_observations".to_string()));
+    assert!(!failure.report.media.summary.divergent_sequences.is_empty());
+    assert!(
+        failure
+            .report
+            .media
+            .fault_stats
+            .source_material_mismatch_observations
+            > 0
+    );
+    assert!(campaign.total_media_source_material_mismatches > 0);
+    assert_eq!(campaign.seeds_with_media_source_material_mismatches, 1);
+}
+
+fn faulted_duplicate_scenario(seed: SimulationSeed) -> DuplicatePublisherScenario {
+    let mut scenario = DuplicatePublisherScenario::new(
+        seed,
+        vec!["nuc-a".to_string(), "nuc-b".to_string()],
+        STREAM,
+        RENDITION,
+        TRACK,
+        PROFILE,
+        0,
+        48,
+    );
+    scenario.segment_step_ms = 40;
+    scenario.base_network_delay_ms = 5;
+    scenario.max_jitter_ms = 75;
+    scenario.transient_drop_per_million = 275_000;
+    scenario.backfill_after_ms = 600;
+    scenario.partitions = vec![
+        SimulationPartition::new("nuc-b", 120, 520, 140),
+        SimulationPartition::new("nuc-a", 940, 1_260, 90),
+    ];
+    scenario.publisher_outages = vec![SimulationOutage::new("nuc-b", 1_360, 1_520, 220)];
+    scenario
+}
+
+fn faulted_control_plane_scenario(seed: SimulationSeed) -> ControlPlanePropagationScenario {
+    let mut scenario = ControlPlanePropagationScenario::new(
+        seed,
+        vec![
+            "nuc-a".to_string(),
+            "nuc-b".to_string(),
+            "tower".to_string(),
+            "forge".to_string(),
+            "relay-lax".to_string(),
+            "relay-nyc".to_string(),
+            "relay-hel".to_string(),
+        ],
+        "nuc-a",
+        "ec.control.broadcast.la-kcop",
+        "la-kcop@42",
+    );
+    scenario.fanout = 3;
+    scenario.gossip_interval_ms = 35;
+    scenario.max_gossip_rounds = 12;
+    scenario.base_network_delay_ms = 6;
+    scenario.max_jitter_ms = 45;
+    scenario.transient_drop_per_million = 120_000;
+    scenario.partitions = vec![
+        SimulationPartition::new("relay-hel", 70, 190, 55),
+        SimulationPartition::new("tower", 220, 310, 40),
+    ];
+    scenario.node_outages = vec![SimulationOutage::new("relay-nyc", 105, 205, 45)];
+    scenario
+}
+
+fn system_duplicate_scenario(
+    seed: SimulationSeed,
+    sequence_clock: PublisherSequenceClock,
+) -> SystemDuplicatePublisherScenario {
+    let mut control = ControlPlanePropagationScenario::new(
+        seed,
+        vec![
+            "forge".to_string(),
+            "nuc-a".to_string(),
+            "nuc-b".to_string(),
+            "tower".to_string(),
+            "relay-lax".to_string(),
+            "relay-nyc".to_string(),
+            "relay-hel".to_string(),
+        ],
+        "forge",
+        "ec.control.broadcast.la-kcop",
+        "la-kcop@42",
+    );
+    control.fanout = 3;
+    control.gossip_interval_ms = 35;
+    control.max_gossip_rounds = 12;
+    control.base_network_delay_ms = 6;
+    control.max_jitter_ms = 45;
+    control.transient_drop_per_million = 120_000;
+    control.partitions = vec![
+        SimulationPartition::new("nuc-b", 0, 180, 40),
+        SimulationPartition::new("relay-hel", 70, 190, 55),
+    ];
+    control.node_outages = vec![SimulationOutage::new("relay-nyc", 105, 205, 45)];
+
+    let mut media = DuplicatePublisherScenario::new(
+        SimulationSeed::new(seed.0 ^ 0x6d65_6469_6121),
+        vec!["nuc-a".to_string(), "nuc-b".to_string()],
+        STREAM,
+        RENDITION,
+        TRACK,
+        PROFILE,
+        0,
+        48,
+    );
+    media.segment_step_ms = 40;
+    media.base_network_delay_ms = 5;
+    media.max_jitter_ms = 75;
+    media.transient_drop_per_million = 275_000;
+    media.backfill_after_ms = 600;
+    media.partitions = vec![SimulationPartition::new("nuc-a", 940, 1_260, 90)];
+    media.publisher_outages = vec![SimulationOutage::new("nuc-b", 1_360, 1_520, 220)];
+
+    let mut scenario = SystemDuplicatePublisherScenario::new(seed, control, media);
+    scenario.publisher_activation_delay_ms = 25;
+    scenario.publisher_backfill_delay_ms = 180;
+    scenario.sequence_clock = sequence_clock;
+    scenario
+}

crates/ec-node/Cargo.toml

@@ -29,17 +29,21 @@ rustls-native-certs = "0.8.3"
 urlencoding = "2"
 serde.workspace = true
 serde_json.workspace = true
+opentelemetry.workspace = true
+opentelemetry-otlp.workspace = true
+opentelemetry_sdk.workspace = true
 tokio = { version = "1", features = ["full"] }
 tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] }
 futures-util = "0.3"
 tracing.workspace = true
+tracing-opentelemetry.workspace = true
 tracing-subscriber.workspace = true
-web-transport-quinn = "0.11.4"
-web-transport-trait = "0.3.3"
-hang = "0.14.0"
-moq-mux = "0.2.1"
-moq-lite = "0.14.0"
-moq-native = { version = "0.13.1", default-features = true }
+web-transport-quinn = "0.11.9"
+web-transport-trait = "0.3.4"
+hang = "0.16.0"
+moq-mux = "0.4.0"
+moq-lite = "0.16.0"
+moq-native = { version = "0.14.0", default-features = true }
 headless_chrome = "1"
 tokio-util = "0.7"
 url = "2"

crates/ec-node/src/nbc.rs

@@ -2,7 +2,7 @@ use anyhow::{anyhow, Context, Result};
 use headless_chrome::protocol::cdp::Page;
 use headless_chrome::{Browser, Tab};
 use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::env;
 use std::fs;
 use std::io::{BufRead, BufReader, Cursor, Read, Write};
@@ -65,11 +65,15 @@ pub struct BootstrapResult {
     pub page_url: String,
     pub interactive_auth_required: bool,
     pub authorized: bool,
+    pub video_ready: bool,
+    pub current_time: f64,
+    pub width: u64,
+    pub height: u64,
     pub screenshot_path: Option<PathBuf>,
 }
 
-#[derive(Debug)]
 struct WaitOutcome {
+    tab: Arc<Tab>,
     state: NbcVideoState,
     trace: NbcTraceState,
     interactive_auth_required: bool,
@@ -199,6 +203,14 @@ fn nbc_bootstrap_timeout() -> Duration {
         .unwrap_or_else(|| Duration::from_secs(1800))
 }
 
+fn nbc_profile_signin_gate_timeout() -> Duration {
+    env::var("EVERY_CHANNEL_NBC_PROFILE_SIGNIN_GATE_TIMEOUT_SECS")
+        .ok()
+        .and_then(|value| value.parse::<u64>().ok())
+        .map(Duration::from_secs)
+        .unwrap_or_else(|| Duration::from_secs(8))
+}
+
 fn nbc_env_flag(name: &str) -> Option<bool> {
     env::var(name).ok().map(|value| {
         let value = value.trim().to_ascii_lowercase();
@@ -403,6 +415,10 @@ pub fn bootstrap_nbc_auth(
     )?;
 
     Ok(BootstrapResult {
+        video_ready: nbc_video_state_has_decoded_frame(&outcome.state),
+        current_time: outcome.state.current_time,
+        width: outcome.state.width,
+        height: outcome.state.height,
         title: outcome.state.title,
         page_url: outcome.state.page_url,
         interactive_auth_required: outcome.interactive_auth_required,
@@ -619,7 +635,7 @@ fn run_nbc_capture_loop(
         register_nbc_trace_handlers(&tab, trace.clone())?;
         tab.navigate_to(&url)?;
         tab.wait_until_navigated()?;
-        wait_for_nbc_playback(
+        let outcome = wait_for_nbc_playback(
             chrome.browser(),
             &tab,
             &url,
@@ -627,31 +643,34 @@ fn run_nbc_capture_loop(
             AuthMode::Forbidden,
             None,
         )?;
+        let capture_tab = outcome.tab;
 
         let frame_interval = Duration::from_millis(1000 / nbc_capture_fps().max(1));
         let quality = nbc_capture_quality();
         let mut first_frame = true;
 
         loop {
-            kick_nbc_player(&tab).ok();
-            let frame = tab
+            kick_nbc_player(&capture_tab).ok();
+            let state = probe_nbc_video(&capture_tab).unwrap_or_default();
+            if !nbc_video_state_has_decoded_frame(&state) {
+                return Err(anyhow!(
+                    "NBC capture tab lost decoded video (title='{}', page_url='{}', current_time={}, ready_state={}, has_video={})",
+                    state.title,
+                    state.page_url,
+                    state.current_time,
+                    state.ready_state,
+                    state.has_video,
+                ));
+            }
+            let video = capture_tab
                 .find_element("video")
-                .and_then(|video| {
-                    video.parent.capture_screenshot(
-                        Page::CaptureScreenshotFormatOption::Jpeg,
-                        Some(quality),
-                        Some(video.get_box_model()?.content_viewport()),
-                        true,
-                    )
-                })
-                .or_else(|_| {
-                    tab.capture_screenshot(
-                        Page::CaptureScreenshotFormatOption::Jpeg,
-                        Some(quality),
-                        None,
-                        true,
-                    )
-                })?;
+                .context("NBC capture tab has no video element after playback readiness")?;
+            let frame = video.parent.capture_screenshot(
+                Page::CaptureScreenshotFormatOption::Jpeg,
+                Some(quality),
+                Some(video.get_box_model()?.content_viewport()),
+                true,
+            )?;
 
             if first_frame {
                 first_frame = false;
@@ -785,6 +804,15 @@ fn nbc_url_is_provider_linked(url: &str) -> bool {
     (host.ends_with("nbc.com") || host.ends_with(".nbc.com")) && path.contains("provider-linked")
 }
 
+fn nbc_url_is_mvpd_complete(url: &str) -> bool {
+    let Ok(url) = Url::parse(url) else {
+        return false;
+    };
+    let host = url.host_str().unwrap_or_default().to_ascii_lowercase();
+    let path = url.path().to_ascii_lowercase();
+    (host.ends_with("nbc.com") || host.ends_with(".nbc.com")) && path.contains("mvpd-complete")
+}
+
 fn nbc_url_is_optional_profile_signin(url: &str) -> bool {
     let Ok(url) = Url::parse(url) else {
         return false;
@@ -812,6 +840,7 @@ fn nbc_page_is_watch_surface(url: &str) -> bool {
     (host.ends_with("nbc.com") || host.ends_with(".nbc.com"))
         && !nbc_url_is_optional_profile_signin(url.as_str())
         && !nbc_url_is_provider_linked(url.as_str())
+        && !nbc_url_is_mvpd_complete(url.as_str())
 }
 
 fn nbc_title_looks_like_verizon_popup(title: &str) -> bool {
@@ -836,6 +865,14 @@ fn nbc_state_is_optional_profile_signin(state: &NbcVideoState) -> bool {
         || nbc_title_looks_like_optional_profile_signin(&state.title)
 }
 
+fn nbc_clues_look_geo_blocked(clues: &NbcPageClues) -> bool {
+    let body_text = clues.body_text.to_ascii_lowercase();
+    body_text.contains("not authorized to access this content from outside of the us")
+        || body_text.contains("not authorized to access this content from outside of the u.s.")
+        || body_text.contains("outside of the us and its territories")
+        || body_text.contains("outside of the u.s. and its territories")
+}
+
 fn browser_tabs(browser: &Browser) -> Vec<Arc<Tab>> {
     browser.register_missing_tabs();
     browser.get_tabs().lock().unwrap().iter().cloned().collect()
@@ -877,20 +914,20 @@ fn find_primary_tab_state<'a>(
         .find(|candidate| candidate.tab.get_target_id() == target_id)
 }
 
-fn find_playing_tab_state(tabs: &[BrowserTabState]) -> Option<&BrowserTabState> {
-    tabs.iter().find(|candidate| {
-        candidate.state.has_video
-            && candidate.state.width > 0
-            && candidate.state.height > 0
-            && !candidate.state.paused
-            && (candidate.state.current_time > 0.0 || candidate.state.ready_state >= 2)
-    })
+fn nbc_video_state_has_decoded_frame(state: &NbcVideoState) -> bool {
+    state.has_video
+        && state.width > 0
+        && state.height > 0
+        && !state.paused
+        && state.current_time > 0.0
+        && state.ready_state >= 2
 }
 
 fn find_provider_linked_tab_state(tabs: &[BrowserTabState]) -> Option<&BrowserTabState> {
     tabs.iter().find(|candidate| {
         nbc_title_looks_like_provider_linked(&candidate.state.title)
             || nbc_url_is_provider_linked(&candidate.state.page_url)
+            || nbc_url_is_mvpd_complete(&candidate.state.page_url)
     })
 }
 
@@ -1038,25 +1075,40 @@ fn advance_nbc_auth_flow(tab: &Arc<Tab>) -> Result<Option<NbcAuthAdvanceResult>>
           }};
           const actions = [];
           const url = window.location.href || "";
+          let host = "";
+          try {{
+            host = new URL(url).hostname.toLowerCase();
+          }} catch (_err) {{}}
+          const title = document.title || "";
+          const titleText = `${{title}} ${{url}}`.toLowerCase();
+          const looksLikeOptionalNbcProfile =
+            (host.endsWith("nbc.com") || host.endsWith(".nbc.com")) &&
+            (url.includes("/sign-in") ||
+              url.includes("/login") ||
+              titleText.includes("nbc account sign in") ||
+              titleText.includes("nbcuniversal profile") ||
+              titleText.includes("nbc profile"));
+          if (looksLikeOptionalNbcProfile) {{
+            return {{ pageUrl: url, title, actions }};
+          }}
           const candidates = Array.from(
             document.querySelectorAll(
               "button,a,[role='button'],[role='option'],label,li,[data-provider-name],[data-provider-id],[data-provider]"
             )
           );
+          const providerCta = candidates.find((node) => {{
+            const text = textOf(node);
+            return visible(node) &&
+              (
+                text === "link tv provider" ||
+                text === "link provider" ||
+                text.startsWith("link tv provider ") ||
+                text.startsWith("link provider ")
+              );
+          }});
+          clickNode(providerCta, "click:link-provider");
 
           if (url.includes("mvpd")) {{
-            const providerCta = candidates.find((node) => {{
-              const text = textOf(node);
-              return visible(node) &&
-                (
-                  text === "link tv provider" ||
-                  text === "link provider" ||
-                  text.startsWith("link tv provider ") ||
-                  text.startsWith("link provider ")
-                );
-            }});
-            clickNode(providerCta, "click:link-provider");
-
             const fullListNode = candidates.find((node) => {{
               const text = textOf(node);
               return visible(node) && (text === "full list" || text.startsWith("full list "));
@@ -1112,7 +1164,7 @@ fn advance_nbc_auth_flow(tab: &Arc<Tab>) -> Result<Option<NbcAuthAdvanceResult>>
 
           return {{
             pageUrl: url,
-            title: document.title || "",
+            title,
             actions,
           }};
         }})())
@@ -1226,16 +1278,30 @@ fn advance_mvpd_login_flow(tab: &Arc<Tab>) -> Result<Option<NbcAuthAdvanceResult
               titleText.includes("nbc profile"));
           if (looksLikeOptionalNbcProfile) {{
             const profileButtons = Array.from(document.querySelectorAll("button,a,[role='button'],input[type='submit'],input[type='button']"));
-            const providerLink = profileButtons.find((node) => {{
+            const dismissButton = profileButtons.find((node) => {{
               const text = textOf(node);
               return visible(node) && (
-                text === "link tv provider" ||
-                text === "link provider" ||
-                text.startsWith("link tv provider ") ||
-                text.startsWith("link provider ")
+                text === "skip" ||
+                text.startsWith("skip ") ||
+                text === "skip for now" ||
+                text === "maybe later" ||
+                text === "not now" ||
+                text === "no thanks" ||
+                text === "close" ||
+                text === "continue watching" ||
+                text.startsWith("continue watching ") ||
+                text === "continue without signing in" ||
+                text === "continue without profile" ||
+                text === "continue as guest" ||
+                text === "watch live" ||
+                text === "watch now" ||
+                text.startsWith("watch live ") ||
+                text.startsWith("watch now ")
               );
             }});
-            clickNode(providerLink, "click:profile-link-provider");
+            if (dismissButton) {{
+              clickNode(dismissButton, `click:profile-dismiss:${{textOf(dismissButton).slice(0, 120)}}`);
+            }}
             return {{ pageUrl: url, title, actions }};
           }}
           if (!looksLikeProviderLogin) {{
@@ -1333,8 +1399,15 @@ fn advance_nbc_post_auth_flow(tab: &Arc<Tab>) -> Result<Option<NbcAuthAdvanceRes
           const bodyText = normalize(document.body?.innerText || "");
           const looksLinked = title.toLowerCase().includes("tv provider linked")
             || url.includes("provider-linked")
+            || url.includes("mvpd-complete")
             || bodyText.includes("tv provider linked");
-          if (!looksLinked) {
+          const looksOptionalProfile =
+            (url.includes("/sign-in") ||
+              url.includes("/login") ||
+              normalize(title).includes("nbc account sign in") ||
+              normalize(title).includes("nbcuniversal profile") ||
+              normalize(title).includes("nbc profile"));
+          if (!looksLinked && !looksOptionalProfile) {
             return { pageUrl: url, title, actions };
           }
 
@@ -1344,8 +1417,22 @@ fn advance_nbc_post_auth_flow(tab: &Arc<Tab>) -> Result<Option<NbcAuthAdvanceRes
             return visible(node) && (
               text === "skip" ||
               text.startsWith("skip ") ||
-              text === "continue" ||
-              text.startsWith("continue watching")
+              text === "skip for now" ||
+              (looksLinked && text === "continue") ||
+              (looksLinked && text.startsWith("continue ")) ||
+              text === "maybe later" ||
+              text === "not now" ||
+              text === "no thanks" ||
+              text === "close" ||
+              text === "continue watching" ||
+              text.startsWith("continue watching") ||
+              text === "continue without signing in" ||
+              text === "continue without profile" ||
+              text === "continue as guest" ||
+              text === "watch live" ||
+              text === "watch now" ||
+              text.startsWith("watch live ") ||
+              text.startsWith("watch now ")
             );
           });
           if (skipButton) {
@@ -1533,6 +1620,7 @@ fn wait_for_nbc_playback(
     screenshot_out: Option<PathBuf>,
 ) -> Result<WaitOutcome> {
     let deadline = Instant::now() + nbc_capture_timeout();
+    let auth_forbidden = matches!(&auth_mode, AuthMode::Forbidden);
     let mut interactive_deadline = None::<Instant>;
     let mut interactive_auth_required = false;
     let mut screenshot_path = None::<PathBuf>;
@@ -1540,11 +1628,13 @@ fn wait_for_nbc_playback(
     let mut last_trace_state = None::<NbcTraceState>;
     let mut last_log = Instant::now() - Duration::from_secs(10);
     let mut last_clue_log = Instant::now() - Duration::from_secs(30);
+    let mut playback_samples = HashMap::<String, (f64, Instant)>::new();
     let mut resumed_after_background_login = false;
     let mut resumed_after_authenticated_surface = false;
     let mut optional_profile_signin_recoveries = 0_u8;
     let mut last_optional_profile_signin_retry = None::<Instant>;
     let mut watch_surface_seen_at = None::<Instant>;
+    let mut optional_profile_signin_seen_at = None::<Instant>;
     let mut tracked_tabs = HashSet::new();
     let mut provider_linked_completed = false;
 
@@ -1554,13 +1644,33 @@ fn wait_for_nbc_playback(
         let primary_state = find_primary_tab_state(&tab_states, tab)
             .map(|value| value.state.clone())
             .unwrap_or_else(|| probe_nbc_video(tab).unwrap_or_default());
-        if let Some(playing_tab) = find_playing_tab_state(&tab_states) {
-            return Ok(WaitOutcome {
-                state: playing_tab.state.clone(),
-                trace: trace.lock().map(|state| state.clone()).unwrap_or_default(),
-                interactive_auth_required,
-                screenshot_path,
-            });
+        let now = Instant::now();
+        for playing_tab in tab_states
+            .iter()
+            .filter(|candidate| nbc_video_state_has_decoded_frame(&candidate.state))
+        {
+            let target_id = playing_tab.tab.get_target_id().to_string();
+            if let Some((previous_time, first_seen)) = playback_samples.get(&target_id) {
+                if playing_tab.state.current_time >= *previous_time + 0.25
+                    && first_seen.elapsed() >= Duration::from_millis(500)
+                {
+                    return Ok(WaitOutcome {
+                        tab: playing_tab.tab.clone(),
+                        state: playing_tab.state.clone(),
+                        trace: trace.lock().map(|state| state.clone()).unwrap_or_default(),
+                        interactive_auth_required,
+                        screenshot_path,
+                    });
+                }
+            }
+            playback_samples
+                .entry(target_id)
+                .and_modify(|(previous_time, _)| {
+                    if playing_tab.state.current_time < *previous_time {
+                        *previous_time = playing_tab.state.current_time;
+                    }
+                })
+                .or_insert((playing_tab.state.current_time, now));
         }
 
         let interaction_tab = find_interaction_tab_state(&tab_states, tab)
@@ -1571,6 +1681,7 @@ fn wait_for_nbc_playback(
         let pre_state = probe_nbc_video(&interaction_tab).unwrap_or_default();
         if nbc_title_looks_like_provider_linked(&pre_state.title)
             || nbc_url_is_provider_linked(&pre_state.page_url)
+            || nbc_url_is_mvpd_complete(&pre_state.page_url)
         {
             provider_linked_completed = true;
         }
@@ -1579,6 +1690,7 @@ fn wait_for_nbc_playback(
         if let Some(progress) = advance_nbc_post_auth_flow(&interaction_tab).ok().flatten() {
             if nbc_title_looks_like_provider_linked(&progress.title)
                 || nbc_url_is_provider_linked(&progress.page_url)
+                || nbc_url_is_mvpd_complete(&progress.page_url)
                 || progress
                     .actions
                     .iter()
@@ -1614,6 +1726,7 @@ fn wait_for_nbc_playback(
         let state = probe_nbc_video(&interaction_tab).unwrap_or_default();
         if nbc_title_looks_like_provider_linked(&state.title)
             || nbc_url_is_provider_linked(&state.page_url)
+            || nbc_url_is_mvpd_complete(&state.page_url)
         {
             provider_linked_completed = true;
         }
@@ -1621,6 +1734,19 @@ fn wait_for_nbc_playback(
         let authorized = nbc_trace_is_authorized(&trace_state) || provider_linked_completed;
         let recent_media_activity = nbc_trace_has_recent_media_activity(&trace_state);
 
+        if !authorized && nbc_state_is_optional_profile_signin(&state) && !state.has_video {
+            let first_seen = *optional_profile_signin_seen_at.get_or_insert_with(Instant::now);
+            if auth_forbidden && first_seen.elapsed() >= nbc_profile_signin_gate_timeout() {
+                return Err(anyhow!(
+                    "NBC account sign-in gate reached before TV-provider auth; refusing non-interactive retry loop without decoded video (title='{}', page_url='{}')",
+                    state.title,
+                    state.page_url,
+                ));
+            }
+        } else {
+            optional_profile_signin_seen_at = None;
+        }
+
         if last_log.elapsed() >= Duration::from_secs(5) {
             last_log = Instant::now();
             tracing::info!(
@@ -1661,8 +1787,9 @@ fn wait_for_nbc_playback(
             }
         }
 
-        if (trace_state.background_login_complete
-            || nbc_url_is_background_login_complete(&state.page_url))
+        let auth_completion_page = nbc_url_is_background_login_complete(&state.page_url)
+            || nbc_url_is_mvpd_complete(&state.page_url);
+        if (trace_state.background_login_complete || auth_completion_page)
             && !resumed_after_background_login
         {
             resumed_after_background_login = true;
@@ -1673,41 +1800,49 @@ fn wait_for_nbc_playback(
             );
             close_auxiliary_browser_tabs(browser, tab);
             let _ = tab.activate();
-            let _ = tab.evaluate("window.location.reload()", true);
+            if nbc_url_is_mvpd_complete(&state.page_url) {
+                tab.navigate_to(source_url)?;
+                tab.wait_until_navigated()?;
+            } else {
+                let _ = tab.evaluate("window.location.reload()", true);
+            }
             std::thread::sleep(Duration::from_secs(2));
             continue;
         }
 
-        if authorized
-            && nbc_state_is_optional_profile_signin(&state)
-            && !recent_media_activity
-            && optional_profile_signin_recoveries < 3
-            && last_optional_profile_signin_retry
-                .map(|instant| instant.elapsed() >= Duration::from_secs(3))
-                .unwrap_or(true)
-        {
-            optional_profile_signin_recoveries += 1;
-            last_optional_profile_signin_retry = Some(Instant::now());
-            tracing::info!(
-                title = %state.title,
-                page_url = %state.page_url,
-                authorized,
-                source_url,
-                optional_profile_signin_recoveries,
-                "NBC profile sign-in surface detected after authorization; returning to the live source URL"
-            );
-            close_auxiliary_browser_tabs(browser, tab);
-            let _ = tab.activate();
-            tab.navigate_to(source_url)?;
-            tab.wait_until_navigated()?;
-            std::thread::sleep(Duration::from_secs(2));
-            continue;
+        if authorized && nbc_state_is_optional_profile_signin(&state) && !state.has_video {
+            if optional_profile_signin_recoveries == 0
+                && last_optional_profile_signin_retry
+                    .map(|instant| instant.elapsed() >= Duration::from_secs(3))
+                    .unwrap_or(true)
+            {
+                optional_profile_signin_recoveries += 1;
+                last_optional_profile_signin_retry = Some(Instant::now());
+                tracing::info!(
+                    title = %state.title,
+                    page_url = %state.page_url,
+                    authorized,
+                    source_url,
+                    "NBC account sign-in gate detected after provider authorization; trying one live-url recovery"
+                );
+                close_auxiliary_browser_tabs(browser, tab);
+                let _ = tab.activate();
+                tab.navigate_to(source_url)?;
+                tab.wait_until_navigated()?;
+                std::thread::sleep(Duration::from_secs(2));
+                continue;
+            }
+            return Err(anyhow!(
+                "NBC account sign-in gate reached after TV-provider auth; refusing retry loop without decoded video (title='{}', page_url='{}')",
+                state.title,
+                state.page_url,
+            ));
         }
-        if authorized && nbc_state_is_optional_profile_signin(&state) && recent_media_activity {
+        if authorized && nbc_state_is_optional_profile_signin(&state) && state.has_video {
             tracing::debug!(
                 title = %state.title,
                 page_url = %state.page_url,
-                "NBC optional profile sign-in is visible but media activity is already in flight; staying on the page"
+                "NBC optional profile sign-in is visible but a video element is already present; staying on the page"
             );
         }
 
@@ -1733,6 +1868,13 @@ fn wait_for_nbc_playback(
                     body_text = %clues.body_text,
                     "NBC watch surface clues"
                 );
+                if nbc_clues_look_geo_blocked(&clues) {
+                    return Err(anyhow!(
+                        "NBC geo-blocked current egress; page says this content is not authorized outside the US/territories (title='{}', page_url='{}')",
+                        primary_state.title,
+                        primary_state.page_url,
+                    ));
+                }
             }
         }
         if fully_loaded_watch_surface && !primary_state.has_video {
@@ -1862,6 +2004,9 @@ mod tests {
         assert!(nbc_url_is_provider_linked(
             "https://www.nbc.com/provider-linked"
         ));
+        assert!(nbc_url_is_mvpd_complete(
+            "https://www.nbc.com/mvpd-complete"
+        ));
         assert!(nbc_title_looks_like_provider_linked("TV Provider Linked"));
         assert!(!nbc_url_is_provider_linked(
             "https://www.nbc.com/live?brand=nbc-sports-philadelphia"
@@ -1884,11 +2029,31 @@ mod tests {
     #[test]
     fn optional_profile_signin_is_not_treated_as_watch_surface() {
         assert!(!nbc_page_is_watch_surface("https://www.nbc.com/sign-in"));
+        assert!(!nbc_page_is_watch_surface(
+            "https://www.nbc.com/mvpd-complete"
+        ));
         assert!(nbc_page_is_watch_surface(
             "https://www.nbc.com/live?brand=nbc-sports-philadelphia"
         ));
     }
 
+    #[test]
+    fn geo_block_clues_fail_closed() {
+        let clues = NbcPageClues {
+            body_text:
+                "We're sorry. You are not authorized to access this content from outside of the US and its territories."
+                    .to_string(),
+            ..NbcPageClues::default()
+        };
+        assert!(nbc_clues_look_geo_blocked(&clues));
+
+        let allowed = NbcPageClues {
+            body_text: "NBC News NOW ON NOW until 7:00 AM".to_string(),
+            ..NbcPageClues::default()
+        };
+        assert!(!nbc_clues_look_geo_blocked(&allowed));
+    }
+
     #[test]
     fn cssott_media_requests_mark_recent_media_activity() {
         let mut trace = NbcTraceState::default();
@@ -1899,4 +2064,25 @@ mod tests {
         assert!(trace.media_activity_seen);
         assert!(nbc_trace_has_recent_media_activity(&trace));
     }
+
+    #[test]
+    fn decoded_frame_detection_requires_advancing_video_surface() {
+        let mut state = NbcVideoState {
+            has_video: true,
+            width: 1920,
+            height: 1080,
+            paused: false,
+            ready_state: 2,
+            current_time: 1.0,
+            ..NbcVideoState::default()
+        };
+        assert!(nbc_video_state_has_decoded_frame(&state));
+
+        state.current_time = 0.0;
+        assert!(!nbc_video_state_has_decoded_frame(&state));
+
+        state.current_time = 1.0;
+        state.width = 0;
+        assert!(!nbc_video_state_has_decoded_frame(&state));
+    }
 }

crates/ec-node/tests/determinism_cmaf_ladder.rs

@@ -1,3 +1,4 @@
+use std::collections::BTreeMap;
 use std::io::{BufRead, BufReader};
 use std::path::Path;
 use std::process::{Command, Stdio};
@@ -46,6 +47,15 @@ fn blake3_hex(path: &Path) -> anyhow::Result<String> {
     Ok(blake3::hash(&bytes).to_hex().to_string())
 }
 
+fn command_available(name: &str) -> bool {
+    Command::new(name)
+        .arg("-version")
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .is_ok()
+}
+
 fn concat_init_and_segment(init: &Path, seg: &Path, out: &Path) -> anyhow::Result<()> {
     let init_bytes = std::fs::read(init)?;
     let seg_bytes = std::fs::read(seg)?;
@@ -157,11 +167,15 @@ fn write_deterministic_ts(out_path: &Path) -> anyhow::Result<()> {
     Ok(())
 }
 
-fn run_ladder(ec_node: &Path, input_ts: &Path, out_dir: &Path) -> anyhow::Result<()> {
+fn run_ladder_with_identity(
+    ec_node: &Path,
+    input_ts: &Path,
+    out_dir: &Path,
+    stream_id: &str,
+    broadcast_name: &str,
+) -> anyhow::Result<()> {
     let signing_key = "11".repeat(32);
     let network_secret = "22".repeat(32);
-    let stream_id = "every.channel/determinism/cmaf-ladder";
-    let broadcast_name = "every.channel/determinism/cmaf-ladder";
 
     let mut cmd = Command::new(ec_node);
     cmd.env("EVERY_CHANNEL_MANIFEST_SIGNING_KEY", &signing_key)
@@ -210,6 +224,40 @@ fn run_ladder(ec_node: &Path, input_ts: &Path, out_dir: &Path) -> anyhow::Result
     Ok(())
 }
 
+fn run_ladder(ec_node: &Path, input_ts: &Path, out_dir: &Path) -> anyhow::Result<()> {
+    run_ladder_with_identity(
+        ec_node,
+        input_ts,
+        out_dir,
+        "every.channel/determinism/cmaf-ladder",
+        "every.channel/determinism/cmaf-ladder",
+    )
+}
+
+fn ladder_artifact_hashes(root: &Path) -> BTreeMap<String, String> {
+    let mut hashes = BTreeMap::new();
+    for variant in ["1080p", "720p", "480p"] {
+        let variant_dir = root.join("cmaf-ladder").join(variant);
+        // `moq-publish --max-chunks 3` publishes init plus segments 0..=2.
+        // ffmpeg can race ahead and leave an unpublished tail segment before it is killed.
+        let init = variant_dir.join("init.mp4");
+        assert!(init.exists(), "missing init for {variant}");
+        hashes.insert(format!("{variant}/init.mp4"), blake3_hex(&init).unwrap());
+
+        for idx in 0..3 {
+            let name = format!("segment_{idx:06}.m4s");
+            let path = variant_dir.join(&name);
+            assert!(path.exists(), "missing {name} for {variant}");
+            hashes.insert(format!("{variant}/{name}"), blake3_hex(&path).unwrap());
+        }
+    }
+    hashes
+}
+
+fn assert_ladder_bytes_match(left: &Path, right: &Path) {
+    assert_eq!(ladder_artifact_hashes(left), ladder_artifact_hashes(right));
+}
+
 #[test]
 #[ignore]
 fn deterministic_cmaf_ladder_outputs_match_across_runs() {
@@ -235,36 +283,53 @@ fn deterministic_cmaf_ladder_outputs_match_across_runs() {
     run_ladder(&ec_node, &input_ts, &run1).expect("run ladder 1");
     run_ladder(&ec_node, &input_ts, &run2).expect("run ladder 2");
 
-    for variant in ["1080p", "720p", "480p"] {
-        let v1 = run1.join("cmaf-ladder").join(variant);
-        let v2 = run2.join("cmaf-ladder").join(variant);
+    assert_ladder_bytes_match(&run1, &run2);
+}
 
-        let init1 = v1.join("init.mp4");
-        let init2 = v2.join("init.mp4");
-        assert!(
-            init1.exists() && init2.exists(),
-            "missing init for {variant}"
-        );
-        assert_eq!(
-            blake3_hex(&init1).unwrap(),
-            blake3_hex(&init2).unwrap(),
-            "init differs for {variant}"
-        );
-
-        for idx in 0..3 {
-            let s1 = v1.join(format!("segment_{idx:06}.m4s"));
-            let s2 = v2.join(format!("segment_{idx:06}.m4s"));
-            assert!(
-                s1.exists() && s2.exists(),
-                "missing segment {idx} for {variant}"
-            );
-            assert_eq!(
-                blake3_hex(&s1).unwrap(),
-                blake3_hex(&s2).unwrap(),
-                "segment {idx} differs for {variant}"
-            );
-        }
+#[test]
+fn duplicate_publishers_same_input_produce_identical_cmaf_ladder_bytes() {
+    if !command_available("ffmpeg") {
+        eprintln!("skipping duplicate publisher CMAF ladder determinism test: ffmpeg unavailable");
+        return;
     }
+
+    let ec_node = ec_node_path();
+
+    let ts = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis();
+    let tmp = std::env::temp_dir().join(format!("ec-duplicate-publisher-cmaf-ladder-{ts}"));
+    let _ = std::fs::create_dir_all(&tmp);
+
+    let input_ts = tmp.join("input.ts");
+    write_deterministic_ts(&input_ts).expect("write deterministic TS");
+
+    let publisher_a = tmp.join("publisher-a");
+    let publisher_b = tmp.join("publisher-b");
+    let _ = std::fs::remove_dir_all(&publisher_a);
+    let _ = std::fs::remove_dir_all(&publisher_b);
+    std::fs::create_dir_all(&publisher_a).unwrap();
+    std::fs::create_dir_all(&publisher_b).unwrap();
+
+    run_ladder_with_identity(
+        &ec_node,
+        &input_ts,
+        &publisher_a,
+        "every.channel/determinism/duplicate/publisher-a/la-kcop",
+        "publisher-a-la-kcop",
+    )
+    .expect("run duplicate publisher a");
+    run_ladder_with_identity(
+        &ec_node,
+        &input_ts,
+        &publisher_b,
+        "every.channel/determinism/duplicate/publisher-b/la-kcop",
+        "publisher-b-la-kcop",
+    )
+    .expect("run duplicate publisher b");
+
+    assert_ladder_bytes_match(&publisher_a, &publisher_b);
 }
 
 #[test]

crates/ec-node/tests/e2e_remote_website_watch_existing.rs

@@ -1,4 +1,5 @@
 use std::ffi::OsStr;
+use std::process::{Command, Stdio};
 use std::time::{Duration, Instant};
 
 fn which(cmd: &str) -> Option<std::path::PathBuf> {
@@ -16,6 +17,24 @@ fn chrome_path() -> Option<std::path::PathBuf> {
         .or_else(|| which("chromium"))
 }
 
+fn ec_node_path() -> std::path::PathBuf {
+    if let Ok(value) = std::env::var("EC_NODE_BIN") {
+        return value.into();
+    }
+    if let Ok(value) = std::env::var("CARGO_BIN_EXE_ec_node") {
+        return value.into();
+    }
+    if let Ok(value) = std::env::var("CARGO_BIN_EXE_ec-node") {
+        return value.into();
+    }
+    let exe = std::env::current_exe().expect("current_exe");
+    let debug_dir = exe
+        .parent()
+        .and_then(|p| p.parent())
+        .expect("expected target/debug/deps");
+    debug_dir.join("ec-node")
+}
+
 fn wait_for_canvas_element(tab: &headless_chrome::Tab, timeout: Duration) -> anyhow::Result<()> {
     let deadline = Instant::now() + timeout;
     while Instant::now() < deadline {
@@ -46,14 +65,41 @@ fn wait_for_moq_watch_element(tab: &headless_chrome::Tab, timeout: Duration) ->
     anyhow::bail!("timed out waiting for <moq-watch> element");
 }
 
+fn wait_for_live_or_archive_player(
+    tab: &headless_chrome::Tab,
+    timeout: Duration,
+) -> anyhow::Result<()> {
+    let deadline = Instant::now() + timeout;
+    while Instant::now() < deadline {
+        let js = r#"(function() {
+  return !!document.querySelector('moq-watch, video.archiveVideo');
+})();"#;
+        let v = tab.evaluate(js, false)?;
+        if v.value.and_then(|v| v.as_bool()).unwrap_or(false) {
+            return Ok(());
+        }
+        std::thread::sleep(Duration::from_millis(200));
+    }
+    anyhow::bail!("timed out waiting for live or archive player");
+}
+
 fn debug_player_state(tab: &headless_chrome::Tab) -> anyhow::Result<String> {
     let js = r#"(function() {
   let watch = document.querySelector('moq-watch');
   let canvas = document.querySelector('moq-watch canvas');
+  let video = document.querySelector('video.archiveVideo');
   let placeholder = document.querySelector('.placeholder');
   let placeholderText = placeholder ? (placeholder.innerText || '') : null;
   let status = document.querySelector('.source-status');
   let statusText = status ? (status.innerText || '') : null;
+  let statusLine = document.querySelector('#statusLine');
+  let statusLineText = statusLine ? (statusLine.innerText || '') : null;
+  let catalog = watch && watch.broadcast && watch.broadcast.catalog && watch.broadcast.catalog.peek
+    ? watch.broadcast.catalog.peek()
+    : null;
+  let established = watch && watch.connection && watch.connection.established && watch.connection.established.peek
+    ? watch.connection.established.peek()
+    : null;
   let sources = Array.from(document.querySelectorAll('button[data-testid="global-watch"]')).length;
   let hint = document.querySelector('#hint');
   let hintText = hint ? (hint.innerText || '') : null;
@@ -62,8 +108,27 @@ fn debug_player_state(tab: &headless_chrome::Tab) -> anyhow::Result<String> {
     hasCanvas: !!canvas,
     canvasWidth: canvas ? canvas.width : null,
     canvasHeight: canvas ? canvas.height : null,
+    hasArchiveVideo: !!video,
+    videoCurrentTime: video ? video.currentTime : null,
+    videoDuration: video ? video.duration : null,
+    videoPaused: video ? video.paused : null,
+    videoReadyState: video ? video.readyState : null,
+    videoMuted: video ? video.muted : null,
+    videoVolume: video ? video.volume : null,
+    videoSrc: video ? (video.currentSrc || video.src || '') : null,
     muted: watch ? watch.muted : null,
     volume: watch ? watch.volume : null,
+    connectionStatus: watch?.connection?.status?.peek ? watch.connection.status.peek() : null,
+    connectionKind: established ? established.constructor?.name || null : null,
+    broadcastStatus: watch?.broadcast?.status?.peek ? watch.broadcast.status.peek() : null,
+    paused: watch?.backend?.paused?.peek ? watch.backend.paused.peek() : null,
+    audioMuted: watch?.backend?.audio?.muted?.peek ? watch.backend.audio.muted.peek() : null,
+    audioVolume: watch?.backend?.audio?.volume?.peek ? watch.backend.audio.volume.peek() : null,
+    catalogSeen: !!catalog,
+    catalogHasVideo: !!(catalog?.video?.renditions),
+    catalogHasAudio: !!(catalog?.audio?.renditions),
+    metrics: window.__ecPlaybackMetrics || null,
+    statusLineText,
     hintText,
     placeholderText,
     statusText,
@@ -110,23 +175,120 @@ fn canvas_motion_sample(tab: &headless_chrome::Tab) -> anyhow::Result<Option<(f6
     Ok(Some((current_time, hash)))
 }
 
-fn wait_for_canvas_motion(tab: &headless_chrome::Tab, timeout: Duration) -> anyhow::Result<()> {
+fn archive_video_motion_sample(
+    tab: &headless_chrome::Tab,
+) -> anyhow::Result<Option<serde_json::Value>> {
+    let js = r#"(function() {
+  let video = document.querySelector('video.archiveVideo');
+  if (!video) return null;
+  if (video.paused) video.play().catch(() => {});
+  return JSON.stringify({
+    wallTime: performance.now() / 1000,
+    currentTime: video.currentTime || 0,
+    readyState: video.readyState || 0,
+    paused: !!video.paused,
+    ended: !!video.ended,
+    muted: !!video.muted,
+    volume: video.volume || 0,
+    src: video.currentSrc || video.src || ''
+  });
+})();"#;
+    let v = tab.evaluate(js, false)?;
+    let Some(s) = v.value.and_then(|v| v.as_str().map(|s| s.to_string())) else {
+        return Ok(None);
+    };
+    Ok(Some(serde_json::from_str(&s)?))
+}
+
+fn wait_for_canvas_or_archive_motion(
+    tab: &headless_chrome::Tab,
+    timeout: Duration,
+) -> anyhow::Result<String> {
     let deadline = Instant::now() + timeout;
-    let mut first: Option<(f64, u32)> = None;
+    let mut first_canvas: Option<(f64, u32)> = None;
+    let mut first_video_time: Option<f64> = None;
     while Instant::now() < deadline {
         if let Some(sample) = canvas_motion_sample(tab)? {
-            if let Some((first_time, first_hash)) = first {
+            if let Some((first_time, first_hash)) = first_canvas {
                 if sample.0 > first_time + 0.5 && sample.1 != first_hash {
-                    return Ok(());
+                    return Ok("moq-canvas".to_string());
                 }
             } else {
-                first = Some(sample);
+                first_canvas = Some(sample);
             }
         }
+
+        if let Some(sample) = archive_video_motion_sample(tab)? {
+            let current_time = sample
+                .get("currentTime")
+                .and_then(|v| v.as_f64())
+                .unwrap_or_default();
+            let ready_state = sample
+                .get("readyState")
+                .and_then(|v| v.as_u64())
+                .unwrap_or_default();
+            let ended = sample
+                .get("ended")
+                .and_then(|v| v.as_bool())
+                .unwrap_or(false);
+            if ready_state >= 2 && !ended {
+                if let Some(first) = first_video_time {
+                    if current_time > first + 0.5 {
+                        return Ok("archive-video".to_string());
+                    }
+                } else {
+                    first_video_time = Some(current_time);
+                }
+            }
+        }
+
         std::thread::sleep(Duration::from_millis(500));
     }
     let st = debug_player_state(tab).unwrap_or_default();
-    anyhow::bail!("timed out waiting for changing canvas frames\nplayer_state={st}");
+    anyhow::bail!("timed out waiting for live or archive motion\nplayer_state={st}");
+}
+
+fn wait_for_playback_probe_ok(
+    tab: &headless_chrome::Tab,
+    timeout: Duration,
+) -> anyhow::Result<String> {
+    let deadline = Instant::now() + timeout;
+    let mut last_metrics = String::new();
+    while Instant::now() < deadline {
+        let js = r#"(function() {
+  const metrics = window.__ecPlaybackMetrics || null;
+  return metrics ? JSON.stringify(metrics) : "";
+})();"#;
+        let v = tab.evaluate(js, false)?;
+        last_metrics = v
+            .value
+            .and_then(|v| v.as_str().map(|s| s.to_string()))
+            .unwrap_or_default();
+        if !last_metrics.is_empty() {
+            let metrics: serde_json::Value = serde_json::from_str(&last_metrics)?;
+            let ok = metrics.get("ok").and_then(|v| v.as_bool()).unwrap_or(false);
+            let samples = metrics
+                .get("samples")
+                .and_then(|v| v.as_u64())
+                .unwrap_or_default();
+            let changed = metrics
+                .get("changed_samples")
+                .and_then(|v| v.as_u64())
+                .unwrap_or_default();
+            let longest_static = metrics
+                .get("longest_same_hash_ms")
+                .and_then(|v| v.as_u64())
+                .unwrap_or_default();
+            if ok && samples >= 8 && changed >= 2 && longest_static < 5_000 {
+                return Ok(last_metrics);
+            }
+        }
+        std::thread::sleep(Duration::from_millis(250));
+    }
+    let st = debug_player_state(tab).unwrap_or_default();
+    anyhow::bail!(
+        "timed out waiting for playback probe ok\nplayer_state={st}\nmetrics={last_metrics}"
+    );
 }
 
 fn wait_for_unmuted_player(tab: &headless_chrome::Tab, timeout: Duration) -> anyhow::Result<()> {
@@ -134,7 +296,9 @@ fn wait_for_unmuted_player(tab: &headless_chrome::Tab, timeout: Duration) -> any
     while Instant::now() < deadline {
         let js = r#"(function() {
   let watch = document.querySelector('moq-watch');
-  return !!watch && watch.muted === false && watch.volume > 0 && !watch.hasAttribute('muted');
+  let video = document.querySelector('video.archiveVideo');
+  return (!!watch && watch.muted === false && watch.volume > 0 && !watch.hasAttribute('muted')) ||
+    (!!video && video.muted === false && video.volume > 0);
 })();"#;
         let v = tab.evaluate(js, false)?;
         if v.value.and_then(|v| v.as_bool()).unwrap_or(false) {
@@ -146,13 +310,21 @@ fn wait_for_unmuted_player(tab: &headless_chrome::Tab, timeout: Duration) -> any
     anyhow::bail!("timed out waiting for unmuted player\nplayer_state={st}");
 }
 
-fn watch_url(site_url: &str, relay_url: &str, stream_id: &str) -> anyhow::Result<String> {
+fn watch_url(
+    site_url: &str,
+    relay_url: &str,
+    stream_id: &str,
+    verify: bool,
+) -> anyhow::Result<String> {
     let mut url = url::Url::parse(site_url)?;
     url.set_path("/watch");
     url.query_pairs_mut()
         .clear()
         .append_pair("url", relay_url)
         .append_pair("name", stream_id);
+    if verify {
+        url.query_pairs_mut().append_pair("verify", "1");
+    }
     Ok(url.to_string())
 }
 
@@ -190,23 +362,104 @@ fn e2e_remote_website_watch_existing_stream_id() -> anyhow::Result<()> {
         .unwrap();
     let browser = headless_chrome::Browser::new(launch_options)?;
     let tab = browser.new_tab()?;
-    tab.navigate_to(&watch_url(&site_url, &relay_url, &stream_id)?)?;
+    tab.navigate_to(&watch_url(&site_url, &relay_url, &stream_id, false)?)?;
     tab.wait_until_navigated()?;
 
-    // Ensure the player is instantiated.
-    if let Err(err) = wait_for_moq_watch_element(&tab, Duration::from_secs(90)) {
+    // Ensure either the native MoQ player or the archive live-edge fallback is instantiated.
+    if let Err(err) = wait_for_live_or_archive_player(&tab, Duration::from_secs(90)) {
         let st = debug_player_state(&tab).unwrap_or_default();
         anyhow::bail!("{err}\nplayer_state={st}");
     }
 
-    if let Err(err) = wait_for_canvas_element(&tab, Duration::from_secs(90)) {
-        let st = debug_player_state(&tab).unwrap_or_default();
-        anyhow::bail!("{err}\nplayer_state={st}");
-    }
-
-    tab.wait_for_element("moq-watch canvas")?.click()?;
+    tab.evaluate(
+        r#"(function() {
+  const canvas = document.querySelector('moq-watch canvas');
+  if (canvas) canvas.click();
+  const audioButton = document.querySelector('#audioBtn');
+  if (audioButton && audioButton.getAttribute('aria-pressed') !== 'true') {
+    audioButton.click();
+  }
+})();"#,
+        false,
+    )?;
     wait_for_unmuted_player(&tab, Duration::from_secs(10))?;
-    wait_for_canvas_motion(&tab, Duration::from_secs(30))?;
+    let playback_path = wait_for_canvas_or_archive_motion(&tab, Duration::from_secs(60))?;
+    eprintln!("playback path: {playback_path}");
 
     Ok(())
 }
+
+#[test]
+#[ignore]
+fn e2e_remote_website_watch_synthetic_relay_stream() -> anyhow::Result<()> {
+    if which("ffmpeg").is_none() {
+        return Ok(()); // skip
+    }
+    let chrome = match chrome_path() {
+        Some(p) => p,
+        None => return Ok(()), // skip
+    };
+
+    let site_url = std::env::var("EVERY_CHANNEL_SITE_URL")
+        .unwrap_or_else(|_| "https://every.channel/".to_string());
+    let relay_url = std::env::var("EVERY_CHANNEL_RELAY_URL")
+        .unwrap_or_else(|_| "https://relay.every.channel/anon".to_string());
+    let tls_disable_verify = std::env::var("EVERY_CHANNEL_RELAY_TLS_DISABLE_VERIFY")
+        .map(|v| v != "0" && v.to_lowercase() != "false")
+        .unwrap_or(true);
+
+    let ts = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis();
+    let stream_id = format!("e2e-synthetic-{ts}");
+    let ec_node = ec_node_path();
+
+    let mut publisher = Command::new(&ec_node);
+    publisher
+        .arg("wt-publish")
+        .arg("--url")
+        .arg(&relay_url)
+        .arg("--name")
+        .arg(&stream_id)
+        .arg("--realtime-input")
+        .arg("--input-format")
+        .arg("lavfi")
+        .arg("--input")
+        .arg("testsrc2=size=1280x720:rate=30")
+        .stdout(Stdio::null())
+        .stderr(Stdio::inherit());
+    if tls_disable_verify {
+        publisher.arg("--tls-disable-verify");
+    }
+    let mut publisher = publisher.spawn()?;
+
+    let test_result = (|| -> anyhow::Result<()> {
+        let launch_options = headless_chrome::LaunchOptionsBuilder::default()
+            .path(Some(chrome))
+            .headless(true)
+            .args(vec![
+                OsStr::new("--autoplay-policy=no-user-gesture-required"),
+                OsStr::new("--disable-application-cache"),
+                OsStr::new("--disable-service-worker"),
+                OsStr::new("--disk-cache-size=0"),
+                OsStr::new("--mute-audio"),
+            ])
+            .build()
+            .unwrap();
+        let browser = headless_chrome::Browser::new(launch_options)?;
+        let tab = browser.new_tab()?;
+        tab.navigate_to(&watch_url(&site_url, &relay_url, &stream_id, true)?)?;
+        tab.wait_until_navigated()?;
+
+        wait_for_moq_watch_element(&tab, Duration::from_secs(90))?;
+        wait_for_canvas_element(&tab, Duration::from_secs(90))?;
+        let metrics = wait_for_playback_probe_ok(&tab, Duration::from_secs(60))?;
+        eprintln!("playback metrics: {metrics}");
+        Ok(())
+    })();
+
+    let _ = publisher.kill();
+    let _ = publisher.wait();
+    test_result
+}