use ec_core::sim::{ check_control_plane_propagation_invariants, check_duplicate_publisher_invariants, check_system_duplicate_publisher_invariants, run_control_plane_propagation_campaign, run_control_plane_propagation_simulation, run_duplicate_publisher_campaign, run_duplicate_publisher_simulation, run_seeded_simulation_campaign, run_system_duplicate_publisher_campaign, run_system_duplicate_publisher_simulation, shrink_duplicate_publisher_failure, simulated_media_hash, ControlPlanePropagationInvariantConfig, ControlPlanePropagationScenario, ControlPlaneTraceEvent, DeterministicSimulation, DuplicatePublisherInvariantConfig, DuplicatePublisherScenario, EncoderDriftFault, FoundationStyleSystemScenarioConfig, PublisherSequenceClock, SimulationOutage, SimulationPartition, SimulationSeed, SystemDuplicatePublisherInvariantConfig, SystemDuplicatePublisherScenario, }; const STREAM: &str = "la-kcop"; const RENDITION: &str = "720p"; const TRACK: &str = "0.m4s"; const PROFILE: &str = "x264-hd3-v1"; fn schedule_publisher_window( sim: &mut DeterministicSimulation, node: &str, start_sequence: u64, end_sequence: u64, first_delivery_ms: u64, step_ms: u64, profile: &str, ) { for sequence in start_sequence..end_sequence { let hash = simulated_media_hash(STREAM, RENDITION, TRACK, sequence, profile); sim.schedule_observation( first_delivery_ms + (sequence - start_sequence) * step_ms, node, STREAM, RENDITION, TRACK, sequence, &hash, ); } } #[test] fn duplicate_publishers_converge_after_delayed_backfill() { let mut sim = DeterministicSimulation::new(); schedule_publisher_window(&mut sim, "nuc-a", 0, 12, 0, 10, PROFILE); schedule_publisher_window(&mut sim, "nuc-b", 0, 4, 30, 10, PROFILE); schedule_publisher_window(&mut sim, "nuc-b", 4, 12, 500, 10, PROFILE); sim.run_until(250); let before_backfill = sim.convergence().summarize(STREAM, RENDITION, TRACK, 0, 12); assert_eq!(before_backfill.expected_sequences, 12); assert_eq!(before_backfill.missing_sequences, Vec::::new()); assert_eq!( before_backfill.matching_duplicate_sequences, vec![0, 1, 2, 3] ); assert!(before_backfill.ok()); sim.run_to_idle(); let after_backfill = sim.convergence().summarize(STREAM, RENDITION, TRACK, 0, 12); let duplicate_complete_at_ms = sim .convergence() .duplicate_complete_at_ms(STREAM, RENDITION, TRACK, 0, 12); assert_eq!(after_backfill.missing_sequences, Vec::::new()); assert_eq!(after_backfill.divergent_sequences, Vec::::new()); assert_eq!( after_backfill.matching_duplicate_sequences, (0_u64..12).collect::>() ); assert_eq!(after_backfill.duplicate_source_records, 24); assert_eq!(duplicate_complete_at_ms, Some(570)); assert_eq!(sim.trace().len(), 24); assert!( sim.trace() .windows(2) .all(|pair| (pair[0].at_ms, pair[0].order) <= (pair[1].at_ms, pair[1].order)), "trace should preserve deterministic event order" ); assert!(after_backfill.ok()); } #[test] fn media_convergence_can_summarize_sparse_observed_sequences() { let mut sim = DeterministicSimulation::new(); for sequence in [7_287_381_184_512, 7_287_381_188_608] { let hash = simulated_media_hash(STREAM, RENDITION, TRACK, sequence, PROFILE); sim.schedule_observation(0, "nuc-a", STREAM, RENDITION, TRACK, sequence, &hash); sim.schedule_observation(1, "nuc-b", STREAM, RENDITION, TRACK, sequence, &hash); } sim.run_to_idle(); let dense = sim.convergence().summarize( STREAM, RENDITION, TRACK, 7_287_381_184_512, 7_287_381_188_609, ); let sparse = sim.convergence().summarize_observed_sequences( STREAM, RENDITION, TRACK, 7_287_381_184_512, 7_287_381_188_609, ); assert!(!dense.missing_sequences.is_empty()); assert_eq!(sparse.expected_sequences, 2); assert_eq!(sparse.missing_sequences, Vec::::new()); assert_eq!( sparse.matching_duplicate_sequences, vec![7_287_381_184_512, 7_287_381_188_608] ); assert!(sparse.ok()); } #[test] fn duplicate_publisher_simulation_detects_encoder_drift() { let mut sim = DeterministicSimulation::new(); schedule_publisher_window(&mut sim, "nuc-a", 0, 8, 0, 10, PROFILE); schedule_publisher_window(&mut sim, "nuc-b", 0, 8, 5, 10, PROFILE); let drift_hash = simulated_media_hash(STREAM, RENDITION, TRACK, 4, "x264-hd3-drift"); sim.schedule_observation(90, "nuc-b", STREAM, RENDITION, TRACK, 4, &drift_hash); sim.run_to_idle(); let summary = sim.convergence().summarize(STREAM, RENDITION, TRACK, 0, 8); assert_eq!(summary.missing_sequences, Vec::::new()); assert_eq!(summary.divergent_sequences, vec![4]); assert!(!summary.ok()); } #[test] fn duplicate_publisher_fault_schedule_replays_from_seed() { let scenario = faulted_duplicate_scenario(SimulationSeed::new(0x6d6f_712d_6475_7021)); let first = run_duplicate_publisher_simulation(&scenario); let second = run_duplicate_publisher_simulation(&scenario); assert_eq!(first, second); assert!(first.duplicate_complete(), "replay {}", first.replay_hint); assert_eq!(first.summary.matching_duplicate_sequences.len(), 48); assert_eq!( first.trace, second.trace, "replayed reports should carry the same event history" ); } #[test] fn duplicate_publisher_many_seed_fault_schedules_converge() { let mut saw_transient_drop = false; let mut saw_partition_delay = false; let mut saw_publisher_outage = false; for seed in 1..=96 { let scenario = faulted_duplicate_scenario(SimulationSeed::new(seed)); let report = run_duplicate_publisher_simulation(&scenario); saw_transient_drop |= report.fault_stats.transient_dropped_observations > 0; saw_partition_delay |= report.fault_stats.partition_delayed_observations > 0; saw_publisher_outage |= report.fault_stats.publisher_outage_observations > 0; assert!( report.duplicate_complete(), "duplicate publisher convergence failed for {}: {:?}", report.replay_hint, report.summary ); assert_eq!(report.summary.missing_sequences, Vec::::new()); assert_eq!(report.summary.divergent_sequences, Vec::::new()); assert_eq!(report.summary.duplicate_source_records, 96); } assert!( saw_transient_drop, "fault suite did not exercise transient drops" ); assert!( saw_partition_delay, "fault suite did not exercise partitions" ); assert!( saw_publisher_outage, "fault suite did not exercise publisher outages" ); } #[test] fn seeded_fault_scenario_detects_encoder_drift() { let mut scenario = faulted_duplicate_scenario(SimulationSeed::new(0x6472_6966_7421)); scenario .encoder_drifts .push(EncoderDriftFault::new("nuc-b", 17, "x264-hd3-drift")); let report = run_duplicate_publisher_simulation(&scenario); assert!(!report.duplicate_complete()); assert_eq!(report.summary.divergent_sequences, vec![17]); assert_eq!(report.duplicate_complete_at_ms, None); assert_eq!(report.fault_stats.encoder_drift_observations, 1); } #[test] fn duplicate_publisher_simulation_detects_unaligned_publisher_phase() { let mut scenario = DuplicatePublisherScenario::new( SimulationSeed::new(0x7068_6173_652d_6275), vec!["nuc-a".to_string(), "nuc-b".to_string()], STREAM, RENDITION, TRACK, PROFILE, 0, 8, ); scenario.base_network_delay_ms = 0; scenario.max_jitter_ms = 0; scenario .publisher_sequence_offsets .insert("nuc-b".to_string(), 3); let report = run_duplicate_publisher_simulation(&scenario); let invariant = check_duplicate_publisher_invariants( &report, &DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(1_000), ); assert!(!report.duplicate_complete()); assert_eq!(report.summary.missing_sequences, Vec::::new()); assert_eq!( report.summary.matching_duplicate_sequences, Vec::::new() ); assert_eq!( report.summary.divergent_sequences, (0_u64..8).collect::>() ); assert_eq!(report.fault_stats.publisher_phase_offset_observations, 8); assert_eq!( invariant.failures, vec![ "divergent_sequences".to_string(), "media_timing_conflict_sequences".to_string(), "duplicate_incomplete".to_string(), "duplicate_complete_deadline_unreached".to_string(), ] ); } #[test] fn duplicate_publisher_simulation_rejects_missing_media_timing() { let mut scenario = DuplicatePublisherScenario::new( SimulationSeed::new(0x7469_6d65_2d6d_6973), vec!["nuc-a".to_string(), "nuc-b".to_string()], STREAM, RENDITION, TRACK, PROFILE, 0, 6, ); scenario.base_network_delay_ms = 0; scenario.max_jitter_ms = 0; scenario .missing_media_timing_publishers .insert("nuc-b".to_string()); let report = run_duplicate_publisher_simulation(&scenario); let invariant = check_duplicate_publisher_invariants( &report, &DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(1_000), ); assert_eq!(report.summary.divergent_sequences, Vec::::new()); assert_eq!(report.summary.media_timing_missing_records, 6); assert_eq!( invariant.failures, vec![ "media_timing_missing_records".to_string(), "duplicate_incomplete".to_string(), "duplicate_complete_deadline_unreached".to_string(), ] ); } #[test] fn duplicate_publisher_simulation_rejects_conflicting_media_timing() { let mut scenario = DuplicatePublisherScenario::new( SimulationSeed::new(0x7469_6d65_2d73_6b65), vec!["nuc-a".to_string(), "nuc-b".to_string()], STREAM, RENDITION, TRACK, PROFILE, 0, 6, ); scenario.base_network_delay_ms = 0; scenario.max_jitter_ms = 0; scenario .publisher_media_time_offsets_ms .insert("nuc-b".to_string(), 17); let report = run_duplicate_publisher_simulation(&scenario); let invariant = check_duplicate_publisher_invariants( &report, &DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(1_000), ); assert_eq!(report.summary.divergent_sequences, Vec::::new()); assert_eq!( report.summary.media_timing_conflict_sequences, (0_u64..6).collect::>() ); assert_eq!( invariant.failures, vec![ "media_timing_conflict_sequences".to_string(), "duplicate_incomplete".to_string(), "duplicate_complete_deadline_unreached".to_string(), ] ); } #[test] fn duplicate_publisher_simulation_rejects_independent_source_material() { let mut scenario = DuplicatePublisherScenario::new( SimulationSeed::new(0x736f_7572_6365_6d61), vec!["nuc-a".to_string(), "nuc-b".to_string()], STREAM, RENDITION, TRACK, PROFILE, 0, 6, ); scenario.base_network_delay_ms = 0; scenario.max_jitter_ms = 0; scenario .publisher_source_material .insert("nuc-b".to_string(), "independent-rf-window".to_string()); let report = run_duplicate_publisher_simulation(&scenario); let invariant = check_duplicate_publisher_invariants( &report, &DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(1_000), ); assert_eq!( report.summary.divergent_sequences, (0_u64..6).collect::>() ); assert_eq!( report.summary.media_timing_conflict_sequences, Vec::::new() ); assert_eq!(report.fault_stats.source_material_mismatch_observations, 12); assert_eq!( invariant.failures, vec![ "divergent_sequences".to_string(), "source_material_mismatch_observations".to_string(), "duplicate_incomplete".to_string(), "duplicate_complete_deadline_unreached".to_string(), ] ); } #[test] fn duplicate_publisher_outage_backfills_after_restart() { let mut scenario = faulted_duplicate_scenario(SimulationSeed::new(0x6f75_7461_6765)); scenario.partitions.clear(); scenario.transient_drop_per_million = 0; scenario.publisher_outages = vec![SimulationOutage::new("nuc-b", 320, 760, 180)]; let report = run_duplicate_publisher_simulation(&scenario); assert!( report.duplicate_complete(), "{} {:?}", report.replay_hint, report.summary ); assert!(report.fault_stats.publisher_outage_observations > 0); assert_eq!( report.fault_stats.backfill_observations, report.fault_stats.publisher_outage_observations ); assert!( report.duplicate_complete_at_ms.unwrap() >= 940, "outage restart should move convergence later than the live path" ); assert!(report.duplicate_complete_at_ms.unwrap() <= 3_000); } #[test] fn duplicate_publisher_simulation_checks_convergence_deadline() { let report = run_duplicate_publisher_simulation(&faulted_duplicate_scenario( SimulationSeed::new(0x6465_6164_6c69_6e65), )); let invariant = check_duplicate_publisher_invariants( &report, &DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(3_000), ); assert!( invariant.ok, "{} {:?}", invariant.replay_hint, invariant.failures ); assert!(invariant.duplicate_complete_at_ms.is_some()); assert!( invariant.duplicate_complete_at_ms.unwrap() <= 3_000, "{} completed too late: {:?}", invariant.replay_hint, invariant.duplicate_complete_at_ms ); } #[test] fn seeded_simulation_campaign_preserves_first_failure() { let campaign = run_seeded_simulation_campaign( "generic-seeded-campaign", SimulationSeed::new(40), 8, |seed| (seed.0 == 44).then_some(seed.replay_hint()), ); assert!(!campaign.all_passed()); assert_eq!(campaign.passed, 7); assert_eq!(campaign.failed, 1); assert_eq!( campaign.first_failure.as_deref(), Some("EC_SIM_SEED=000000000000002c") ); } #[test] fn control_plane_propagation_replays_from_seed() { let scenario = faulted_control_plane_scenario(SimulationSeed::new(0x6374_726c_7265_706c)); let first = run_control_plane_propagation_simulation(&scenario); let second = run_control_plane_propagation_simulation(&scenario); assert_eq!(first, second); assert!( first.propagation_complete(), "control propagation failed for {}: {:?}", first.replay_hint, first.missing_nodes ); assert_eq!(first.known_count, scenario.nodes.len() as u64); assert_eq!( first.trace, second.trace, "replayed control-plane schedules should carry identical traces" ); assert!(first .trace .iter() .any(|entry| matches!(entry.event, ControlPlaneTraceEvent::MessageScheduled { .. }))); assert!(first .trace .iter() .any(|entry| matches!(entry.event, ControlPlaneTraceEvent::NodeLearned { .. }))); } #[test] fn control_plane_campaign_runs_many_fault_schedules() { let invariant = ControlPlanePropagationInvariantConfig::complete_with_deadline(7, 900); let campaign = run_control_plane_propagation_campaign( "control-plane-gossip-fault-campaign", SimulationSeed::new(1), 512, &invariant, faulted_control_plane_scenario, ); assert!( campaign.all_passed(), "campaign failed: {:?}", campaign.first_failure ); assert_eq!(campaign.passed, 512); assert_eq!(campaign.failed, 0); assert!(campaign.total_transient_dropped_messages > 0); assert!(campaign.total_partition_delayed_messages > 0); assert!(campaign.total_node_outage_delayed_messages > 0); assert!(campaign.total_duplicate_messages > 0); assert!(campaign.max_propagation_complete_ms_observed <= 900); } #[test] fn control_plane_simulation_detects_dead_fanout() { let mut scenario = faulted_control_plane_scenario(SimulationSeed::new(0x6661_6e6f_7574)); scenario.fanout = 0; scenario.transient_drop_per_million = 0; scenario.partitions.clear(); scenario.node_outages.clear(); let report = run_control_plane_propagation_simulation(&scenario); let invariant = check_control_plane_propagation_invariants( &report, &ControlPlanePropagationInvariantConfig::complete_with_deadline(7, 900), ); assert!(!report.propagation_complete()); assert_eq!(report.known_nodes, vec!["nuc-a".to_string()]); assert_eq!(report.missing_nodes.len(), 6); assert_eq!( invariant.failures, vec![ "propagation_incomplete".to_string(), "propagation_deadline_unreached".to_string(), ] ); } #[test] fn duplicate_publisher_campaign_runs_many_seed_schedules() { let invariant = DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(3_000); let campaign = run_duplicate_publisher_campaign( "duplicate-publisher-fault-campaign", SimulationSeed::new(1), 512, &invariant, faulted_duplicate_scenario, ); assert!( campaign.all_passed(), "campaign failed: {:?}", campaign.first_failure ); assert_eq!(campaign.passed, 512); assert_eq!(campaign.failed, 0); assert!(campaign.total_transient_dropped_observations > 0); assert!(campaign.total_partition_delayed_observations > 0); assert!(campaign.total_publisher_outage_observations > 0); assert!(campaign.total_backfill_observations > 0); assert!(campaign.max_duplicate_complete_ms_observed <= 3_000); } #[test] fn duplicate_publisher_shrinker_minimizes_noisy_drift_failure() { let invariant = DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(3_000); let mut scenario = faulted_duplicate_scenario(SimulationSeed::new(19)); scenario .encoder_drifts .push(EncoderDriftFault::new("nuc-b", 17, "x264-hd3-drift")); let shrunk = shrink_duplicate_publisher_failure(&scenario, &invariant) .expect("drift should fail and be shrinkable"); assert_eq!(shrunk.seed, SimulationSeed::new(19)); assert_eq!(shrunk.scenario.expected_sequences(), 18); assert_eq!(shrunk.scenario.partitions.len(), 0); assert_eq!(shrunk.scenario.publisher_outages.len(), 0); assert_eq!(shrunk.scenario.transient_drop_per_million, 0); assert_eq!(shrunk.scenario.max_jitter_ms, 0); assert_eq!(shrunk.scenario.base_network_delay_ms, 0); assert_eq!(shrunk.report.summary.divergent_sequences, vec![17]); assert_eq!( shrunk.invariant.failures, vec![ "divergent_sequences".to_string(), "duplicate_incomplete".to_string(), "duplicate_complete_deadline_unreached".to_string(), ] ); assert!( shrunk .steps .iter() .any(|step| step.dimension == "sequence_count" && step.after == "18"), "shrink steps should record the minimized failing media window" ); } #[test] fn duplicate_publisher_campaign_keeps_first_replayable_failure() { let invariant = DuplicatePublisherInvariantConfig::duplicate_complete_with_deadline(3_000); let campaign = run_duplicate_publisher_campaign( "duplicate-publisher-replayable-failure", SimulationSeed::new(10), 32, &invariant, |seed| { let mut scenario = faulted_duplicate_scenario(seed); if seed.0 == 19 { scenario .encoder_drifts .push(EncoderDriftFault::new("nuc-b", 17, "x264-hd3-drift")); } scenario }, ); let failure = campaign .first_failure .as_ref() .expect("campaign should preserve first failure"); let shrunk = failure .shrunk_failure .as_ref() .expect("campaign should preserve a shrunk replay"); assert_eq!(failure.seed, SimulationSeed::new(19)); assert_eq!( failure.invariant.failures, vec![ "divergent_sequences".to_string(), "duplicate_incomplete".to_string(), "duplicate_complete_deadline_unreached".to_string(), ] ); let mut replay = faulted_duplicate_scenario(failure.seed); replay .encoder_drifts .push(EncoderDriftFault::new("nuc-b", 17, "x264-hd3-drift")); let replayed_report = run_duplicate_publisher_simulation(&replay); assert_eq!(replayed_report, failure.report); assert_eq!(shrunk.scenario.expected_sequences(), 18); assert_eq!(shrunk.report.summary.divergent_sequences, vec![17]); } #[test] fn system_duplicate_publishers_converge_with_global_sequence_clock() { let scenario = system_duplicate_scenario( SimulationSeed::new(0x7379_7374_656d_676c), PublisherSequenceClock::Global, ); let report = run_system_duplicate_publisher_simulation(&scenario); let invariant = check_system_duplicate_publisher_invariants( &report, &SystemDuplicatePublisherInvariantConfig::complete_with_deadline(3_500), ); assert!( report.system_complete(), "{} control={:?} media={:?}", report.replay_hint, report.control.missing_nodes, report.media.summary ); assert!( invariant.ok, "{} {:?}", invariant.replay_hint, invariant.failures ); assert_eq!(report.media.summary.divergent_sequences, Vec::::new()); assert_eq!( report.media.summary.matching_duplicate_sequences.len() as u64, scenario.media.expected_sequences() ); assert!( report .publisher_activation_ms .get("nuc-b") .copied() .unwrap_or_default() > report .publisher_activation_ms .get("nuc-a") .copied() .unwrap_or_default(), "faulted control plane should activate nuc-b later than nuc-a" ); } #[test] fn system_duplicate_publishers_reject_local_activation_sequence_clock() { let scenario = system_duplicate_scenario( SimulationSeed::new(0x7379_7374_656d_6c6f), PublisherSequenceClock::LocalActivation, ); let report = run_system_duplicate_publisher_simulation(&scenario); let invariant = check_system_duplicate_publisher_invariants( &report, &SystemDuplicatePublisherInvariantConfig::complete_with_deadline(3_500), ); assert!(report.control.propagation_complete()); assert!(!report.media.duplicate_complete()); assert!( !report.media.summary.divergent_sequences.is_empty(), "local activation clock should cause same advertised sequence to hash differently" ); assert_eq!( invariant.failures, vec![ "media_divergent_sequences".to_string(), "media_timing_conflict_sequences".to_string(), "media_duplicate_incomplete".to_string(), "system_complete_deadline_unreached".to_string(), ] ); } #[test] fn system_duplicate_publisher_campaign_runs_many_seed_schedules() { let invariant = SystemDuplicatePublisherInvariantConfig::complete_with_deadline(3_500); let campaign = run_system_duplicate_publisher_campaign( "system-duplicate-publisher-fault-campaign", SimulationSeed::new(1), 256, &invariant, |seed| system_duplicate_scenario(seed, PublisherSequenceClock::Global), ); assert!( campaign.all_passed(), "campaign failed: {:?}", campaign.first_failure ); assert_eq!(campaign.passed, 256); assert_eq!(campaign.failed, 0); assert!(campaign.max_control_propagation_ms_observed > 0); assert!(campaign.max_media_duplicate_complete_ms_observed > 0); assert!(campaign.max_system_complete_ms_observed <= 3_500); assert!(campaign.total_system_complete_ms_observed > 0); assert!(campaign.total_control_trace_events > 0); assert!(campaign.total_media_trace_events > 0); assert_eq!( campaign.total_trace_events, campaign.total_control_trace_events + campaign.total_media_trace_events ); assert!(campaign.total_control_transient_drops > 0); assert!(campaign.total_media_transient_drops > 0); assert!(campaign.total_media_backfill_observations > 0); assert!(campaign.seeds_with_system_convergence_time > 0); assert!(campaign.seeds_with_control_transient_drops > 0); assert!(campaign.seeds_with_media_transient_drops > 0); assert!(campaign.seeds_with_media_backfill_observations > 0); assert!(!campaign.slowest_system_runs.is_empty()); assert!(campaign.slowest_system_runs.len() <= 16); assert!(campaign .slowest_system_runs .windows(2) .all(|pair| pair[0].system_complete_at_ms.unwrap_or(u64::MAX) >= pair[1].system_complete_at_ms.unwrap_or(u64::MAX))); assert_eq!(campaign.total_media_publisher_phase_offsets, 0); } #[test] fn foundation_style_system_campaign_runs_replayable_fault_schedules() { let invariant = SystemDuplicatePublisherInvariantConfig::complete_with_deadline(6_000); let config = FoundationStyleSystemScenarioConfig::default(); let campaign = run_system_duplicate_publisher_campaign( "foundation-style-system-campaign", SimulationSeed::new(1), 512, &invariant, |seed| ec_core::sim::foundation_style_system_duplicate_publisher_scenario(seed, &config), ); assert!( campaign.all_passed(), "campaign failed: {:?}", campaign.first_failure ); assert_eq!(campaign.passed, 512); assert_eq!(campaign.failed, 0); assert!(campaign.max_system_complete_ms_observed <= 6_000); assert!(campaign.total_system_complete_ms_observed > 0); assert!(campaign.total_control_trace_events > 0); assert!(campaign.total_media_trace_events > 0); assert_eq!( campaign.total_trace_events, campaign.total_control_trace_events + campaign.total_media_trace_events ); assert!(campaign.total_control_transient_drops > 0); assert!(campaign.total_control_partition_delays > 0); assert!(campaign.total_control_node_outage_delays > 0); assert!(campaign.total_control_duplicate_messages > 0); assert!(campaign.total_media_transient_drops > 0); assert!(campaign.total_media_partition_delays > 0); assert!(campaign.total_media_publisher_outages > 0); assert!(campaign.total_media_backfill_observations > 0); assert!(campaign.seeds_with_system_convergence_time > 0); assert!(campaign.seeds_with_control_propagation_time > 0); assert!(campaign.seeds_with_media_duplicate_convergence_time > 0); assert!(campaign.seeds_with_control_transient_drops > 0); assert!(campaign.seeds_with_control_partition_delays > 0); assert!(campaign.seeds_with_control_node_outage_delays > 0); assert!(campaign.seeds_with_control_duplicate_messages > 0); assert!(campaign.seeds_with_media_transient_drops > 0); assert!(campaign.seeds_with_media_partition_delays > 0); assert!(campaign.seeds_with_media_publisher_outages > 0); assert!(campaign.seeds_with_media_backfill_observations > 0); assert!(campaign.fault_coverage_ok()); assert!(!campaign.slowest_system_runs.is_empty()); assert!(campaign.slowest_system_runs.len() <= 16); assert_eq!(campaign.total_media_publisher_phase_offsets, 0); } #[test] fn foundation_style_system_campaign_rejects_local_activation_sequence_clock() { let invariant = SystemDuplicatePublisherInvariantConfig::complete_with_deadline(6_000); let mut config = FoundationStyleSystemScenarioConfig::default(); config.sequence_clock = PublisherSequenceClock::LocalActivation; let campaign = run_system_duplicate_publisher_campaign( "foundation-style-local-activation-failure", SimulationSeed::new(1), 32, &invariant, |seed| ec_core::sim::foundation_style_system_duplicate_publisher_scenario(seed, &config), ); let failure = campaign .first_failure .as_ref() .expect("local activation clock should fail under foundation-style faults"); assert!(!campaign.all_passed()); assert!(failure .invariant .failures .contains(&"media_divergent_sequences".to_string())); assert!(!failure.report.media.summary.divergent_sequences.is_empty()); assert!( failure .report .media .fault_stats .publisher_phase_offset_observations > 0 ); assert!(campaign.total_media_publisher_phase_offsets > 0); assert!(campaign.seeds_with_media_publisher_phase_offsets > 0); } #[test] fn system_duplicate_publisher_campaign_classifies_source_material_mismatch() { let invariant = SystemDuplicatePublisherInvariantConfig::complete_with_deadline(3_500); let campaign = run_system_duplicate_publisher_campaign( "system-source-material-failure", SimulationSeed::new(1), 1, &invariant, |seed| { let mut scenario = system_duplicate_scenario(seed, PublisherSequenceClock::Global); scenario .media .publisher_source_material .insert("nuc-b".to_string(), "independent-rf-window".to_string()); scenario }, ); let failure = campaign .first_failure .as_ref() .expect("source material mismatch should fail"); assert!(!campaign.all_passed()); assert!(failure .invariant .failures .contains(&"media_source_material_mismatch_observations".to_string())); assert!(!failure.report.media.summary.divergent_sequences.is_empty()); assert!( failure .report .media .fault_stats .source_material_mismatch_observations > 0 ); assert!(campaign.total_media_source_material_mismatches > 0); assert_eq!(campaign.seeds_with_media_source_material_mismatches, 1); } fn faulted_duplicate_scenario(seed: SimulationSeed) -> DuplicatePublisherScenario { let mut scenario = DuplicatePublisherScenario::new( seed, vec!["nuc-a".to_string(), "nuc-b".to_string()], STREAM, RENDITION, TRACK, PROFILE, 0, 48, ); scenario.segment_step_ms = 40; scenario.base_network_delay_ms = 5; scenario.max_jitter_ms = 75; scenario.transient_drop_per_million = 275_000; scenario.backfill_after_ms = 600; scenario.partitions = vec![ SimulationPartition::new("nuc-b", 120, 520, 140), SimulationPartition::new("nuc-a", 940, 1_260, 90), ]; scenario.publisher_outages = vec![SimulationOutage::new("nuc-b", 1_360, 1_520, 220)]; scenario } fn faulted_control_plane_scenario(seed: SimulationSeed) -> ControlPlanePropagationScenario { let mut scenario = ControlPlanePropagationScenario::new( seed, vec![ "nuc-a".to_string(), "nuc-b".to_string(), "tower".to_string(), "forge".to_string(), "relay-lax".to_string(), "relay-nyc".to_string(), "relay-hel".to_string(), ], "nuc-a", "ec.control.broadcast.la-kcop", "la-kcop@42", ); scenario.fanout = 3; scenario.gossip_interval_ms = 35; scenario.max_gossip_rounds = 12; scenario.base_network_delay_ms = 6; scenario.max_jitter_ms = 45; scenario.transient_drop_per_million = 120_000; scenario.partitions = vec![ SimulationPartition::new("relay-hel", 70, 190, 55), SimulationPartition::new("tower", 220, 310, 40), ]; scenario.node_outages = vec![SimulationOutage::new("relay-nyc", 105, 205, 45)]; scenario } fn system_duplicate_scenario( seed: SimulationSeed, sequence_clock: PublisherSequenceClock, ) -> SystemDuplicatePublisherScenario { let mut control = ControlPlanePropagationScenario::new( seed, vec![ "forge".to_string(), "nuc-a".to_string(), "nuc-b".to_string(), "tower".to_string(), "relay-lax".to_string(), "relay-nyc".to_string(), "relay-hel".to_string(), ], "forge", "ec.control.broadcast.la-kcop", "la-kcop@42", ); control.fanout = 3; control.gossip_interval_ms = 35; control.max_gossip_rounds = 12; control.base_network_delay_ms = 6; control.max_jitter_ms = 45; control.transient_drop_per_million = 120_000; control.partitions = vec![ SimulationPartition::new("nuc-b", 0, 180, 40), SimulationPartition::new("relay-hel", 70, 190, 55), ]; control.node_outages = vec![SimulationOutage::new("relay-nyc", 105, 205, 45)]; let mut media = DuplicatePublisherScenario::new( SimulationSeed::new(seed.0 ^ 0x6d65_6469_6121), vec!["nuc-a".to_string(), "nuc-b".to_string()], STREAM, RENDITION, TRACK, PROFILE, 0, 48, ); media.segment_step_ms = 40; media.base_network_delay_ms = 5; media.max_jitter_ms = 75; media.transient_drop_per_million = 275_000; media.backfill_after_ms = 600; media.partitions = vec![SimulationPartition::new("nuc-a", 940, 1_260, 90)]; media.publisher_outages = vec![SimulationOutage::new("nuc-b", 1_360, 1_520, 220)]; let mut scenario = SystemDuplicatePublisherScenario::new(seed, control, media); scenario.publisher_activation_delay_ms = 25; scenario.publisher_backfill_delay_ms = 180; scenario.sequence_clock = sequence_clock; scenario }