Add duplicate publisher determinism proof

crates/ec-node/Cargo.toml

@@ -29,17 +29,21 @@ rustls-native-certs = "0.8.3"
 urlencoding = "2"
 serde.workspace = true
 serde_json.workspace = true
+opentelemetry.workspace = true
+opentelemetry-otlp.workspace = true
+opentelemetry_sdk.workspace = true
 tokio = { version = "1", features = ["full"] }
 tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] }
 futures-util = "0.3"
 tracing.workspace = true
+tracing-opentelemetry.workspace = true
 tracing-subscriber.workspace = true
-web-transport-quinn = "0.11.4"
-web-transport-trait = "0.3.3"
-hang = "0.14.0"
-moq-mux = "0.2.1"
-moq-lite = "0.14.0"
-moq-native = { version = "0.13.1", default-features = true }
+web-transport-quinn = "0.11.9"
+web-transport-trait = "0.3.4"
+hang = "0.16.0"
+moq-mux = "0.4.0"
+moq-lite = "0.16.0"
+moq-native = { version = "0.14.0", default-features = true }
 headless_chrome = "1"
 tokio-util = "0.7"
 url = "2"

crates/ec-node/src/nbc.rs

@@ -2,7 +2,7 @@ use anyhow::{anyhow, Context, Result};
 use headless_chrome::protocol::cdp::Page;
 use headless_chrome::{Browser, Tab};
 use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::env;
 use std::fs;
 use std::io::{BufRead, BufReader, Cursor, Read, Write};
@@ -65,11 +65,15 @@ pub struct BootstrapResult {
     pub page_url: String,
     pub interactive_auth_required: bool,
     pub authorized: bool,
+    pub video_ready: bool,
+    pub current_time: f64,
+    pub width: u64,
+    pub height: u64,
     pub screenshot_path: Option<PathBuf>,
 }
 
-#[derive(Debug)]
 struct WaitOutcome {
+    tab: Arc<Tab>,
     state: NbcVideoState,
     trace: NbcTraceState,
     interactive_auth_required: bool,
@@ -199,6 +203,14 @@ fn nbc_bootstrap_timeout() -> Duration {
         .unwrap_or_else(|| Duration::from_secs(1800))
 }
 
+fn nbc_profile_signin_gate_timeout() -> Duration {
+    env::var("EVERY_CHANNEL_NBC_PROFILE_SIGNIN_GATE_TIMEOUT_SECS")
+        .ok()
+        .and_then(|value| value.parse::<u64>().ok())
+        .map(Duration::from_secs)
+        .unwrap_or_else(|| Duration::from_secs(8))
+}
+
 fn nbc_env_flag(name: &str) -> Option<bool> {
     env::var(name).ok().map(|value| {
         let value = value.trim().to_ascii_lowercase();
@@ -403,6 +415,10 @@ pub fn bootstrap_nbc_auth(
     )?;
 
     Ok(BootstrapResult {
+        video_ready: nbc_video_state_has_decoded_frame(&outcome.state),
+        current_time: outcome.state.current_time,
+        width: outcome.state.width,
+        height: outcome.state.height,
         title: outcome.state.title,
         page_url: outcome.state.page_url,
         interactive_auth_required: outcome.interactive_auth_required,
@@ -619,7 +635,7 @@ fn run_nbc_capture_loop(
         register_nbc_trace_handlers(&tab, trace.clone())?;
         tab.navigate_to(&url)?;
         tab.wait_until_navigated()?;
-        wait_for_nbc_playback(
+        let outcome = wait_for_nbc_playback(
             chrome.browser(),
             &tab,
             &url,
@@ -627,31 +643,34 @@ fn run_nbc_capture_loop(
             AuthMode::Forbidden,
             None,
         )?;
+        let capture_tab = outcome.tab;
 
         let frame_interval = Duration::from_millis(1000 / nbc_capture_fps().max(1));
         let quality = nbc_capture_quality();
         let mut first_frame = true;
 
         loop {
-            kick_nbc_player(&tab).ok();
-            let frame = tab
+            kick_nbc_player(&capture_tab).ok();
+            let state = probe_nbc_video(&capture_tab).unwrap_or_default();
+            if !nbc_video_state_has_decoded_frame(&state) {
+                return Err(anyhow!(
+                    "NBC capture tab lost decoded video (title='{}', page_url='{}', current_time={}, ready_state={}, has_video={})",
+                    state.title,
+                    state.page_url,
+                    state.current_time,
+                    state.ready_state,
+                    state.has_video,
+                ));
+            }
+            let video = capture_tab
                 .find_element("video")
-                .and_then(|video| {
-                    video.parent.capture_screenshot(
-                        Page::CaptureScreenshotFormatOption::Jpeg,
-                        Some(quality),
-                        Some(video.get_box_model()?.content_viewport()),
-                        true,
-                    )
-                })
-                .or_else(|_| {
-                    tab.capture_screenshot(
-                        Page::CaptureScreenshotFormatOption::Jpeg,
-                        Some(quality),
-                        None,
-                        true,
-                    )
-                })?;
+                .context("NBC capture tab has no video element after playback readiness")?;
+            let frame = video.parent.capture_screenshot(
+                Page::CaptureScreenshotFormatOption::Jpeg,
+                Some(quality),
+                Some(video.get_box_model()?.content_viewport()),
+                true,
+            )?;
 
             if first_frame {
                 first_frame = false;
@@ -785,6 +804,15 @@ fn nbc_url_is_provider_linked(url: &str) -> bool {
     (host.ends_with("nbc.com") || host.ends_with(".nbc.com")) && path.contains("provider-linked")
 }
 
+fn nbc_url_is_mvpd_complete(url: &str) -> bool {
+    let Ok(url) = Url::parse(url) else {
+        return false;
+    };
+    let host = url.host_str().unwrap_or_default().to_ascii_lowercase();
+    let path = url.path().to_ascii_lowercase();
+    (host.ends_with("nbc.com") || host.ends_with(".nbc.com")) && path.contains("mvpd-complete")
+}
+
 fn nbc_url_is_optional_profile_signin(url: &str) -> bool {
     let Ok(url) = Url::parse(url) else {
         return false;
@@ -812,6 +840,7 @@ fn nbc_page_is_watch_surface(url: &str) -> bool {
     (host.ends_with("nbc.com") || host.ends_with(".nbc.com"))
         && !nbc_url_is_optional_profile_signin(url.as_str())
         && !nbc_url_is_provider_linked(url.as_str())
+        && !nbc_url_is_mvpd_complete(url.as_str())
 }
 
 fn nbc_title_looks_like_verizon_popup(title: &str) -> bool {
@@ -836,6 +865,14 @@ fn nbc_state_is_optional_profile_signin(state: &NbcVideoState) -> bool {
         || nbc_title_looks_like_optional_profile_signin(&state.title)
 }
 
+fn nbc_clues_look_geo_blocked(clues: &NbcPageClues) -> bool {
+    let body_text = clues.body_text.to_ascii_lowercase();
+    body_text.contains("not authorized to access this content from outside of the us")
+        || body_text.contains("not authorized to access this content from outside of the u.s.")
+        || body_text.contains("outside of the us and its territories")
+        || body_text.contains("outside of the u.s. and its territories")
+}
+
 fn browser_tabs(browser: &Browser) -> Vec<Arc<Tab>> {
     browser.register_missing_tabs();
     browser.get_tabs().lock().unwrap().iter().cloned().collect()
@@ -877,20 +914,20 @@ fn find_primary_tab_state<'a>(
         .find(|candidate| candidate.tab.get_target_id() == target_id)
 }
 
-fn find_playing_tab_state(tabs: &[BrowserTabState]) -> Option<&BrowserTabState> {
-    tabs.iter().find(|candidate| {
-        candidate.state.has_video
-            && candidate.state.width > 0
-            && candidate.state.height > 0
-            && !candidate.state.paused
-            && (candidate.state.current_time > 0.0 || candidate.state.ready_state >= 2)
-    })
+fn nbc_video_state_has_decoded_frame(state: &NbcVideoState) -> bool {
+    state.has_video
+        && state.width > 0
+        && state.height > 0
+        && !state.paused
+        && state.current_time > 0.0
+        && state.ready_state >= 2
 }
 
 fn find_provider_linked_tab_state(tabs: &[BrowserTabState]) -> Option<&BrowserTabState> {
     tabs.iter().find(|candidate| {
         nbc_title_looks_like_provider_linked(&candidate.state.title)
             || nbc_url_is_provider_linked(&candidate.state.page_url)
+            || nbc_url_is_mvpd_complete(&candidate.state.page_url)
     })
 }
 
@@ -1038,25 +1075,40 @@ fn advance_nbc_auth_flow(tab: &Arc<Tab>) -> Result<Option<NbcAuthAdvanceResult>>
           }};
           const actions = [];
           const url = window.location.href || "";
+          let host = "";
+          try {{
+            host = new URL(url).hostname.toLowerCase();
+          }} catch (_err) {{}}
+          const title = document.title || "";
+          const titleText = `${{title}} ${{url}}`.toLowerCase();
+          const looksLikeOptionalNbcProfile =
+            (host.endsWith("nbc.com") || host.endsWith(".nbc.com")) &&
+            (url.includes("/sign-in") ||
+              url.includes("/login") ||
+              titleText.includes("nbc account sign in") ||
+              titleText.includes("nbcuniversal profile") ||
+              titleText.includes("nbc profile"));
+          if (looksLikeOptionalNbcProfile) {{
+            return {{ pageUrl: url, title, actions }};
+          }}
           const candidates = Array.from(
             document.querySelectorAll(
               "button,a,[role='button'],[role='option'],label,li,[data-provider-name],[data-provider-id],[data-provider]"
             )
           );
+          const providerCta = candidates.find((node) => {{
+            const text = textOf(node);
+            return visible(node) &&
+              (
+                text === "link tv provider" ||
+                text === "link provider" ||
+                text.startsWith("link tv provider ") ||
+                text.startsWith("link provider ")
+              );
+          }});
+          clickNode(providerCta, "click:link-provider");
 
           if (url.includes("mvpd")) {{
-            const providerCta = candidates.find((node) => {{
-              const text = textOf(node);
-              return visible(node) &&
-                (
-                  text === "link tv provider" ||
-                  text === "link provider" ||
-                  text.startsWith("link tv provider ") ||
-                  text.startsWith("link provider ")
-                );
-            }});
-            clickNode(providerCta, "click:link-provider");
-
             const fullListNode = candidates.find((node) => {{
               const text = textOf(node);
               return visible(node) && (text === "full list" || text.startsWith("full list "));
@@ -1112,7 +1164,7 @@ fn advance_nbc_auth_flow(tab: &Arc<Tab>) -> Result<Option<NbcAuthAdvanceResult>>
 
           return {{
             pageUrl: url,
-            title: document.title || "",
+            title,
             actions,
           }};
         }})())
@@ -1226,16 +1278,30 @@ fn advance_mvpd_login_flow(tab: &Arc<Tab>) -> Result<Option<NbcAuthAdvanceResult
               titleText.includes("nbc profile"));
           if (looksLikeOptionalNbcProfile) {{
             const profileButtons = Array.from(document.querySelectorAll("button,a,[role='button'],input[type='submit'],input[type='button']"));
-            const providerLink = profileButtons.find((node) => {{
+            const dismissButton = profileButtons.find((node) => {{
               const text = textOf(node);
               return visible(node) && (
-                text === "link tv provider" ||
-                text === "link provider" ||
-                text.startsWith("link tv provider ") ||
-                text.startsWith("link provider ")
+                text === "skip" ||
+                text.startsWith("skip ") ||
+                text === "skip for now" ||
+                text === "maybe later" ||
+                text === "not now" ||
+                text === "no thanks" ||
+                text === "close" ||
+                text === "continue watching" ||
+                text.startsWith("continue watching ") ||
+                text === "continue without signing in" ||
+                text === "continue without profile" ||
+                text === "continue as guest" ||
+                text === "watch live" ||
+                text === "watch now" ||
+                text.startsWith("watch live ") ||
+                text.startsWith("watch now ")
               );
             }});
-            clickNode(providerLink, "click:profile-link-provider");
+            if (dismissButton) {{
+              clickNode(dismissButton, `click:profile-dismiss:${{textOf(dismissButton).slice(0, 120)}}`);
+            }}
             return {{ pageUrl: url, title, actions }};
           }}
           if (!looksLikeProviderLogin) {{
@@ -1333,8 +1399,15 @@ fn advance_nbc_post_auth_flow(tab: &Arc<Tab>) -> Result<Option<NbcAuthAdvanceRes
           const bodyText = normalize(document.body?.innerText || "");
           const looksLinked = title.toLowerCase().includes("tv provider linked")
             || url.includes("provider-linked")
+            || url.includes("mvpd-complete")
             || bodyText.includes("tv provider linked");
-          if (!looksLinked) {
+          const looksOptionalProfile =
+            (url.includes("/sign-in") ||
+              url.includes("/login") ||
+              normalize(title).includes("nbc account sign in") ||
+              normalize(title).includes("nbcuniversal profile") ||
+              normalize(title).includes("nbc profile"));
+          if (!looksLinked && !looksOptionalProfile) {
             return { pageUrl: url, title, actions };
           }
 
@@ -1344,8 +1417,22 @@ fn advance_nbc_post_auth_flow(tab: &Arc<Tab>) -> Result<Option<NbcAuthAdvanceRes
             return visible(node) && (
               text === "skip" ||
               text.startsWith("skip ") ||
-              text === "continue" ||
-              text.startsWith("continue watching")
+              text === "skip for now" ||
+              (looksLinked && text === "continue") ||
+              (looksLinked && text.startsWith("continue ")) ||
+              text === "maybe later" ||
+              text === "not now" ||
+              text === "no thanks" ||
+              text === "close" ||
+              text === "continue watching" ||
+              text.startsWith("continue watching") ||
+              text === "continue without signing in" ||
+              text === "continue without profile" ||
+              text === "continue as guest" ||
+              text === "watch live" ||
+              text === "watch now" ||
+              text.startsWith("watch live ") ||
+              text.startsWith("watch now ")
             );
           });
           if (skipButton) {
@@ -1533,6 +1620,7 @@ fn wait_for_nbc_playback(
     screenshot_out: Option<PathBuf>,
 ) -> Result<WaitOutcome> {
     let deadline = Instant::now() + nbc_capture_timeout();
+    let auth_forbidden = matches!(&auth_mode, AuthMode::Forbidden);
     let mut interactive_deadline = None::<Instant>;
     let mut interactive_auth_required = false;
     let mut screenshot_path = None::<PathBuf>;
@@ -1540,11 +1628,13 @@ fn wait_for_nbc_playback(
     let mut last_trace_state = None::<NbcTraceState>;
     let mut last_log = Instant::now() - Duration::from_secs(10);
     let mut last_clue_log = Instant::now() - Duration::from_secs(30);
+    let mut playback_samples = HashMap::<String, (f64, Instant)>::new();
     let mut resumed_after_background_login = false;
     let mut resumed_after_authenticated_surface = false;
     let mut optional_profile_signin_recoveries = 0_u8;
     let mut last_optional_profile_signin_retry = None::<Instant>;
     let mut watch_surface_seen_at = None::<Instant>;
+    let mut optional_profile_signin_seen_at = None::<Instant>;
     let mut tracked_tabs = HashSet::new();
     let mut provider_linked_completed = false;
 
@@ -1554,13 +1644,33 @@ fn wait_for_nbc_playback(
         let primary_state = find_primary_tab_state(&tab_states, tab)
             .map(|value| value.state.clone())
             .unwrap_or_else(|| probe_nbc_video(tab).unwrap_or_default());
-        if let Some(playing_tab) = find_playing_tab_state(&tab_states) {
-            return Ok(WaitOutcome {
-                state: playing_tab.state.clone(),
-                trace: trace.lock().map(|state| state.clone()).unwrap_or_default(),
-                interactive_auth_required,
-                screenshot_path,
-            });
+        let now = Instant::now();
+        for playing_tab in tab_states
+            .iter()
+            .filter(|candidate| nbc_video_state_has_decoded_frame(&candidate.state))
+        {
+            let target_id = playing_tab.tab.get_target_id().to_string();
+            if let Some((previous_time, first_seen)) = playback_samples.get(&target_id) {
+                if playing_tab.state.current_time >= *previous_time + 0.25
+                    && first_seen.elapsed() >= Duration::from_millis(500)
+                {
+                    return Ok(WaitOutcome {
+                        tab: playing_tab.tab.clone(),
+                        state: playing_tab.state.clone(),
+                        trace: trace.lock().map(|state| state.clone()).unwrap_or_default(),
+                        interactive_auth_required,
+                        screenshot_path,
+                    });
+                }
+            }
+            playback_samples
+                .entry(target_id)
+                .and_modify(|(previous_time, _)| {
+                    if playing_tab.state.current_time < *previous_time {
+                        *previous_time = playing_tab.state.current_time;
+                    }
+                })
+                .or_insert((playing_tab.state.current_time, now));
         }
 
         let interaction_tab = find_interaction_tab_state(&tab_states, tab)
@@ -1571,6 +1681,7 @@ fn wait_for_nbc_playback(
         let pre_state = probe_nbc_video(&interaction_tab).unwrap_or_default();
         if nbc_title_looks_like_provider_linked(&pre_state.title)
             || nbc_url_is_provider_linked(&pre_state.page_url)
+            || nbc_url_is_mvpd_complete(&pre_state.page_url)
         {
             provider_linked_completed = true;
         }
@@ -1579,6 +1690,7 @@ fn wait_for_nbc_playback(
         if let Some(progress) = advance_nbc_post_auth_flow(&interaction_tab).ok().flatten() {
             if nbc_title_looks_like_provider_linked(&progress.title)
                 || nbc_url_is_provider_linked(&progress.page_url)
+                || nbc_url_is_mvpd_complete(&progress.page_url)
                 || progress
                     .actions
                     .iter()
@@ -1614,6 +1726,7 @@ fn wait_for_nbc_playback(
         let state = probe_nbc_video(&interaction_tab).unwrap_or_default();
         if nbc_title_looks_like_provider_linked(&state.title)
             || nbc_url_is_provider_linked(&state.page_url)
+            || nbc_url_is_mvpd_complete(&state.page_url)
         {
             provider_linked_completed = true;
         }
@@ -1621,6 +1734,19 @@ fn wait_for_nbc_playback(
         let authorized = nbc_trace_is_authorized(&trace_state) || provider_linked_completed;
         let recent_media_activity = nbc_trace_has_recent_media_activity(&trace_state);
 
+        if !authorized && nbc_state_is_optional_profile_signin(&state) && !state.has_video {
+            let first_seen = *optional_profile_signin_seen_at.get_or_insert_with(Instant::now);
+            if auth_forbidden && first_seen.elapsed() >= nbc_profile_signin_gate_timeout() {
+                return Err(anyhow!(
+                    "NBC account sign-in gate reached before TV-provider auth; refusing non-interactive retry loop without decoded video (title='{}', page_url='{}')",
+                    state.title,
+                    state.page_url,
+                ));
+            }
+        } else {
+            optional_profile_signin_seen_at = None;
+        }
+
         if last_log.elapsed() >= Duration::from_secs(5) {
             last_log = Instant::now();
             tracing::info!(
@@ -1661,8 +1787,9 @@ fn wait_for_nbc_playback(
             }
         }
 
-        if (trace_state.background_login_complete
-            || nbc_url_is_background_login_complete(&state.page_url))
+        let auth_completion_page = nbc_url_is_background_login_complete(&state.page_url)
+            || nbc_url_is_mvpd_complete(&state.page_url);
+        if (trace_state.background_login_complete || auth_completion_page)
             && !resumed_after_background_login
         {
             resumed_after_background_login = true;
@@ -1673,41 +1800,49 @@ fn wait_for_nbc_playback(
             );
             close_auxiliary_browser_tabs(browser, tab);
             let _ = tab.activate();
-            let _ = tab.evaluate("window.location.reload()", true);
+            if nbc_url_is_mvpd_complete(&state.page_url) {
+                tab.navigate_to(source_url)?;
+                tab.wait_until_navigated()?;
+            } else {
+                let _ = tab.evaluate("window.location.reload()", true);
+            }
             std::thread::sleep(Duration::from_secs(2));
             continue;
         }
 
-        if authorized
-            && nbc_state_is_optional_profile_signin(&state)
-            && !recent_media_activity
-            && optional_profile_signin_recoveries < 3
-            && last_optional_profile_signin_retry
-                .map(|instant| instant.elapsed() >= Duration::from_secs(3))
-                .unwrap_or(true)
-        {
-            optional_profile_signin_recoveries += 1;
-            last_optional_profile_signin_retry = Some(Instant::now());
-            tracing::info!(
-                title = %state.title,
-                page_url = %state.page_url,
-                authorized,
-                source_url,
-                optional_profile_signin_recoveries,
-                "NBC profile sign-in surface detected after authorization; returning to the live source URL"
-            );
-            close_auxiliary_browser_tabs(browser, tab);
-            let _ = tab.activate();
-            tab.navigate_to(source_url)?;
-            tab.wait_until_navigated()?;
-            std::thread::sleep(Duration::from_secs(2));
-            continue;
+        if authorized && nbc_state_is_optional_profile_signin(&state) && !state.has_video {
+            if optional_profile_signin_recoveries == 0
+                && last_optional_profile_signin_retry
+                    .map(|instant| instant.elapsed() >= Duration::from_secs(3))
+                    .unwrap_or(true)
+            {
+                optional_profile_signin_recoveries += 1;
+                last_optional_profile_signin_retry = Some(Instant::now());
+                tracing::info!(
+                    title = %state.title,
+                    page_url = %state.page_url,
+                    authorized,
+                    source_url,
+                    "NBC account sign-in gate detected after provider authorization; trying one live-url recovery"
+                );
+                close_auxiliary_browser_tabs(browser, tab);
+                let _ = tab.activate();
+                tab.navigate_to(source_url)?;
+                tab.wait_until_navigated()?;
+                std::thread::sleep(Duration::from_secs(2));
+                continue;
+            }
+            return Err(anyhow!(
+                "NBC account sign-in gate reached after TV-provider auth; refusing retry loop without decoded video (title='{}', page_url='{}')",
+                state.title,
+                state.page_url,
+            ));
         }
-        if authorized && nbc_state_is_optional_profile_signin(&state) && recent_media_activity {
+        if authorized && nbc_state_is_optional_profile_signin(&state) && state.has_video {
             tracing::debug!(
                 title = %state.title,
                 page_url = %state.page_url,
-                "NBC optional profile sign-in is visible but media activity is already in flight; staying on the page"
+                "NBC optional profile sign-in is visible but a video element is already present; staying on the page"
             );
         }
 
@@ -1733,6 +1868,13 @@ fn wait_for_nbc_playback(
                     body_text = %clues.body_text,
                     "NBC watch surface clues"
                 );
+                if nbc_clues_look_geo_blocked(&clues) {
+                    return Err(anyhow!(
+                        "NBC geo-blocked current egress; page says this content is not authorized outside the US/territories (title='{}', page_url='{}')",
+                        primary_state.title,
+                        primary_state.page_url,
+                    ));
+                }
             }
         }
         if fully_loaded_watch_surface && !primary_state.has_video {
@@ -1862,6 +2004,9 @@ mod tests {
         assert!(nbc_url_is_provider_linked(
             "https://www.nbc.com/provider-linked"
         ));
+        assert!(nbc_url_is_mvpd_complete(
+            "https://www.nbc.com/mvpd-complete"
+        ));
         assert!(nbc_title_looks_like_provider_linked("TV Provider Linked"));
         assert!(!nbc_url_is_provider_linked(
             "https://www.nbc.com/live?brand=nbc-sports-philadelphia"
@@ -1884,11 +2029,31 @@ mod tests {
     #[test]
     fn optional_profile_signin_is_not_treated_as_watch_surface() {
         assert!(!nbc_page_is_watch_surface("https://www.nbc.com/sign-in"));
+        assert!(!nbc_page_is_watch_surface(
+            "https://www.nbc.com/mvpd-complete"
+        ));
         assert!(nbc_page_is_watch_surface(
             "https://www.nbc.com/live?brand=nbc-sports-philadelphia"
         ));
     }
 
+    #[test]
+    fn geo_block_clues_fail_closed() {
+        let clues = NbcPageClues {
+            body_text:
+                "We're sorry. You are not authorized to access this content from outside of the US and its territories."
+                    .to_string(),
+            ..NbcPageClues::default()
+        };
+        assert!(nbc_clues_look_geo_blocked(&clues));
+
+        let allowed = NbcPageClues {
+            body_text: "NBC News NOW ON NOW until 7:00 AM".to_string(),
+            ..NbcPageClues::default()
+        };
+        assert!(!nbc_clues_look_geo_blocked(&allowed));
+    }
+
     #[test]
     fn cssott_media_requests_mark_recent_media_activity() {
         let mut trace = NbcTraceState::default();
@@ -1899,4 +2064,25 @@ mod tests {
         assert!(trace.media_activity_seen);
         assert!(nbc_trace_has_recent_media_activity(&trace));
     }
+
+    #[test]
+    fn decoded_frame_detection_requires_advancing_video_surface() {
+        let mut state = NbcVideoState {
+            has_video: true,
+            width: 1920,
+            height: 1080,
+            paused: false,
+            ready_state: 2,
+            current_time: 1.0,
+            ..NbcVideoState::default()
+        };
+        assert!(nbc_video_state_has_decoded_frame(&state));
+
+        state.current_time = 0.0;
+        assert!(!nbc_video_state_has_decoded_frame(&state));
+
+        state.current_time = 1.0;
+        state.width = 0;
+        assert!(!nbc_video_state_has_decoded_frame(&state));
+    }
 }

crates/ec-node/tests/determinism_cmaf_ladder.rs

@@ -1,3 +1,4 @@
+use std::collections::BTreeMap;
 use std::io::{BufRead, BufReader};
 use std::path::Path;
 use std::process::{Command, Stdio};
@@ -46,6 +47,15 @@ fn blake3_hex(path: &Path) -> anyhow::Result<String> {
     Ok(blake3::hash(&bytes).to_hex().to_string())
 }
 
+fn command_available(name: &str) -> bool {
+    Command::new(name)
+        .arg("-version")
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .is_ok()
+}
+
 fn concat_init_and_segment(init: &Path, seg: &Path, out: &Path) -> anyhow::Result<()> {
     let init_bytes = std::fs::read(init)?;
     let seg_bytes = std::fs::read(seg)?;
@@ -157,11 +167,15 @@ fn write_deterministic_ts(out_path: &Path) -> anyhow::Result<()> {
     Ok(())
 }
 
-fn run_ladder(ec_node: &Path, input_ts: &Path, out_dir: &Path) -> anyhow::Result<()> {
+fn run_ladder_with_identity(
+    ec_node: &Path,
+    input_ts: &Path,
+    out_dir: &Path,
+    stream_id: &str,
+    broadcast_name: &str,
+) -> anyhow::Result<()> {
     let signing_key = "11".repeat(32);
     let network_secret = "22".repeat(32);
-    let stream_id = "every.channel/determinism/cmaf-ladder";
-    let broadcast_name = "every.channel/determinism/cmaf-ladder";
 
     let mut cmd = Command::new(ec_node);
     cmd.env("EVERY_CHANNEL_MANIFEST_SIGNING_KEY", &signing_key)
@@ -210,6 +224,40 @@ fn run_ladder(ec_node: &Path, input_ts: &Path, out_dir: &Path) -> anyhow::Result
     Ok(())
 }
 
+fn run_ladder(ec_node: &Path, input_ts: &Path, out_dir: &Path) -> anyhow::Result<()> {
+    run_ladder_with_identity(
+        ec_node,
+        input_ts,
+        out_dir,
+        "every.channel/determinism/cmaf-ladder",
+        "every.channel/determinism/cmaf-ladder",
+    )
+}
+
+fn ladder_artifact_hashes(root: &Path) -> BTreeMap<String, String> {
+    let mut hashes = BTreeMap::new();
+    for variant in ["1080p", "720p", "480p"] {
+        let variant_dir = root.join("cmaf-ladder").join(variant);
+        // `moq-publish --max-chunks 3` publishes init plus segments 0..=2.
+        // ffmpeg can race ahead and leave an unpublished tail segment before it is killed.
+        let init = variant_dir.join("init.mp4");
+        assert!(init.exists(), "missing init for {variant}");
+        hashes.insert(format!("{variant}/init.mp4"), blake3_hex(&init).unwrap());
+
+        for idx in 0..3 {
+            let name = format!("segment_{idx:06}.m4s");
+            let path = variant_dir.join(&name);
+            assert!(path.exists(), "missing {name} for {variant}");
+            hashes.insert(format!("{variant}/{name}"), blake3_hex(&path).unwrap());
+        }
+    }
+    hashes
+}
+
+fn assert_ladder_bytes_match(left: &Path, right: &Path) {
+    assert_eq!(ladder_artifact_hashes(left), ladder_artifact_hashes(right));
+}
+
 #[test]
 #[ignore]
 fn deterministic_cmaf_ladder_outputs_match_across_runs() {
@@ -235,36 +283,53 @@ fn deterministic_cmaf_ladder_outputs_match_across_runs() {
     run_ladder(&ec_node, &input_ts, &run1).expect("run ladder 1");
     run_ladder(&ec_node, &input_ts, &run2).expect("run ladder 2");
 
-    for variant in ["1080p", "720p", "480p"] {
-        let v1 = run1.join("cmaf-ladder").join(variant);
-        let v2 = run2.join("cmaf-ladder").join(variant);
+    assert_ladder_bytes_match(&run1, &run2);
+}
 
-        let init1 = v1.join("init.mp4");
-        let init2 = v2.join("init.mp4");
-        assert!(
-            init1.exists() && init2.exists(),
-            "missing init for {variant}"
-        );
-        assert_eq!(
-            blake3_hex(&init1).unwrap(),
-            blake3_hex(&init2).unwrap(),
-            "init differs for {variant}"
-        );
-
-        for idx in 0..3 {
-            let s1 = v1.join(format!("segment_{idx:06}.m4s"));
-            let s2 = v2.join(format!("segment_{idx:06}.m4s"));
-            assert!(
-                s1.exists() && s2.exists(),
-                "missing segment {idx} for {variant}"
-            );
-            assert_eq!(
-                blake3_hex(&s1).unwrap(),
-                blake3_hex(&s2).unwrap(),
-                "segment {idx} differs for {variant}"
-            );
-        }
+#[test]
+fn duplicate_publishers_same_input_produce_identical_cmaf_ladder_bytes() {
+    if !command_available("ffmpeg") {
+        eprintln!("skipping duplicate publisher CMAF ladder determinism test: ffmpeg unavailable");
+        return;
     }
+
+    let ec_node = ec_node_path();
+
+    let ts = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis();
+    let tmp = std::env::temp_dir().join(format!("ec-duplicate-publisher-cmaf-ladder-{ts}"));
+    let _ = std::fs::create_dir_all(&tmp);
+
+    let input_ts = tmp.join("input.ts");
+    write_deterministic_ts(&input_ts).expect("write deterministic TS");
+
+    let publisher_a = tmp.join("publisher-a");
+    let publisher_b = tmp.join("publisher-b");
+    let _ = std::fs::remove_dir_all(&publisher_a);
+    let _ = std::fs::remove_dir_all(&publisher_b);
+    std::fs::create_dir_all(&publisher_a).unwrap();
+    std::fs::create_dir_all(&publisher_b).unwrap();
+
+    run_ladder_with_identity(
+        &ec_node,
+        &input_ts,
+        &publisher_a,
+        "every.channel/determinism/duplicate/publisher-a/la-kcop",
+        "publisher-a-la-kcop",
+    )
+    .expect("run duplicate publisher a");
+    run_ladder_with_identity(
+        &ec_node,
+        &input_ts,
+        &publisher_b,
+        "every.channel/determinism/duplicate/publisher-b/la-kcop",
+        "publisher-b-la-kcop",
+    )
+    .expect("run duplicate publisher b");
+
+    assert_ladder_bytes_match(&publisher_a, &publisher_b);
 }
 
 #[test]

crates/ec-node/tests/e2e_remote_website_watch_existing.rs

@@ -1,4 +1,5 @@
 use std::ffi::OsStr;
+use std::process::{Command, Stdio};
 use std::time::{Duration, Instant};
 
 fn which(cmd: &str) -> Option<std::path::PathBuf> {
@@ -16,6 +17,24 @@ fn chrome_path() -> Option<std::path::PathBuf> {
         .or_else(|| which("chromium"))
 }
 
+fn ec_node_path() -> std::path::PathBuf {
+    if let Ok(value) = std::env::var("EC_NODE_BIN") {
+        return value.into();
+    }
+    if let Ok(value) = std::env::var("CARGO_BIN_EXE_ec_node") {
+        return value.into();
+    }
+    if let Ok(value) = std::env::var("CARGO_BIN_EXE_ec-node") {
+        return value.into();
+    }
+    let exe = std::env::current_exe().expect("current_exe");
+    let debug_dir = exe
+        .parent()
+        .and_then(|p| p.parent())
+        .expect("expected target/debug/deps");
+    debug_dir.join("ec-node")
+}
+
 fn wait_for_canvas_element(tab: &headless_chrome::Tab, timeout: Duration) -> anyhow::Result<()> {
     let deadline = Instant::now() + timeout;
     while Instant::now() < deadline {
@@ -46,14 +65,41 @@ fn wait_for_moq_watch_element(tab: &headless_chrome::Tab, timeout: Duration) ->
     anyhow::bail!("timed out waiting for <moq-watch> element");
 }
 
+fn wait_for_live_or_archive_player(
+    tab: &headless_chrome::Tab,
+    timeout: Duration,
+) -> anyhow::Result<()> {
+    let deadline = Instant::now() + timeout;
+    while Instant::now() < deadline {
+        let js = r#"(function() {
+  return !!document.querySelector('moq-watch, video.archiveVideo');
+})();"#;
+        let v = tab.evaluate(js, false)?;
+        if v.value.and_then(|v| v.as_bool()).unwrap_or(false) {
+            return Ok(());
+        }
+        std::thread::sleep(Duration::from_millis(200));
+    }
+    anyhow::bail!("timed out waiting for live or archive player");
+}
+
 fn debug_player_state(tab: &headless_chrome::Tab) -> anyhow::Result<String> {
     let js = r#"(function() {
   let watch = document.querySelector('moq-watch');
   let canvas = document.querySelector('moq-watch canvas');
+  let video = document.querySelector('video.archiveVideo');
   let placeholder = document.querySelector('.placeholder');
   let placeholderText = placeholder ? (placeholder.innerText || '') : null;
   let status = document.querySelector('.source-status');
   let statusText = status ? (status.innerText || '') : null;
+  let statusLine = document.querySelector('#statusLine');
+  let statusLineText = statusLine ? (statusLine.innerText || '') : null;
+  let catalog = watch && watch.broadcast && watch.broadcast.catalog && watch.broadcast.catalog.peek
+    ? watch.broadcast.catalog.peek()
+    : null;
+  let established = watch && watch.connection && watch.connection.established && watch.connection.established.peek
+    ? watch.connection.established.peek()
+    : null;
   let sources = Array.from(document.querySelectorAll('button[data-testid="global-watch"]')).length;
   let hint = document.querySelector('#hint');
   let hintText = hint ? (hint.innerText || '') : null;
@@ -62,8 +108,27 @@ fn debug_player_state(tab: &headless_chrome::Tab) -> anyhow::Result<String> {
     hasCanvas: !!canvas,
     canvasWidth: canvas ? canvas.width : null,
     canvasHeight: canvas ? canvas.height : null,
+    hasArchiveVideo: !!video,
+    videoCurrentTime: video ? video.currentTime : null,
+    videoDuration: video ? video.duration : null,
+    videoPaused: video ? video.paused : null,
+    videoReadyState: video ? video.readyState : null,
+    videoMuted: video ? video.muted : null,
+    videoVolume: video ? video.volume : null,
+    videoSrc: video ? (video.currentSrc || video.src || '') : null,
     muted: watch ? watch.muted : null,
     volume: watch ? watch.volume : null,
+    connectionStatus: watch?.connection?.status?.peek ? watch.connection.status.peek() : null,
+    connectionKind: established ? established.constructor?.name || null : null,
+    broadcastStatus: watch?.broadcast?.status?.peek ? watch.broadcast.status.peek() : null,
+    paused: watch?.backend?.paused?.peek ? watch.backend.paused.peek() : null,
+    audioMuted: watch?.backend?.audio?.muted?.peek ? watch.backend.audio.muted.peek() : null,
+    audioVolume: watch?.backend?.audio?.volume?.peek ? watch.backend.audio.volume.peek() : null,
+    catalogSeen: !!catalog,
+    catalogHasVideo: !!(catalog?.video?.renditions),
+    catalogHasAudio: !!(catalog?.audio?.renditions),
+    metrics: window.__ecPlaybackMetrics || null,
+    statusLineText,
     hintText,
     placeholderText,
     statusText,
@@ -110,23 +175,120 @@ fn canvas_motion_sample(tab: &headless_chrome::Tab) -> anyhow::Result<Option<(f6
     Ok(Some((current_time, hash)))
 }
 
-fn wait_for_canvas_motion(tab: &headless_chrome::Tab, timeout: Duration) -> anyhow::Result<()> {
+fn archive_video_motion_sample(
+    tab: &headless_chrome::Tab,
+) -> anyhow::Result<Option<serde_json::Value>> {
+    let js = r#"(function() {
+  let video = document.querySelector('video.archiveVideo');
+  if (!video) return null;
+  if (video.paused) video.play().catch(() => {});
+  return JSON.stringify({
+    wallTime: performance.now() / 1000,
+    currentTime: video.currentTime || 0,
+    readyState: video.readyState || 0,
+    paused: !!video.paused,
+    ended: !!video.ended,
+    muted: !!video.muted,
+    volume: video.volume || 0,
+    src: video.currentSrc || video.src || ''
+  });
+})();"#;
+    let v = tab.evaluate(js, false)?;
+    let Some(s) = v.value.and_then(|v| v.as_str().map(|s| s.to_string())) else {
+        return Ok(None);
+    };
+    Ok(Some(serde_json::from_str(&s)?))
+}
+
+fn wait_for_canvas_or_archive_motion(
+    tab: &headless_chrome::Tab,
+    timeout: Duration,
+) -> anyhow::Result<String> {
     let deadline = Instant::now() + timeout;
-    let mut first: Option<(f64, u32)> = None;
+    let mut first_canvas: Option<(f64, u32)> = None;
+    let mut first_video_time: Option<f64> = None;
     while Instant::now() < deadline {
         if let Some(sample) = canvas_motion_sample(tab)? {
-            if let Some((first_time, first_hash)) = first {
+            if let Some((first_time, first_hash)) = first_canvas {
                 if sample.0 > first_time + 0.5 && sample.1 != first_hash {
-                    return Ok(());
+                    return Ok("moq-canvas".to_string());
                 }
             } else {
-                first = Some(sample);
+                first_canvas = Some(sample);
             }
         }
+
+        if let Some(sample) = archive_video_motion_sample(tab)? {
+            let current_time = sample
+                .get("currentTime")
+                .and_then(|v| v.as_f64())
+                .unwrap_or_default();
+            let ready_state = sample
+                .get("readyState")
+                .and_then(|v| v.as_u64())
+                .unwrap_or_default();
+            let ended = sample
+                .get("ended")
+                .and_then(|v| v.as_bool())
+                .unwrap_or(false);
+            if ready_state >= 2 && !ended {
+                if let Some(first) = first_video_time {
+                    if current_time > first + 0.5 {
+                        return Ok("archive-video".to_string());
+                    }
+                } else {
+                    first_video_time = Some(current_time);
+                }
+            }
+        }
+
         std::thread::sleep(Duration::from_millis(500));
     }
     let st = debug_player_state(tab).unwrap_or_default();
-    anyhow::bail!("timed out waiting for changing canvas frames\nplayer_state={st}");
+    anyhow::bail!("timed out waiting for live or archive motion\nplayer_state={st}");
+}
+
+fn wait_for_playback_probe_ok(
+    tab: &headless_chrome::Tab,
+    timeout: Duration,
+) -> anyhow::Result<String> {
+    let deadline = Instant::now() + timeout;
+    let mut last_metrics = String::new();
+    while Instant::now() < deadline {
+        let js = r#"(function() {
+  const metrics = window.__ecPlaybackMetrics || null;
+  return metrics ? JSON.stringify(metrics) : "";
+})();"#;
+        let v = tab.evaluate(js, false)?;
+        last_metrics = v
+            .value
+            .and_then(|v| v.as_str().map(|s| s.to_string()))
+            .unwrap_or_default();
+        if !last_metrics.is_empty() {
+            let metrics: serde_json::Value = serde_json::from_str(&last_metrics)?;
+            let ok = metrics.get("ok").and_then(|v| v.as_bool()).unwrap_or(false);
+            let samples = metrics
+                .get("samples")
+                .and_then(|v| v.as_u64())
+                .unwrap_or_default();
+            let changed = metrics
+                .get("changed_samples")
+                .and_then(|v| v.as_u64())
+                .unwrap_or_default();
+            let longest_static = metrics
+                .get("longest_same_hash_ms")
+                .and_then(|v| v.as_u64())
+                .unwrap_or_default();
+            if ok && samples >= 8 && changed >= 2 && longest_static < 5_000 {
+                return Ok(last_metrics);
+            }
+        }
+        std::thread::sleep(Duration::from_millis(250));
+    }
+    let st = debug_player_state(tab).unwrap_or_default();
+    anyhow::bail!(
+        "timed out waiting for playback probe ok\nplayer_state={st}\nmetrics={last_metrics}"
+    );
 }
 
 fn wait_for_unmuted_player(tab: &headless_chrome::Tab, timeout: Duration) -> anyhow::Result<()> {
@@ -134,7 +296,9 @@ fn wait_for_unmuted_player(tab: &headless_chrome::Tab, timeout: Duration) -> any
     while Instant::now() < deadline {
         let js = r#"(function() {
   let watch = document.querySelector('moq-watch');
-  return !!watch && watch.muted === false && watch.volume > 0 && !watch.hasAttribute('muted');
+  let video = document.querySelector('video.archiveVideo');
+  return (!!watch && watch.muted === false && watch.volume > 0 && !watch.hasAttribute('muted')) ||
+    (!!video && video.muted === false && video.volume > 0);
 })();"#;
         let v = tab.evaluate(js, false)?;
         if v.value.and_then(|v| v.as_bool()).unwrap_or(false) {
@@ -146,13 +310,21 @@ fn wait_for_unmuted_player(tab: &headless_chrome::Tab, timeout: Duration) -> any
     anyhow::bail!("timed out waiting for unmuted player\nplayer_state={st}");
 }
 
-fn watch_url(site_url: &str, relay_url: &str, stream_id: &str) -> anyhow::Result<String> {
+fn watch_url(
+    site_url: &str,
+    relay_url: &str,
+    stream_id: &str,
+    verify: bool,
+) -> anyhow::Result<String> {
     let mut url = url::Url::parse(site_url)?;
     url.set_path("/watch");
     url.query_pairs_mut()
         .clear()
         .append_pair("url", relay_url)
         .append_pair("name", stream_id);
+    if verify {
+        url.query_pairs_mut().append_pair("verify", "1");
+    }
     Ok(url.to_string())
 }
 
@@ -190,23 +362,104 @@ fn e2e_remote_website_watch_existing_stream_id() -> anyhow::Result<()> {
         .unwrap();
     let browser = headless_chrome::Browser::new(launch_options)?;
     let tab = browser.new_tab()?;
-    tab.navigate_to(&watch_url(&site_url, &relay_url, &stream_id)?)?;
+    tab.navigate_to(&watch_url(&site_url, &relay_url, &stream_id, false)?)?;
     tab.wait_until_navigated()?;
 
-    // Ensure the player is instantiated.
-    if let Err(err) = wait_for_moq_watch_element(&tab, Duration::from_secs(90)) {
+    // Ensure either the native MoQ player or the archive live-edge fallback is instantiated.
+    if let Err(err) = wait_for_live_or_archive_player(&tab, Duration::from_secs(90)) {
         let st = debug_player_state(&tab).unwrap_or_default();
         anyhow::bail!("{err}\nplayer_state={st}");
     }
 
-    if let Err(err) = wait_for_canvas_element(&tab, Duration::from_secs(90)) {
-        let st = debug_player_state(&tab).unwrap_or_default();
-        anyhow::bail!("{err}\nplayer_state={st}");
-    }
-
-    tab.wait_for_element("moq-watch canvas")?.click()?;
+    tab.evaluate(
+        r#"(function() {
+  const canvas = document.querySelector('moq-watch canvas');
+  if (canvas) canvas.click();
+  const audioButton = document.querySelector('#audioBtn');
+  if (audioButton && audioButton.getAttribute('aria-pressed') !== 'true') {
+    audioButton.click();
+  }
+})();"#,
+        false,
+    )?;
     wait_for_unmuted_player(&tab, Duration::from_secs(10))?;
-    wait_for_canvas_motion(&tab, Duration::from_secs(30))?;
+    let playback_path = wait_for_canvas_or_archive_motion(&tab, Duration::from_secs(60))?;
+    eprintln!("playback path: {playback_path}");
 
     Ok(())
 }
+
+#[test]
+#[ignore]
+fn e2e_remote_website_watch_synthetic_relay_stream() -> anyhow::Result<()> {
+    if which("ffmpeg").is_none() {
+        return Ok(()); // skip
+    }
+    let chrome = match chrome_path() {
+        Some(p) => p,
+        None => return Ok(()), // skip
+    };
+
+    let site_url = std::env::var("EVERY_CHANNEL_SITE_URL")
+        .unwrap_or_else(|_| "https://every.channel/".to_string());
+    let relay_url = std::env::var("EVERY_CHANNEL_RELAY_URL")
+        .unwrap_or_else(|_| "https://relay.every.channel/anon".to_string());
+    let tls_disable_verify = std::env::var("EVERY_CHANNEL_RELAY_TLS_DISABLE_VERIFY")
+        .map(|v| v != "0" && v.to_lowercase() != "false")
+        .unwrap_or(true);
+
+    let ts = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis();
+    let stream_id = format!("e2e-synthetic-{ts}");
+    let ec_node = ec_node_path();
+
+    let mut publisher = Command::new(&ec_node);
+    publisher
+        .arg("wt-publish")
+        .arg("--url")
+        .arg(&relay_url)
+        .arg("--name")
+        .arg(&stream_id)
+        .arg("--realtime-input")
+        .arg("--input-format")
+        .arg("lavfi")
+        .arg("--input")
+        .arg("testsrc2=size=1280x720:rate=30")
+        .stdout(Stdio::null())
+        .stderr(Stdio::inherit());
+    if tls_disable_verify {
+        publisher.arg("--tls-disable-verify");
+    }
+    let mut publisher = publisher.spawn()?;
+
+    let test_result = (|| -> anyhow::Result<()> {
+        let launch_options = headless_chrome::LaunchOptionsBuilder::default()
+            .path(Some(chrome))
+            .headless(true)
+            .args(vec![
+                OsStr::new("--autoplay-policy=no-user-gesture-required"),
+                OsStr::new("--disable-application-cache"),
+                OsStr::new("--disable-service-worker"),
+                OsStr::new("--disk-cache-size=0"),
+                OsStr::new("--mute-audio"),
+            ])
+            .build()
+            .unwrap();
+        let browser = headless_chrome::Browser::new(launch_options)?;
+        let tab = browser.new_tab()?;
+        tab.navigate_to(&watch_url(&site_url, &relay_url, &stream_id, true)?)?;
+        tab.wait_until_navigated()?;
+
+        wait_for_moq_watch_element(&tab, Duration::from_secs(90))?;
+        wait_for_canvas_element(&tab, Duration::from_secs(90))?;
+        let metrics = wait_for_playback_probe_ok(&tab, Duration::from_secs(60))?;
+        eprintln!("playback metrics: {metrics}");
+        Ok(())
+    })();
+
+    let _ = publisher.kill();
+    let _ = publisher.wait();
+    test_result
+}