every-channel/every.channel
1
0
Fork 0
Code Releases Activity Actions 2

every.channel: sanitized baseline

Browse source
This commit is contained in:
every.channel 2026-02-15 16:17:27 -05:00
commit 897e556bea
No known key found for this signature in database
258 changed files with 74298 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

88
third_party/iroh-live/moq-media/Cargo.toml vendored Normal file
View file

@ -0,0 +1,88 @@
[package]
name = "moq-media"
version = "0.1.0"
edition = "2024"
description = "native audio and video capturing, playback, encoding, decoding"
authors = ["Franz Heinzmann <frando@n0.computer>"]
repository = "https://github.com/n0-computer/iroh-live"
license = "MIT OR Apache-2.0"
[dependencies]
anyhow = "1.0.100"
bytemuck = "1.24.0"
byte-unit = { version = "5.1", features = ["bit"] }
data-encoding = "2.9.0"
derive_more = { version = "2.0.1", features = ["display", "debug", "eq"] }
ffmpeg-next = { version = "8.0.0", default-features = false, features = ["device", "format", "filter", "software-resampling", "software-scaling"] }
ffmpeg-sys-next = { version = "8.0.1", optional = true }
firewheel = { version = "0.9.1", features = ["cpal", "peak_meter_node", "std", "stream_nodes", "cpal_resample_inputs"] }
hang = "0.9.0"
image = { version = "0.25.8", default-features = false }
moq-lite = "0.10.1"
n0-error = { version = "0.1.2", features = ["anyhow"] }
n0-future = "0.3.1"
n0-watcher = "0.6.0"
nokhwa = { version = "0.10", features = [
"input-native",
"input-v4l",
"output-threaded",
] }
postcard = "1.1.3"
rand = "0.9.2"
serde = { version = "1.0.228", features = ["derive"] }
strum = { version = "0.27", features = ["derive"] }
tokio = { version = "1.48.0", features = ["sync"] }
tokio-util = "0.7.17"
tracing = "0.1.41"
xcap = "0.8"
webrtc-audio-processing = { version = "0.5.0", features = ["bundled"] }
bytes = "1.11.0"
buf-list = "1.1.2"
[dev-dependencies]
clap = { version = "4.5", features = ["derive"] }
eframe = "0.33.0"
postcard = "1.1.3"
tokio = { version = "1.48.0", features = ["full"] }
tracing-subscriber = "0.3.20"
[features]
default = []
# Enable static build of ffmpeg
static = [
"ffmpeg-next/static",
"ffmpeg-next/build-lib-openssl",
"ffmpeg-next/build-license-version3",
"ffmpeg-next/build-lib-opus",
"ffmpeg-next/build-lib-x264",
"ffmpeg-next/build-license-gpl",
"dep:ffmpeg-sys-next",
]
[target.'cfg(target_os = "macos")'.dependencies]
ffmpeg-sys-next = { version = "8.0.1", optional = true, features = [
"build-videotoolbox",
"build-audiotoolbox",
] }
[target.'cfg(target_os = "linux")'.dependencies]
ffmpeg-sys-next = { version = "8.0.1", optional = true, features = [
"build-vaapi",
# "build-vulkan",
# "build-lib-libmfx",
] }
[target.'cfg(target_os = "windows")'.dependencies]
ffmpeg-sys-next = { version = "8.0.1", optional = true, features = [
"build-lib-d3d11va",
"build-lib-dxva2",
# "build-nvidia",
# "build-amf",
] }
[target.'cfg(target_os = "android")'.dependencies]
ffmpeg-sys-next = { version = "8.0.1", optional = true, features = [
# "build-mediacodec",
] }

527
third_party/iroh-live/moq-media/src/audio.rs vendored Normal file
View file

@ -0,0 +1,527 @@
use std::{
collections::HashMap,
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
},
time::{Duration, Instant},
};
use anyhow::{Context, Result};
use firewheel::{
CpalConfig, CpalInputConfig, CpalOutputConfig, FirewheelConfig, FirewheelContext,
channel_config::{ChannelConfig, ChannelCount, NonZeroChannelCount},
dsp::volume::{DEFAULT_DB_EPSILON, DbMeterNormalizer},
graph::PortIdx,
node::NodeID,
nodes::{
peak_meter::{PeakMeterNode, PeakMeterSmoother, PeakMeterState},
stream::{
ResamplingChannelConfig,
reader::{StreamReaderConfig, StreamReaderNode, StreamReaderState},
writer::{StreamWriterConfig, StreamWriterNode, StreamWriterState},
},
},
};
use tokio::sync::{mpsc, mpsc::error::TryRecvError, oneshot};
use tracing::{debug, error, info, trace, warn};
use self::aec::{AecCaptureNode, AecProcessor, AecProcessorConfig, AecRenderNode};
use crate::{
av::{AudioFormat, AudioSink, AudioSinkHandle, AudioSource},
util::spawn_thread,
};
mod aec;
type StreamWriterHandle = Arc<Mutex<StreamWriterState>>;
type StreamReaderHandle = Arc<Mutex<StreamReaderState>>;
#[derive(Debug, Clone)]
pub struct AudioBackend {
tx: mpsc::Sender<DriverMessage>,
}
impl AudioBackend {
pub fn new() -> Self {
let (tx, rx) = mpsc::channel(32);
let _handle = spawn_thread("audiodriver", move || AudioDriver::new(rx).run());
Self { tx }
}
pub async fn default_input(&self) -> Result<InputStream> {
self.input(AudioFormat::mono_48k()).await
}
pub async fn input(&self, format: AudioFormat) -> Result<InputStream> {
let (reply, reply_rx) = oneshot::channel();
self.tx
.send(DriverMessage::InputStream { format, reply })
.await?;
let handle = reply_rx.await??;
Ok(InputStream { handle, format })
}
pub async fn default_output(&self) -> Result<OutputStream> {
self.output(AudioFormat::stereo_48k()).await
}
pub async fn output(&self, format: AudioFormat) -> Result<OutputStream> {
let (reply, reply_rx) = oneshot::channel();
self.tx
.send(DriverMessage::OutputStream { format, reply })
.await?;
let handle = reply_rx.await??;
Ok(handle)
}
}
#[derive(Clone)]
pub struct OutputStream {
handle: StreamWriterHandle,
paused: Arc<AtomicBool>,
peaks: Arc<Mutex<PeakMeterSmoother<2>>>,
normalizer: DbMeterNormalizer,
}
impl AudioSinkHandle for OutputStream {
fn is_paused(&self) -> bool {
self.paused.load(Ordering::Relaxed)
}
fn pause(&self) {
self.paused.store(true, Ordering::Relaxed);
self.handle.lock().expect("poisoned").pause_stream();
}
fn resume(&self) {
self.paused.store(false, Ordering::Relaxed);
self.handle.lock().expect("poisoned").resume();
}
fn toggle_pause(&self) {
let was_paused = self.paused.fetch_xor(true, Ordering::Relaxed);
if was_paused {
self.handle.lock().expect("poisoned").resume();
} else {
self.handle.lock().expect("poisoned").pause_stream();
}
}
fn smoothed_peak_normalized(&self) -> Option<f32> {
Some(
self.peaks
.lock()
.expect("poisoned")
.smoothed_peaks_normalized_mono(&self.normalizer),
)
}
}
impl AudioSink for OutputStream {
fn handle(&self) -> Box<dyn AudioSinkHandle> {
Box::new(self.clone())
}
fn format(&self) -> Result<AudioFormat> {
let info = self.handle.lock().expect("poisoned");
let sample_rate = info
.sample_rate()
.context("output stream misses sample rate")?
.get();
let channel_count = info.num_channels().get().get();
Ok(AudioFormat {
sample_rate,
channel_count,
})
}
fn push_samples(&mut self, samples: &[f32]) -> Result<()> {
let mut handle = self.handle.lock().unwrap();
// If this happens excessively in Release mode, you may want to consider
// increasing [`StreamWriterConfig::channel_config.latency_seconds`].
if handle.underflow_occurred() {
warn!("Underflow occured in stream writer node!");
}
// If this happens excessively in Release mode, you may want to consider
// increasing [`StreamWriterConfig::channel_config.capacity_seconds`]. For
// example, if you are streaming data from a network, you may want to
// increase the capacity to several seconds.
if handle.overflow_occurred() {
warn!("Overflow occured in stream writer node!");
}
// Wait until the node's processor is ready to receive data.
if handle.is_ready() {
// let expected_bytes =
// frame.samples() * frame.channels() as usize * core::mem::size_of::<f32>();
// let cpal_sample_data: &[f32] = bytemuck::cast_slice(&frame.data(0)[..expected_bytes]);
handle.push_interleaved(samples);
trace!("pushed samples {}", samples.len());
} else {
warn!("output handle is inactive")
}
Ok(())
}
}
impl OutputStream {
#[allow(unused)]
pub fn is_active(&self) -> bool {
self.handle.lock().expect("poisoned").is_active()
}
}
/// A simple AudioSource that reads from the default microphone via Firewheel.
#[derive(Clone)]
pub struct InputStream {
handle: StreamReaderHandle,
format: AudioFormat,
}
impl AudioSource for InputStream {
fn cloned_boxed(&self) -> Box<dyn AudioSource> {
Box::new(self.clone())
}
fn format(&self) -> AudioFormat {
self.format
}
fn pop_samples(&mut self, buf: &mut [f32]) -> Result<Option<usize>> {
use firewheel::nodes::stream::ReadStatus;
let mut handle = self.handle.lock().expect("poisoned");
match handle.read_interleaved(buf) {
Some(ReadStatus::Ok) => Ok(Some(buf.len())),
Some(ReadStatus::InputNotReady) => {
tracing::warn!("audio input not ready");
// Maintain pacing; still return a frame-sized buffer
Ok(Some(buf.len()))
}
Some(ReadStatus::UnderflowOccurred { num_frames_read }) => {
tracing::warn!(
"audio input underflow: {} frames missing",
buf.len() - num_frames_read
);
Ok(Some(buf.len()))
}
Some(ReadStatus::OverflowCorrected {
num_frames_discarded,
}) => {
tracing::warn!("audio input overflow: {num_frames_discarded} frames discarded");
Ok(Some(buf.len()))
}
None => {
tracing::warn!("audio input stream is inactive");
Ok(None)
}
}
}
}
#[derive(derive_more::Debug)]
enum DriverMessage {
OutputStream {
format: AudioFormat,
#[debug("Sender")]
reply: oneshot::Sender<Result<OutputStream>>,
},
InputStream {
format: AudioFormat,
#[debug("Sender")]
reply: oneshot::Sender<Result<StreamReaderHandle>>,
},
}
struct AudioDriver {
cx: FirewheelContext,
rx: mpsc::Receiver<DriverMessage>,
aec_processor: AecProcessor,
aec_render_node: NodeID,
aec_capture_node: NodeID,
peak_meters: HashMap<NodeID, Arc<Mutex<PeakMeterSmoother<2>>>>,
}
impl AudioDriver {
fn new(rx: mpsc::Receiver<DriverMessage>) -> Self {
let config = FirewheelConfig {
num_graph_inputs: ChannelCount::new(1).unwrap(),
..Default::default()
};
let mut cx = FirewheelContext::new(config);
info!("inputs: {:?}", cx.available_input_devices());
info!("outputs: {:?}", cx.available_output_devices());
let config = CpalConfig {
output: CpalOutputConfig {
#[cfg(target_os = "linux")]
device_name: Some("pipewire".to_string()),
..Default::default()
},
input: Some(CpalInputConfig {
#[cfg(target_os = "linux")]
device_name: Some("pipewire".to_string()),
fail_on_no_input: true,
..Default::default()
}),
};
cx.start_stream(config).unwrap();
info!(
"audio graph in: {:?}",
cx.node_info(cx.graph_in_node_id()).map(|x| &x.info)
);
info!(
"audio graph out: {:?}",
cx.node_info(cx.graph_out_node_id()).map(|x| &x.info)
);
cx.set_graph_channel_config(ChannelConfig {
num_inputs: ChannelCount::new(2).unwrap(),
num_outputs: ChannelCount::new(2).unwrap(),
});
let aec_processor = AecProcessor::new(AecProcessorConfig::stereo_in_out(), true)
.expect("failed to initialize AEC processor");
let aec_render_node = cx.add_node(AecRenderNode::default(), Some(aec_processor.clone()));
let aec_capture_node = cx.add_node(AecCaptureNode::default(), Some(aec_processor.clone()));
let layout = &[(0, 0), (1, 1)];
cx.connect(cx.graph_in_node_id(), aec_capture_node, layout, true)
.unwrap();
cx.connect(aec_render_node, cx.graph_out_node_id(), layout, true)
.unwrap();
Self {
cx,
rx,
aec_processor,
aec_render_node,
aec_capture_node,
peak_meters: Default::default(),
}
}
fn run(&mut self) {
const INTERVAL: Duration = Duration::from_millis(10);
const PEAK_UPDATE_INTERVAL: Duration = Duration::from_millis(40);
let mut last_delay: f64 = 0.;
let mut last_peak_update = Instant::now();
loop {
let tick = Instant::now();
if self.drain_messages().is_err() {
info!("closing audio driver: message channel closed");
break;
}
if let Err(e) = self.cx.update() {
error!("audio backend error: {:?}", &e);
// if let UpdateError::StreamStoppedUnexpectedly(_) = e {
// // Notify the stream node handles that the output stream has stopped.
// // This will automatically stop any active streams on the nodes.
// cx.node_state_mut::<StreamWriterState>(stream_writer_id)
// .unwrap()
// .stop_stream();
// cx.node_state_mut::<StreamReaderState>(stream_reader_id)
// .unwrap()
// .stop_stream();
// // The stream has stopped unexpectedly (i.e the user has
// // unplugged their headphones.)
// //
// // Typically you should start a new stream as soon as
// // possible to resume processing (event if it's a dummy
// // output device).
// //
// // In this example we just quit the application.
// break;
// }
}
if let Some(info) = self.cx.stream_info() {
let delay = info.input_to_output_latency_seconds;
if (last_delay - delay).abs() > (1. / 1000.) {
let delay_ms = (delay * 1000.) as u32;
info!("update processor delay to {delay_ms}ms");
self.aec_processor.set_stream_delay(delay_ms);
last_delay = delay;
}
}
// Update peak meters
let delta = last_peak_update.elapsed();
if delta > PEAK_UPDATE_INTERVAL {
for (id, smoother) in self.peak_meters.iter_mut() {
smoother.lock().expect("poisoned").update(
self.cx
.node_state::<PeakMeterState<2>>(*id)
.unwrap()
.peak_gain_db(DEFAULT_DB_EPSILON),
delta.as_secs_f32(),
);
}
last_peak_update = Instant::now();
}
std::thread::sleep(INTERVAL.saturating_sub(tick.elapsed()));
}
}
fn drain_messages(&mut self) -> Result<(), ()> {
loop {
match self.rx.try_recv() {
Err(TryRecvError::Disconnected) => {
info!("stopping audio thread: backend handle dropped");
break Err(());
}
Err(TryRecvError::Empty) => {
break Ok(());
}
Ok(message) => self.handle_message(message),
}
}
}
fn handle_message(&mut self, message: DriverMessage) {
debug!("handle {message:?}");
match message {
DriverMessage::OutputStream { format, reply } => {
let res = self
.output_stream(format)
.inspect_err(|err| warn!("failed to create audio output stream: {err:#}"));
reply.send(res).ok();
}
DriverMessage::InputStream { format, reply } => {
let res = self
.input_stream(format)
.inspect_err(|err| warn!("failed to create audio input stream: {err:#}"));
reply.send(res).ok();
}
}
}
fn output_stream(&mut self, format: AudioFormat) -> Result<OutputStream> {
let channel_count = format.channel_count;
let sample_rate = format.sample_rate;
// setup stream
let stream_writer_id = self.cx.add_node(
StreamWriterNode,
Some(StreamWriterConfig {
channels: NonZeroChannelCount::new(channel_count)
.context("channel count may not be zero")?,
..Default::default()
}),
);
let graph_out = self.aec_render_node;
// let graph_out_info = self
// .cx
// .node_info(graph_out)
// .context("missing audio output node")?;
let peak_meter_node = PeakMeterNode::<2> { enabled: true };
let peak_meter_id = self.cx.add_node(peak_meter_node.clone(), None);
let peak_meter_smoother =
Arc::new(Mutex::new(PeakMeterSmoother::<2>::new(Default::default())));
self.peak_meters
.insert(peak_meter_id, peak_meter_smoother.clone());
self.cx
.connect(peak_meter_id, graph_out, &[(0, 0), (1, 1)], true)
.unwrap();
let layout: &[(PortIdx, PortIdx)] = match channel_count {
0 => anyhow::bail!("audio stream has no channels"),
1 => &[(0, 0), (0, 1)],
_ => &[(0, 0), (1, 1)],
};
self.cx
.connect(stream_writer_id, peak_meter_id, layout, false)
.unwrap();
let output_stream_sample_rate = self.cx.stream_info().unwrap().sample_rate;
let event = self
.cx
.node_state_mut::<StreamWriterState>(stream_writer_id)
.unwrap()
.start_stream(
sample_rate.try_into().unwrap(),
output_stream_sample_rate,
ResamplingChannelConfig {
capacity_seconds: 3.,
..Default::default()
},
)
.unwrap();
info!("started output stream");
self.cx.queue_event_for(stream_writer_id, event.into());
// Wrap the handles in an `Arc<Mutex<T>>>` so that we can send them to other threads.
let handle = self
.cx
.node_state::<StreamWriterState>(stream_writer_id)
.unwrap()
.handle();
Ok(OutputStream {
handle: Arc::new(handle),
paused: Arc::new(AtomicBool::new(false)),
peaks: peak_meter_smoother,
normalizer: DbMeterNormalizer::new(-60., 0., -20.),
})
}
fn input_stream(&mut self, format: AudioFormat) -> Result<StreamReaderHandle> {
let sample_rate = format.sample_rate;
let channel_count = format.channel_count;
// Setup stream reader node
let stream_reader_id = self.cx.add_node(
StreamReaderNode,
Some(StreamReaderConfig {
channels: NonZeroChannelCount::new(channel_count)
.context("channel count may not be zero")?,
..Default::default()
}),
);
let graph_in_node_id = self.aec_capture_node;
let graph_in_info = self
.cx
.node_info(graph_in_node_id)
.context("missing audio input node")?;
let layout: &[(PortIdx, PortIdx)] = match (
graph_in_info.info.channel_config.num_outputs.get(),
channel_count,
) {
(0, _) => anyhow::bail!("audio input has no channels"),
(1, 2) => &[(0, 0), (0, 1)],
(2, 2) => &[(0, 0), (1, 1)],
(_, 1) => &[(0, 0)],
_ => &[(0, 0), (1, 1)],
};
self.cx
.connect(graph_in_node_id, stream_reader_id, layout, false)
.unwrap();
let input_stream_sample_rate = self.cx.stream_info().unwrap().sample_rate;
let event = self
.cx
.node_state_mut::<StreamReaderState>(stream_reader_id)
.unwrap()
.start_stream(
sample_rate.try_into().unwrap(),
input_stream_sample_rate,
ResamplingChannelConfig {
capacity_seconds: 3.0,
..Default::default()
},
)
.unwrap();
self.cx.queue_event_for(stream_reader_id, event.into());
let handle = self
.cx
.node_state::<StreamReaderState>(stream_reader_id)
.unwrap()
.handle();
Ok(Arc::new(handle))
}
}

452
third_party/iroh-live/moq-media/src/audio/aec.rs vendored Normal file
View file

@ -0,0 +1,452 @@
pub use self::{
firewheel_nodes::{AecCaptureNode, AecRenderNode},
processor::{AecProcessor, AecProcessorConfig},
};
mod processor {
use std::{
num::NonZeroU32,
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
},
};
use anyhow::Result;
use tracing::{debug, info};
use webrtc_audio_processing::{
Config, EchoCancellation, EchoCancellationSuppressionLevel, InitializationConfig,
};
#[derive(Debug, Clone)]
pub struct AecProcessorConfig {
pub num_input_channels: NonZeroU32,
pub num_output_channels: NonZeroU32,
}
impl Default for AecProcessorConfig {
fn default() -> Self {
Self {
num_input_channels: 2.try_into().unwrap(),
num_output_channels: 2.try_into().unwrap(),
}
}
}
impl AecProcessorConfig {
pub fn stereo_in_out() -> Self {
Self::default()
}
}
#[derive(Clone, Debug)]
pub struct AecProcessor(Arc<Inner>);
#[derive(derive_more::Debug)]
struct Inner {
#[debug("Processor")]
processor: Mutex<webrtc_audio_processing::Processor>,
config: Mutex<Config>,
// capture_delay: AtomicU64,
// playback_delay: AtomicU64,
enabled: AtomicBool,
// capture_channels: AtomicUsize,
// playback_channels: AtomicUsize,
}
impl Default for AecProcessor {
fn default() -> Self {
Self::new(Default::default(), true).expect("failed to initialize AecProcessor")
}
}
impl AecProcessor {
pub fn new(config: AecProcessorConfig, enabled: bool) -> anyhow::Result<Self> {
let suppression_level = EchoCancellationSuppressionLevel::High;
// High pass filter is a prerequisite to running echo cancellation.
let processor_config = Config {
echo_cancellation: Some(EchoCancellation {
suppression_level,
stream_delay_ms: None,
enable_delay_agnostic: true,
enable_extended_filter: true,
}),
enable_high_pass_filter: true,
..Config::default()
};
let mut processor = webrtc_audio_processing::Processor::new(&InitializationConfig {
num_capture_channels: config.num_input_channels.get() as i32,
num_render_channels: config.num_output_channels.get() as i32,
enable_experimental_agc: true,
enable_intelligibility_enhancer: true, // ..InitializationConfig::default()
})?;
processor.set_config(processor_config.clone());
// processor.set_config(config.clone());
info!("init audio processor (config={config:?})");
Ok(Self(Arc::new(Inner {
processor: Mutex::new(processor),
config: Mutex::new(processor_config),
enabled: AtomicBool::new(enabled),
})))
}
pub fn is_enabled(&self) -> bool {
self.0.enabled.load(Ordering::SeqCst)
}
#[allow(unused)]
pub fn set_enabled(&self, enabled: bool) {
let _prev = self.0.enabled.swap(enabled, Ordering::SeqCst);
}
/// Processes and modifies the audio frame from a capture device by applying
/// signal processing as specified in the config. `frame` should hold an
/// interleaved f32 audio frame, with [`NUM_SAMPLES_PER_FRAME`] samples.
// webrtc-audio-processing expects a 10ms chunk for each process call.
pub fn process_capture_frame(
&self,
frame: &mut [f32],
) -> Result<(), webrtc_audio_processing::Error> {
if !self.is_enabled() {
return Ok(());
}
self.0
.processor
.lock()
.expect("poisoned")
.process_capture_frame(frame)
}
/// Processes and optionally modifies the audio frame from a playback device.
/// `frame` should hold an interleaved `f32` audio frame, with
/// [`NUM_SAMPLES_PER_FRAME`] samples.
pub fn process_render_frame(
&self,
frame: &mut [f32],
) -> Result<(), webrtc_audio_processing::Error> {
if !self.is_enabled() {
return Ok(());
}
self.0
.processor
.lock()
.expect("poisoned")
.process_render_frame(frame)
}
pub fn set_stream_delay(&self, delay_ms: u32) {
debug!("updating stream delay to {delay_ms}ms");
// let playback = self.0.playback_delay.load(Ordering::Relaxed);
// let capture = self.0.capture_delay.load(Ordering::Relaxed);
// let total = playback + capture;
let mut config = self.0.config.lock().expect("poisoned");
config.echo_cancellation.as_mut().unwrap().stream_delay_ms = Some(delay_ms as i32);
self.0
.processor
.lock()
.expect("poisoned")
.set_config(config.clone());
}
}
}
mod firewheel_nodes {
use std::collections::VecDeque;
use firewheel::{
StreamInfo,
channel_config::{ChannelConfig, ChannelCount},
diff::{Diff, Patch},
event::ProcEvents,
node::{
AudioNode, AudioNodeInfo, AudioNodeProcessor, ConstructProcessorContext, ProcBuffers,
ProcExtra, ProcInfo, ProcStreamCtx, ProcessStatus,
},
};
use webrtc_audio_processing::NUM_SAMPLES_PER_FRAME;
use super::AecProcessor;
const CHANNELS: usize = 2;
const FRAME_SAMPLES: usize = (NUM_SAMPLES_PER_FRAME as usize) * CHANNELS;
/// Simple render-side node: feeds output audio into WebRTC's render stream.
#[derive(Diff, Patch, Debug, Clone, Copy, PartialEq)]
pub struct AecRenderNode {
pub enabled: bool,
}
impl Default for AecRenderNode {
fn default() -> Self {
Self { enabled: true }
}
}
impl AudioNode for AecRenderNode {
/// We use the wrapped WebRTC processor as our configuration object.
///
/// Note: `WebrtcAudioProcessor` already internally wraps an `Arc<Inner>`,
/// so cloning this config shares the underlying processor between nodes.
type Configuration = AecProcessor;
fn info(&self, _config: &Self::Configuration) -> AudioNodeInfo {
AudioNodeInfo::new()
.debug_name("webrtc_render")
.channel_config(ChannelConfig {
num_inputs: ChannelCount::STEREO,
num_outputs: ChannelCount::STEREO,
})
}
fn construct_processor(
&self,
config: &Self::Configuration,
_cx: ConstructProcessorContext,
) -> impl AudioNodeProcessor {
// Clone = share the same underlying Arc<Inner>.
let webrtc = config.clone();
// Inform the processor how many playback channels we have.
// (You can handle errors here instead of unwrap() in real code.)
// webrtc.init_playback(CHANNELS).ok();
RenderProcessor {
enabled: self.enabled,
processor: webrtc,
in_ring: VecDeque::with_capacity(FRAME_SAMPLES * 4),
out_ring: VecDeque::with_capacity(FRAME_SAMPLES * 4),
tmp_chunk: vec![0.0; FRAME_SAMPLES],
}
}
}
struct RenderProcessor {
enabled: bool,
processor: AecProcessor,
// Interleaved input samples to be fed into WebRTC in 10ms chunks.
in_ring: VecDeque<f32>,
// Interleaved processed samples coming back from WebRTC.
out_ring: VecDeque<f32>,
// Scratch buffer for one NUM_SAMPLES_PER_FRAME chunk (interleaved).
tmp_chunk: Vec<f32>,
}
impl AudioNodeProcessor for RenderProcessor {
fn process(
&mut self,
info: &ProcInfo,
buffers: ProcBuffers,
events: &mut ProcEvents,
_extra: &mut ProcExtra,
) -> ProcessStatus {
// Handle parameter patches.
for patch in events.drain_patches::<AecRenderNode>() {
match patch {
AecRenderNodePatch::Enabled(enabled) => {
self.enabled = enabled;
if !self.enabled {
// Clear any buffered state when disabling to avoid stale audio.
self.in_ring.clear();
self.out_ring.clear();
}
}
}
}
let num_frames = info.frames as usize;
// println!("num_frames: {num_frames}");
// Get input/output slices like in the FilterNode example.
let in_l = &buffers.inputs[0][..num_frames];
let in_r = &buffers.inputs[1][..num_frames];
let (out_l, out_rest) = buffers.outputs.split_first_mut().unwrap();
let out_l = &mut out_l[..num_frames];
let out_r = &mut out_rest[0][..num_frames];
// If disabled, just pass through.
if !self.enabled {
out_l.copy_from_slice(in_l);
out_r.copy_from_slice(in_r);
return ProcessStatus::OutputsModified;
}
// 1. Push current block into the interleaved input ring buffer.
for i in 0..num_frames {
self.in_ring.push_back(in_l[i]);
self.in_ring.push_back(in_r[i]);
}
// 2. While we have at least one full 10ms frame, process it.
while self.in_ring.len() >= FRAME_SAMPLES {
// Fill tmp_chunk with a full frame of interleaved samples.
for s in &mut self.tmp_chunk[..FRAME_SAMPLES] {
*s = self.in_ring.pop_front().unwrap();
}
// Feed into processor render stream.
let _ = self.processor.process_render_frame(&mut self.tmp_chunk);
// Store processed samples into the output ring.
for &s in &self.tmp_chunk[..FRAME_SAMPLES] {
self.out_ring.push_back(s);
}
}
// 3. Produce outputs for this audio block.
//
// We always need `num_frames * CHANNELS` samples. If we don't have
// enough processed samples yet, we output silence for the missing part.
for i in 0..num_frames {
if self.out_ring.len() >= CHANNELS {
out_l[i] = self.out_ring.pop_front().unwrap();
out_r[i] = self.out_ring.pop_front().unwrap();
} else {
// Not enough processed data yet -> output silence.
out_l[i] = 0.0;
out_r[i] = 0.0;
}
}
ProcessStatus::OutputsModified
}
fn new_stream(&mut self, _stream_info: &StreamInfo, _ctx: &mut ProcStreamCtx) {
// Reset buffers for new stream.
self.in_ring.clear();
self.out_ring.clear();
}
}
/// Capture-side node: feeds mic audio into [`AecProcessor`]'s capture stream.
#[derive(Diff, Patch, Debug, Clone, Copy, PartialEq)]
pub struct AecCaptureNode {
pub enabled: bool,
}
impl Default for AecCaptureNode {
fn default() -> Self {
Self { enabled: true }
}
}
impl AudioNode for AecCaptureNode {
type Configuration = AecProcessor;
fn info(&self, _config: &Self::Configuration) -> AudioNodeInfo {
AudioNodeInfo::new()
.debug_name("webrtc_capture")
.channel_config(ChannelConfig {
num_inputs: ChannelCount::STEREO,
num_outputs: ChannelCount::STEREO,
})
}
fn construct_processor(
&self,
config: &Self::Configuration,
_cx: ConstructProcessorContext,
) -> impl AudioNodeProcessor {
CaptureProcessor {
enabled: self.enabled,
processor: config.clone(),
in_ring: VecDeque::with_capacity(FRAME_SAMPLES * 4),
out_ring: VecDeque::with_capacity(FRAME_SAMPLES * 4),
tmp_chunk: vec![0.0; FRAME_SAMPLES],
}
}
}
struct CaptureProcessor {
enabled: bool,
processor: AecProcessor,
// Interleaved input samples to be fed into WebRTC in 10ms chunks.
in_ring: VecDeque<f32>,
// Interleaved processed samples coming back from WebRTC.
out_ring: VecDeque<f32>,
// Scratch buffer for one NUM_SAMPLES_PER_FRAME chunk (interleaved).
tmp_chunk: Vec<f32>,
}
impl AudioNodeProcessor for CaptureProcessor {
fn process(
&mut self,
info: &ProcInfo,
buffers: ProcBuffers,
events: &mut ProcEvents,
_extra: &mut ProcExtra,
) -> ProcessStatus {
for patch in events.drain_patches::<AecCaptureNode>() {
match patch {
AecCaptureNodePatch::Enabled(enabled) => {
self.enabled = enabled;
if !self.enabled {
self.in_ring.clear();
self.out_ring.clear();
}
}
}
}
let frames = info.frames;
let num_frames = frames as usize;
let in_l = &buffers.inputs[0][..num_frames];
let in_r = &buffers.inputs[1][..num_frames];
let (out_l, out_rest) = buffers.outputs.split_first_mut().unwrap();
let out_l = &mut out_l[..num_frames];
let out_r = &mut out_rest[0][..num_frames];
if !self.enabled {
// Bypass if disabled.
out_l.copy_from_slice(in_l);
out_r.copy_from_slice(in_r);
return ProcessStatus::OutputsModified;
}
// 1. Push current block into the interleaved input ring buffer.
for i in 0..num_frames {
self.in_ring.push_back(in_l[i]);
self.in_ring.push_back(in_r[i]);
}
// 2. While we have at least one full 10ms frame, process it.
while self.in_ring.len() >= FRAME_SAMPLES {
for s in &mut self.tmp_chunk[..FRAME_SAMPLES] {
*s = self.in_ring.pop_front().unwrap();
}
let _ = self.processor.process_capture_frame(&mut self.tmp_chunk);
for &s in &self.tmp_chunk[..FRAME_SAMPLES] {
self.out_ring.push_back(s);
}
}
// 3. Produce outputs for this audio block.
//
// If we don't have enough processed samples to cover the whole block,
// we output silence for the missing frames.
for i in 0..num_frames {
if self.out_ring.len() >= CHANNELS {
out_l[i] = self.out_ring.pop_front().unwrap();
out_r[i] = self.out_ring.pop_front().unwrap();
} else {
out_l[i] = 0.0;
out_r[i] = 0.0;
}
}
ProcessStatus::OutputsModified
}
fn new_stream(&mut self, _stream_info: &StreamInfo, _ctx: &mut ProcStreamCtx) {
// Reset state for new stream.
self.in_ring.clear();
self.out_ring.clear();
}
}
}

265
third_party/iroh-live/moq-media/src/av.rs vendored Normal file
View file

@ -0,0 +1,265 @@
use std::time::Duration;
use anyhow::Result;
use image::RgbaImage;
use strum::{Display, EnumString, VariantNames};
#[derive(Copy, Clone, Debug)]
pub struct AudioFormat {
pub sample_rate: u32,
pub channel_count: u32,
}
impl AudioFormat {
pub fn mono_48k() -> Self {
Self {
sample_rate: 48_000,
channel_count: 1,
}
}
pub fn stereo_48k() -> Self {
Self {
sample_rate: 48_000,
channel_count: 2,
}
}
pub fn from_hang_config(config: &hang::catalog::AudioConfig) -> Self {
Self {
channel_count: config.channel_count,
sample_rate: config.sample_rate,
}
}
}
pub trait Decoders {
type Audio: AudioDecoder;
type Video: VideoDecoder;
}
pub trait AudioSource: Send + 'static {
fn cloned_boxed(&self) -> Box<dyn AudioSource>;
fn format(&self) -> AudioFormat;
fn pop_samples(&mut self, buf: &mut [f32]) -> Result<Option<usize>>;
}
pub trait AudioSink: AudioSinkHandle {
fn format(&self) -> Result<AudioFormat>;
fn push_samples(&mut self, buf: &[f32]) -> Result<()>;
fn handle(&self) -> Box<dyn AudioSinkHandle>;
}
pub trait AudioSinkHandle: Send + 'static {
fn pause(&self);
fn resume(&self);
fn is_paused(&self) -> bool;
fn toggle_pause(&self);
/// Smoothed peak, normalized to 0..1.
// TODO: document how smoothing and normalization are expected
fn smoothed_peak_normalized(&self) -> Option<f32> {
None
}
}
pub trait AudioEncoder: AudioEncoderInner {
fn with_preset(format: AudioFormat, preset: AudioPreset) -> Result<Self>
where
Self: Sized;
}
pub trait AudioEncoderInner: Send + 'static {
fn name(&self) -> &str;
fn config(&self) -> hang::catalog::AudioConfig;
fn push_samples(&mut self, samples: &[f32]) -> Result<()>;
fn pop_packet(&mut self) -> Result<Option<hang::Frame>>;
}
impl AudioEncoderInner for Box<dyn AudioEncoder> {
fn name(&self) -> &str {
(&**self).name()
}
fn config(&self) -> hang::catalog::AudioConfig {
(&**self).config()
}
fn push_samples(&mut self, samples: &[f32]) -> Result<()> {
(&mut **self).push_samples(samples)
}
fn pop_packet(&mut self) -> Result<Option<hang::Frame>> {
(&mut **self).pop_packet()
}
}
pub trait AudioDecoder: Send + 'static {
fn new(config: &hang::catalog::AudioConfig, target_format: AudioFormat) -> Result<Self>
where
Self: Sized;
fn push_packet(&mut self, packet: hang::Frame) -> Result<()>;
fn pop_samples(&mut self) -> Result<Option<&[f32]>>;
}
#[derive(Copy, Clone, Debug, PartialEq)]
pub enum PixelFormat {
Rgba,
Bgra,
}
impl Default for PixelFormat {
fn default() -> Self {
PixelFormat::Rgba
}
}
#[derive(Clone, Debug)]
pub struct VideoFormat {
pub pixel_format: PixelFormat,
pub dimensions: [u32; 2],
}
#[derive(Clone, Debug)]
pub struct VideoFrame {
pub format: VideoFormat,
pub raw: bytes::Bytes,
}
pub trait VideoSource: Send + 'static {
fn name(&self) -> &str;
fn format(&self) -> VideoFormat;
fn pop_frame(&mut self) -> Result<Option<VideoFrame>>;
fn start(&mut self) -> Result<()>;
fn stop(&mut self) -> Result<()>;
}
pub trait VideoEncoder: VideoEncoderInner {
fn with_preset(preset: VideoPreset) -> Result<Self>
where
Self: Sized;
}
pub trait VideoEncoderInner: Send + 'static {
fn name(&self) -> &str;
fn config(&self) -> hang::catalog::VideoConfig;
fn push_frame(&mut self, frame: VideoFrame) -> Result<()>;
fn pop_packet(&mut self) -> Result<Option<hang::Frame>>;
}
impl VideoEncoderInner for Box<dyn VideoEncoder> {
fn name(&self) -> &str {
(&**self).name()
}
fn config(&self) -> hang::catalog::VideoConfig {
(&**self).config()
}
fn push_frame(&mut self, frame: VideoFrame) -> Result<()> {
(&mut **self).push_frame(frame)
}
fn pop_packet(&mut self) -> Result<Option<hang::Frame>> {
(&mut **self).pop_packet()
}
}
pub trait VideoDecoder: Send + 'static {
fn new(config: &hang::catalog::VideoConfig, playback_config: &DecodeConfig) -> Result<Self>
where
Self: Sized;
fn name(&self) -> &str;
fn pop_frame(&mut self) -> Result<Option<DecodedFrame>>;
fn push_packet(&mut self, packet: hang::Frame) -> Result<()>;
fn set_viewport(&mut self, w: u32, h: u32);
}
pub struct DecodedFrame {
pub frame: image::Frame,
pub timestamp: Duration,
}
impl DecodedFrame {
pub fn img(&self) -> &RgbaImage {
self.frame.buffer()
}
}
#[derive(Debug, Clone, Copy, Display, EnumString, VariantNames)]
#[strum(serialize_all = "lowercase")]
pub enum AudioCodec {
Opus,
}
#[derive(Debug, Clone, Copy, Display, EnumString, VariantNames)]
#[strum(serialize_all = "lowercase")]
pub enum VideoCodec {
H264,
Av1,
}
#[derive(Debug, Clone, Copy, Display, EnumString, VariantNames, Eq, PartialEq, Ord, PartialOrd)]
pub enum VideoPreset {
#[strum(serialize = "180p")]
P180,
#[strum(serialize = "360p")]
P360,
#[strum(serialize = "720p")]
P720,
#[strum(serialize = "1080p")]
P1080,
}
impl VideoPreset {
pub fn all() -> [VideoPreset; 4] {
[Self::P180, Self::P360, Self::P720, Self::P1080]
}
pub fn dimensions(&self) -> (u32, u32) {
match self {
Self::P180 => (320, 180),
Self::P360 => (640, 360),
Self::P720 => (1280, 720),
Self::P1080 => (1920, 1080),
}
}
pub fn width(&self) -> u32 {
self.dimensions().0
}
pub fn height(&self) -> u32 {
self.dimensions().1
}
pub fn fps(&self) -> u32 {
30
}
}
#[derive(Debug, Clone, Copy, Display, EnumString, VariantNames, Eq, PartialEq)]
#[strum(serialize_all = "lowercase")]
pub enum AudioPreset {
Hq,
Lq,
}
#[derive(Debug, Clone, Copy, Display, EnumString, VariantNames, Eq, PartialEq, Default)]
#[strum(serialize_all = "lowercase")]
pub enum Quality {
Highest,
#[default]
High,
Mid,
Low,
}
#[derive(Clone, Default)]
pub struct DecodeConfig {
pub pixel_format: PixelFormat,
}
#[derive(Clone, Default)]
pub struct PlaybackConfig {
pub decode_config: DecodeConfig,
pub quality: Quality,
}

233
third_party/iroh-live/moq-media/src/capture.rs vendored Normal file
View file

@ -0,0 +1,233 @@
use std::str::FromStr;
use anyhow::{Context, Result};
use nokhwa::{
nokhwa_initialize,
pixel_format::RgbFormat,
utils::{
CameraFormat, CameraIndex, FrameFormat, RequestedFormat, RequestedFormatType, Resolution,
},
};
use tracing::{debug, info, trace, warn};
use xcap::{Monitor, VideoRecorder};
use crate::{
av::{PixelFormat, VideoFormat, VideoFrame, VideoSource},
ffmpeg::util::MjpgDecoder,
};
pub struct ScreenCapturer {
pub(crate) _monitor: Monitor,
pub(crate) width: u32,
pub(crate) height: u32,
pub(crate) video_recorder: VideoRecorder,
pub(crate) rx: std::sync::mpsc::Receiver<xcap::Frame>,
}
// TODO: Review if sound.
unsafe impl Send for ScreenCapturer {}
impl Drop for ScreenCapturer {
fn drop(&mut self) {
self.video_recorder.stop().ok();
}
}
impl ScreenCapturer {
pub fn new() -> Result<Self> {
info!("Initializing screen capturer (xcap)");
let monitors = Monitor::all().context("Failed to get monitors")?;
if monitors.is_empty() {
return Err(anyhow::anyhow!("No monitors available"));
}
info!("Available monitors: {monitors:?}");
let monitor = monitors.into_iter().next().unwrap();
let width = monitor.width()?;
let height = monitor.height()?;
let name = monitor
.name()
.unwrap_or_else(|_| "Unknown Monitor".to_string());
info!("Using monitor: {} ({}x{})", name, width, height);
let (video_recorder, rx) = monitor.video_recorder()?;
Ok(Self {
_monitor: monitor,
video_recorder,
rx,
width,
height,
})
}
}
impl VideoSource for ScreenCapturer {
fn name(&self) -> &str {
"screen"
}
fn format(&self) -> VideoFormat {
VideoFormat {
pixel_format: PixelFormat::Rgba,
dimensions: [self.width, self.height],
}
}
fn start(&mut self) -> Result<()> {
self.video_recorder.start()?;
Ok(())
}
fn stop(&mut self) -> Result<()> {
self.video_recorder.stop()?;
Ok(())
}
fn pop_frame(&mut self) -> anyhow::Result<Option<VideoFrame>> {
let mut raw_frame = None;
// We are only interested in the latest frame.
// Drain the channel to not build up memory.
while let Ok(next) = self.rx.try_recv() {
raw_frame = Some(next)
}
let raw_frame = match raw_frame {
Some(frame) => frame,
None => self
.rx
.recv()
.context("Screen recorder did not produce new frame")?,
};
Ok(Some(VideoFrame {
format: VideoFormat {
pixel_format: PixelFormat::Rgba,
dimensions: [raw_frame.width, raw_frame.height],
},
raw: raw_frame.raw.into(),
}))
}
}
pub struct CameraCapturer {
pub(crate) camera: nokhwa::Camera,
pub(crate) mjpg_decoder: MjpgDecoder,
pub(crate) width: u32,
pub(crate) height: u32,
}
impl CameraCapturer {
pub fn new() -> Result<Self> {
info!("Initializing camera capturer (nokhwa)");
nokhwa_initialize(|granted| {
debug!("User selected camera access: {}", granted);
});
let cameras = nokhwa::query(nokhwa::utils::ApiBackend::Auto)?;
if cameras.is_empty() {
return Err(anyhow::anyhow!("No cameras available"));
}
info!("Available cameras: {cameras:?}");
let camera_index = match std::env::var("IROH_LIVE_CAMERA").ok() {
None => {
// Order of cameras in nokhwa is reversed from usual order (primary camera is last).
let first_camera = cameras.last().unwrap();
info!(": {}", first_camera.human_name());
first_camera.index().clone()
}
Some(camera_name) => match u32::from_str(&camera_name).ok() {
Some(num) => CameraIndex::Index(num),
None => CameraIndex::String(camera_name),
},
};
let mut camera = nokhwa::Camera::new(
camera_index,
RequestedFormat::new::<RgbFormat>(RequestedFormatType::AbsoluteHighestResolution),
)?;
info!("Using camera: {}", camera.info().human_name());
let available_formats = camera.compatible_camera_formats()?;
debug!("Available formats: {available_formats:?}",);
if let Some(format) = Self::select_format(available_formats, Resolution::new(1920, 1080)) {
if let Err(err) = camera.set_camera_requset(RequestedFormat::new::<RgbFormat>(
RequestedFormatType::Exact(format),
)) {
warn!(?format, "Failed to change camera format: {err:#}");
}
}
info!("Using format: {}", camera.camera_format());
let resolution = camera.resolution();
Ok(Self {
camera,
mjpg_decoder: MjpgDecoder::new()?,
width: resolution.width(),
height: resolution.height(),
})
}
fn select_format(
mut formats: Vec<CameraFormat>,
desired_resolution: Resolution,
) -> Option<CameraFormat> {
formats.sort_by(|a, b| {
a.resolution()
.cmp(&b.resolution())
.then(a.frame_rate().cmp(&b.frame_rate()))
});
formats
.iter()
.find(|format| format.resolution() >= desired_resolution)
.or_else(|| formats.last())
.cloned()
}
}
impl VideoSource for CameraCapturer {
fn name(&self) -> &str {
"cam"
}
fn format(&self) -> VideoFormat {
VideoFormat {
pixel_format: PixelFormat::Rgba,
dimensions: [self.width, self.height],
}
}
fn start(&mut self) -> Result<()> {
self.camera.open_stream()?;
Ok(())
}
fn stop(&mut self) -> Result<()> {
self.camera.stop_stream()?;
Ok(())
}
fn pop_frame(&mut self) -> anyhow::Result<Option<VideoFrame>> {
let start = std::time::Instant::now();
let frame = self
.camera
.frame()
.context("Failed to capture camera frame")?;
trace!("pop frame: capture took {:?}", start.elapsed());
let start = std::time::Instant::now();
let frame = match frame.source_frame_format() {
FrameFormat::MJPEG if std::env::var("IROH_LIVE_MJPEG_FFMPEG").is_ok() => {
trace!("decode ffmpeg");
self.mjpg_decoder.decode_frame(frame.buffer())?
}
_ => {
let image = frame
.decode_image::<nokhwa::pixel_format::RgbAFormat>()
.context("Failed to decode camera frame")?;
VideoFrame {
format: self.format(),
raw: image.into_raw().into(),
}
}
};
trace!("pop frame: decode took {:?}", start.elapsed());
Ok(Some(frame))
}
}

93
third_party/iroh-live/moq-media/src/ffmpeg/audio/decoder.rs vendored Normal file
View file

@ -0,0 +1,93 @@
use anyhow::Result;
use ffmpeg_next::{self as ffmpeg, util::channel_layout::ChannelLayout};
use hang::catalog::AudioConfig;
use crate::{
av::{AudioDecoder, AudioFormat},
ffmpeg::ext::{CodecContextExt, PacketExt},
};
pub struct FfmpegAudioDecoder {
codec: ffmpeg::decoder::Audio,
resampler: ffmpeg::software::resampling::Context,
decoded_frame: ffmpeg::util::frame::Audio,
resampled_frame: ffmpeg::util::frame::Audio,
}
impl AudioDecoder for FfmpegAudioDecoder {
fn new(config: &AudioConfig, target_format: AudioFormat) -> Result<Self>
where
Self: Sized,
{
let codec = match config.codec {
hang::catalog::AudioCodec::Opus => {
let codec_id = ffmpeg::codec::Id::OPUS;
let codec = ffmpeg::decoder::find(codec_id).unwrap();
let mut ctx = ffmpeg::codec::Context::new_with_codec(codec)
.decoder()
.audio()?;
if let Some(extradata) = &config.description {
ctx.set_extradata(&extradata)?;
}
ctx.set_channel_layout(if config.channel_count == 1 {
ChannelLayout::MONO
} else {
ChannelLayout::STEREO
});
unsafe {
let ctx_mut = ctx.as_mut_ptr();
(*ctx_mut).sample_rate = config.sample_rate as i32;
}
ctx
}
_ => anyhow::bail!(
"Unsupported codec {} (only opus is supported)",
config.codec
),
};
let target_channel_layout = match target_format.channel_count {
1 => ChannelLayout::MONO,
2 => ChannelLayout::STEREO,
_ => anyhow::bail!("unsupported target channel count"),
};
let target_sample_format = ffmpeg_next::util::format::sample::Sample::F32(
ffmpeg_next::util::format::sample::Type::Packed,
);
let resampler = ffmpeg::software::resampling::Context::get(
codec.format(),
codec.channel_layout(),
codec.rate(),
target_sample_format,
target_channel_layout,
target_format.sample_rate,
)?;
Ok(Self {
codec,
resampler,
decoded_frame: ffmpeg::util::frame::Audio::empty(),
resampled_frame: ffmpeg::util::frame::Audio::empty(),
})
}
fn push_packet(&mut self, packet: hang::Frame) -> Result<()> {
let packet = packet.payload.to_ffmpeg_packet();
self.codec.send_packet(&packet)?;
Ok(())
}
fn pop_samples(&mut self) -> Result<Option<&[f32]>> {
match self.codec.receive_frame(&mut self.decoded_frame) {
Err(err) => Err(err.into()),
Ok(()) => {
// Create an empty frame to hold the resampled audio data.
self.resampler
.run(&self.decoded_frame, &mut self.resampled_frame)
.unwrap();
let frame = &self.resampled_frame;
let expected_bytes =
frame.samples() * frame.channels() as usize * core::mem::size_of::<f32>();
Ok(Some(bytemuck::cast_slice(&frame.data(0)[..expected_bytes])))
}
}
}
}

153
third_party/iroh-live/moq-media/src/ffmpeg/audio/encoder.rs vendored Normal file
View file

@ -0,0 +1,153 @@
use anyhow::{Context, Result};
use ffmpeg_next::{self as ffmpeg, Rational};
use hang::{Timestamp, catalog::AudioConfig};
use tracing::trace;
use crate::{
av::{AudioEncoder, AudioEncoderInner, AudioFormat, AudioPreset},
ffmpeg::ext::CodecContextExt,
};
const SAMPLE_RATE: u32 = 48_000;
const BITRATE: u64 = 128_000; // 128 kbps
pub struct OpusEncoder {
encoder: ffmpeg::encoder::Audio,
frame_count: u64,
sample_rate: u32,
bitrate: u64,
channel_count: u32,
extradata: Vec<u8>,
}
impl OpusEncoder {
pub fn stereo() -> Result<Self> {
Self::new(SAMPLE_RATE, 2, BITRATE)
}
pub fn mono() -> Result<Self> {
Self::new(SAMPLE_RATE, 1, BITRATE)
}
pub fn new(sample_rate: u32, channel_count: u32, bitrate: u64) -> Result<Self> {
tracing::info!(
"Initializing Opus encoder: {}Hz, {} channels",
sample_rate,
channel_count
);
ffmpeg::init()?;
let codec =
ffmpeg::encoder::find(ffmpeg::codec::Id::OPUS).context("Opus encoder not found")?;
tracing::debug!("Found Opus codec: {:?}", codec.name());
let mut ctx = ffmpeg::codec::context::Context::new_with_codec(codec)
.encoder()
.audio()?;
let sample_rate = sample_rate as i32;
ctx.set_rate(sample_rate);
ctx.set_bit_rate(bitrate as usize);
ctx.set_format(ffmpeg::format::Sample::F32(
ffmpeg_next::format::sample::Type::Packed,
));
ctx.set_time_base(Rational::new(1, sample_rate));
ctx.set_channel_layout(if channel_count == 1 {
ffmpeg::util::channel_layout::ChannelLayout::MONO
} else {
ffmpeg::util::channel_layout::ChannelLayout::STEREO
});
let encoder = ctx.open()?;
let extradata = encoder.extradata().unwrap_or(&[]).to_vec();
tracing::info!("Opus encoder initialized successfully");
Ok(Self {
encoder,
frame_count: 0,
sample_rate: sample_rate as u32,
channel_count,
extradata,
bitrate,
})
}
}
impl AudioEncoder for OpusEncoder {
fn with_preset(format: AudioFormat, preset: AudioPreset) -> Result<Self>
where
Self: Sized,
{
let channel_count = format.channel_count;
let bitrate = match preset {
AudioPreset::Hq => BITRATE,
AudioPreset::Lq => 32_000,
};
Self::new(SAMPLE_RATE, channel_count, bitrate)
}
}
impl AudioEncoderInner for OpusEncoder {
fn name(&self) -> &str {
self.encoder.id().name()
}
fn config(&self) -> AudioConfig {
hang::catalog::AudioConfig {
codec: hang::catalog::AudioCodec::Opus,
sample_rate: self.sample_rate,
channel_count: self.channel_count,
bitrate: Some(self.bitrate),
description: Some(self.extradata.clone().into()),
}
}
fn push_samples(&mut self, samples: &[f32]) -> Result<()> {
if samples.is_empty() {
return Ok(());
}
let samples_per_channel = samples.len() / self.channel_count as usize;
debug_assert_eq!(samples_per_channel as u32, self.encoder.frame_size());
let mut audio_frame = ffmpeg::util::frame::Audio::new(
ffmpeg::util::format::sample::Sample::F32(ffmpeg::util::format::sample::Type::Packed),
samples_per_channel,
ffmpeg::util::channel_layout::ChannelLayout::default(self.channel_count as i32),
);
// Copy interleaved samples directly since we're using packed format
let frame_data = audio_frame.data_mut(0);
let frame_samples: &mut [f32] = bytemuck::cast_slice_mut(frame_data);
let copy_len = samples.len().min(frame_samples.len());
frame_samples[..copy_len].copy_from_slice(&samples[..copy_len]);
audio_frame.set_pts(Some(self.frame_count as i64));
self.frame_count += samples_per_channel as u64;
trace!("push samples {}", audio_frame.samples());
self.encoder.send_frame(&audio_frame)?;
Ok(())
}
fn pop_packet(&mut self) -> Result<Option<hang::Frame>> {
let mut packet = ffmpeg::packet::Packet::empty();
match self.encoder.receive_packet(&mut packet) {
Ok(()) => {
let payload = packet.data().unwrap_or(&[]).to_vec();
let hang_frame = hang::Frame {
payload: payload.into(),
timestamp: Timestamp::from_micros(
(self.frame_count * 1_000_000) / self.sample_rate as u64,
)?,
keyframe: true, // Audio frames are generally independent
};
trace!("poll frame {}", hang_frame.payload.num_bytes());
Ok(Some(hang_frame))
}
Err(ffmpeg::Error::Eof) => Ok(None),
Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::util::error::EAGAIN => Ok(None),
Err(e) => Err(e.into()),
}
}
}

108
third_party/iroh-live/moq-media/src/ffmpeg/mod.rs vendored Normal file
View file

@ -0,0 +1,108 @@
use crate::av::Decoders;
pub use self::{audio::*, ext::ffmpeg_log_init, video::*};
#[derive(Debug, Clone, Copy)]
pub struct FfmpegDecoders;
impl Decoders for FfmpegDecoders {
type Audio = FfmpegAudioDecoder;
type Video = FfmpegVideoDecoder;
}
mod audio {
mod decoder;
mod encoder;
pub use decoder::*;
pub use encoder::*;
}
pub mod video {
mod decoder;
mod encoder;
pub(crate) mod util;
pub use decoder::*;
pub use encoder::*;
}
pub(crate) mod ext {
use buf_list::BufList;
use bytes::Buf;
use ffmpeg_next as ffmpeg;
pub fn ffmpeg_log_init() {
use ffmpeg::util::log::Level::*;
let level = if let Ok(val) = std::env::var("FFMPEG_LOG") {
match val.as_str() {
"quiet" => Quiet,
"panic" => Panic,
"fatal" => Fatal,
"error" => Error,
"warn" | "warning" => Warning,
"info" => Info,
"verbose" => Verbose,
"debug" => Debug,
"trace" => Trace,
_ => Warning,
}
} else {
Warning
};
ffmpeg::util::log::set_level(level);
}
pub trait PacketExt {
fn to_ffmpeg_packet(self) -> ffmpeg::Packet;
}
impl PacketExt for BufList {
fn to_ffmpeg_packet(mut self) -> ffmpeg_next::Packet {
let mut packet = ffmpeg::Packet::new(self.num_bytes());
let dst = packet.data_mut().unwrap();
self.copy_to_slice(dst);
packet
}
}
pub trait CodecContextExt {
fn extradata(&self) -> Option<&[u8]>;
fn set_extradata(&mut self, extradata: &[u8]) -> Result<(), ffmpeg::Error>;
}
impl CodecContextExt for ffmpeg::codec::Context {
// SAFETY: Written by ChatGPT, so, dunno.
fn extradata(&self) -> Option<&[u8]> {
unsafe {
let ctx = self.as_ptr();
if (*ctx).extradata.is_null() || (*ctx).extradata_size <= 0 {
return None;
}
Some(std::slice::from_raw_parts(
(*ctx).extradata as *const u8,
(*ctx).extradata_size as usize,
))
}
}
// SAFETY: Written by ChatGPT, so, dunno.
fn set_extradata(&mut self, extradata: &[u8]) -> Result<(), ffmpeg::Error> {
unsafe {
let ctx = self.as_mut_ptr();
// allocate extradata + padding
let pad = ffmpeg::ffi::AV_INPUT_BUFFER_PADDING_SIZE as usize;
let size = extradata.len() + pad;
(*ctx).extradata = ffmpeg::ffi::av_mallocz(size).cast::<u8>();
if (*ctx).extradata.is_null() {
return Err(ffmpeg::Error::Bug.into());
}
// copy bytes and zero the padding
std::ptr::copy_nonoverlapping(
extradata.as_ptr(),
(*ctx).extradata,
extradata.len(),
);
(*ctx).extradata_size = extradata.len() as i32;
}
Ok(())
}
}
}

139
third_party/iroh-live/moq-media/src/ffmpeg/video/decoder.rs vendored Normal file
View file

@ -0,0 +1,139 @@
use anyhow::{Context, Result};
use ffmpeg_next::{
self as ffmpeg, codec, codec::Id as CodecId, util::frame::video::Video as FfmpegFrame,
};
use crate::{
av::{self, DecodeConfig, DecodedFrame, VideoDecoder},
ffmpeg::{
ext::{CodecContextExt, PacketExt},
video::util::{Rescaler, StreamClock},
},
};
pub struct FfmpegVideoDecoder {
codec: ffmpeg::decoder::Video,
rescaler: Rescaler,
clock: StreamClock,
decoded: FfmpegFrame,
viewport_changed: Option<(u32, u32)>,
last_timestamp: Option<hang::Timestamp>,
}
impl VideoDecoder for FfmpegVideoDecoder {
fn name(&self) -> &str {
self.codec.id().name()
}
fn new(config: &hang::catalog::VideoConfig, playback_config: &DecodeConfig) -> Result<Self>
where
Self: Sized,
{
ffmpeg::init()?;
// Build a decoder context for H.264 and attach extradata (e.g., avcC)
let codec = match &config.codec {
hang::catalog::VideoCodec::H264(_meta) => {
let codec =
codec::decoder::find(CodecId::H264).context("H.264 decoder not found")?;
let mut ctx = codec::context::Context::new_with_codec(codec);
if let Some(description) = &config.description {
ctx.set_extradata(&description)?;
}
ctx.decoder().video().unwrap()
}
hang::catalog::VideoCodec::AV1(_meta) => {
let codec = codec::decoder::find(CodecId::AV1).context("AV1 decoder not found")?;
let mut ctx = codec::context::Context::new_with_codec(codec);
if let Some(description) = &config.description {
ctx.set_extradata(&description)?;
}
ctx.decoder().video().unwrap()
}
_ => anyhow::bail!(
"Unsupported codec {} (only h264 and av1 are supported)",
config.codec
),
};
let rescaler = Rescaler::new(playback_config.pixel_format.to_ffmpeg(), None)?;
let clock = StreamClock::default();
let decoded = FfmpegFrame::empty();
Ok(Self {
codec,
rescaler,
clock,
decoded,
viewport_changed: None,
last_timestamp: None,
})
}
fn set_viewport(&mut self, w: u32, h: u32) {
self.viewport_changed = Some((w, h));
}
fn push_packet(&mut self, packet: hang::Frame) -> Result<()> {
let ffmpeg_packet = packet.payload.to_ffmpeg_packet();
self.codec.send_packet(&ffmpeg_packet)?;
self.last_timestamp = Some(packet.timestamp);
Ok(())
}
fn pop_frame(&mut self) -> Result<Option<av::DecodedFrame>> {
// Pull all available decoded frames
match self.codec.receive_frame(&mut self.decoded) {
Ok(()) => {
// Apply clamped target size.
if let Some((max_width, max_height)) = self.viewport_changed.take() {
let (width, height) =
calculate_resized_size(&self.decoded, max_width, max_height);
self.rescaler.set_target_dimensions(width, height);
}
let frame = self.rescaler.process(&mut self.decoded)?;
let last_timestamp = self
.last_timestamp
.as_ref()
.context("missing last packet")?;
let frame = DecodedFrame::from_ffmpeg(
frame,
self.clock.frame_delay(&last_timestamp),
std::time::Duration::from(*last_timestamp),
);
Ok(Some(frame))
}
Err(ffmpeg::util::error::Error::BufferTooSmall) => Ok(None),
Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::util::error::EAGAIN => Ok(None),
Err(err) => {
// tracing::warn!("decoder error: {err} {err:?} {err:#?}");
// Ok(None)
Err(err.into())
}
}
}
}
/// Calculates the target frame size to fit into the requested bounds while preserving aspect ratio.
fn calculate_resized_size(decoded: &FfmpegFrame, max_width: u32, max_height: u32) -> (u32, u32) {
let src_w = decoded.width().max(1);
let src_h = decoded.height().max(1);
let max_w = max_width.max(1);
let max_h = max_height.max(1);
// Fit within requested bounds, preserve aspect ratio, never upscale
let scale_w = (max_w as f32) / (src_w as f32);
let scale_h = (max_h as f32) / (src_h as f32);
let scale = scale_w.min(scale_h).min(1.0).max(0.0);
let target_width = ((src_w as f32) * scale).floor().max(1.0) as u32;
let target_height = ((src_h as f32) * scale).floor().max(1.0) as u32;
tracing::debug!(
src_w,
src_h,
max_w,
max_h,
target_width,
target_height,
"scale"
);
(target_width, target_height)
}

568
third_party/iroh-live/moq-media/src/ffmpeg/video/encoder.rs vendored Normal file
View file

@ -0,0 +1,568 @@
use std::{
ffi::{CString, c_int},
ptr,
task::Poll,
};
use anyhow::{Context, Result, anyhow};
use ffmpeg_next::{self as ffmpeg, codec, format::Pixel, frame::Video as VideoFrame};
use hang::Timestamp;
use tracing::{debug, info, trace};
use crate::{
av,
ffmpeg::{ext::CodecContextExt, util::Rescaler},
};
#[derive(Debug, Clone, Copy, Default)]
// Allow unused because usage is cfg-gated on platform.
#[allow(unused)]
enum HwBackend {
#[default]
Software,
/// Linux
Vaapi,
/// macOS
Videotoolbox,
/// Nvidia GPUs
Nvenc,
/// Intel GPUs
Qsv,
/// AMD GPUs
Amf,
// TODO:
// Add DirectX (Windows)
// Add MediaCodec (Android)
}
impl HwBackend {
fn codec_name(&self) -> &'static str {
match self {
Self::Software => "libx264",
Self::Vaapi => "h264_vaapi",
Self::Videotoolbox => "h264_videotoolbox",
Self::Nvenc => "h264_nvenc",
Self::Qsv => "h264_qsv",
Self::Amf => "h264_amf",
}
}
fn candidates() -> Vec<Self> {
// vec![HwBackend::Software]
let mut candidates = Vec::new();
// Platform-preferred order
#[cfg(target_os = "macos")]
candidates.extend_from_slice(&[HwBackend::Videotoolbox]);
#[cfg(target_os = "windows")]
candidates.extend_from_slice(&[HwBackend::Nvenc, HwBackend::Qsv, HwBackend::Amf]);
#[cfg(target_os = "linux")]
candidates.extend_from_slice(&[HwBackend::Vaapi, HwBackend::Nvenc, HwBackend::Qsv]);
// Always end with software
candidates.push(HwBackend::Software);
candidates
}
fn pixel_format(&self) -> Pixel {
match self {
HwBackend::Vaapi | HwBackend::Qsv => Pixel::NV12,
// These rest accepts yuv420p SW frames:
_ => Pixel::YUV420P,
}
}
fn hardware_pixel_format(&self) -> Pixel {
match self {
HwBackend::Vaapi => Pixel::VAAPI,
HwBackend::Qsv => Pixel::NV12,
// These rest accepts yuv420p SW frames:
_ => Pixel::YUV420P,
}
}
}
#[derive(Debug, Clone)]
struct EncoderOpts {
width: u32,
height: u32,
framerate: u32,
bitrate: u64,
}
pub struct H264Encoder {
encoder: ffmpeg::encoder::video::Encoder,
rescaler: Rescaler,
backend: HwBackend,
vaapi: Option<VaapiState>,
opts: EncoderOpts,
frame_count: u64,
}
impl H264Encoder {
pub fn new(width: u32, height: u32, framerate: u32) -> Result<Self> {
info!("Initializing H264 encoder: {width}x{height} @ {framerate}fps");
ffmpeg::init()?;
// Bitrate heuristic (from your original)
let pixels = width * height;
let framerate_factor = 30.0 + (framerate as f32 - 30.) / 2.;
let bitrate = (pixels as f32 * 0.07 * framerate_factor).round() as u64;
let opts = EncoderOpts {
width,
height,
framerate,
bitrate,
};
let candidates = HwBackend::candidates();
// Try each backend
let mut last_err: Option<anyhow::Error> = None;
for backend in candidates {
match Self::open_encoder(backend, &opts) {
Ok((encoder, rescaler, vaapi)) => {
info!(
"Using encoder backend: {} ({backend:?})",
backend.codec_name()
);
return Ok(Self {
encoder,
rescaler,
vaapi,
backend,
opts,
frame_count: 0,
});
}
Err(e) => {
debug!(
"Backend {backend:?} ({}) not available: {e:#}",
backend.codec_name()
);
last_err = Some(e);
}
}
}
Err(last_err.unwrap_or_else(|| anyhow!("no H.264 encoder available")))
}
fn open_encoder(
backend: HwBackend,
opts: &EncoderOpts,
) -> Result<(
ffmpeg::encoder::video::Encoder,
Rescaler,
Option<VaapiState>,
)> {
// Find encoder
let codec = ffmpeg::codec::encoder::find_by_name(backend.codec_name())
.with_context(|| format!("encoder {} not found", backend.codec_name()))?;
debug!("Found encoder: {}", codec.name());
// Build ctx
let mut ctx = codec::context::Context::new_with_codec(codec);
unsafe {
let ctx_mut = ctx.as_mut_ptr();
(*ctx_mut).width = opts.width as i32;
(*ctx_mut).height = opts.height as i32;
(*ctx_mut).time_base.num = 1;
(*ctx_mut).time_base.den = opts.framerate as i32;
(*ctx_mut).framerate.num = opts.framerate as i32;
(*ctx_mut).framerate.den = 1;
(*ctx_mut).gop_size = opts.framerate as i32;
(*ctx_mut).bit_rate = opts.bitrate as i64;
(*ctx_mut).flags = (*ctx_mut).flags | codec::Flags::GLOBAL_HEADER.bits() as c_int;
(*ctx_mut).pix_fmt = backend.hardware_pixel_format().into();
}
// Backend-specific prep
let vaapi_state = if matches!(backend, HwBackend::Vaapi) {
// single-GPU default; make configurable if needed
let va = VaapiState::new(opts.width, opts.height, "/dev/dri/renderD128")?;
va.bind_to_context(&mut ctx);
Some(va)
} else {
None
};
// Setup encoder options
let enc_opts = {
let mut opts = vec![
// Disable annexB so that we get an avcC header in extradata
// annexb=0 → MP4/ISO BMFF style (length-prefixed NAL units + avcC extradata),
// as opposed to Annex B start codes (00 00 00 01).
("annexB", "0"),
];
if matches!(backend, HwBackend::Software) {
opts.extend_from_slice(&[
("preset", "ultrafast"),
("tune", "zerolatency"),
("profile", "baseline"),
]);
}
ffmpeg::Dictionary::from_iter(opts.into_iter())
};
// Open encoder
let encoder = ctx.encoder().video()?.open_as_with(codec, enc_opts)?;
// Build rescaler to SW input fmt expected per-backend
let rescaler = Rescaler::new(backend.pixel_format(), Some((opts.width, opts.height)))?;
Ok((encoder, rescaler, vaapi_state))
}
pub fn video_config(&self) -> Result<hang::catalog::VideoConfig> {
Ok(hang::catalog::VideoConfig {
codec: hang::catalog::VideoCodec::H264(hang::catalog::H264 {
profile: 0x42, // Baseline
constraints: 0xE0,
level: 0x1E, // Level 3.0
inline: false, // TODO: is this correct?
}),
description: Some(self.avcc_description()?.to_vec().into()),
coded_width: Some(self.opts.width),
coded_height: Some(self.opts.height),
display_ratio_width: None,
display_ratio_height: None,
bitrate: Some(self.opts.bitrate),
framerate: Some(self.opts.framerate as f64),
optimize_for_latency: Some(true),
})
}
pub fn avcc_description(&self) -> Result<&[u8]> {
self.encoder.extradata().context("missing avcC extradata")
}
pub fn receive_packet(&mut self) -> Result<Poll<Option<hang::Frame>>> {
loop {
let mut packet = ffmpeg::packet::Packet::empty();
match self.encoder.receive_packet(&mut packet) {
Ok(()) => {
let payload = packet.data().unwrap_or(&[]).to_vec();
let hang_frame = hang::Frame {
payload: payload.into(),
timestamp: Timestamp::from_micros(
self.frame_count * 1_000_000 / self.opts.framerate as u64,
)?,
keyframe: packet.is_key(),
};
return Ok(Poll::Ready(Some(hang_frame)));
}
Err(ffmpeg::Error::Eof) => return Ok(Poll::Ready(None)),
Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::util::error::EAGAIN => {
return Ok(Poll::Pending);
}
Err(e) => return Err(e.into()),
}
}
}
pub fn encode_frame(&mut self, mut frame: VideoFrame) -> Result<()> {
frame.set_pts(Some(self.frame_count as i64));
self.frame_count += 1;
if self.frame_count % self.opts.framerate as u64 == 0 {
tracing::trace!(
"Encoding {}: {}x{} fmt={:?} pts={:?} backend={:?}",
self.frame_count,
frame.width(),
frame.height(),
frame.format(),
frame.pts(),
self.backend
);
}
// Convert incoming frame to the SW format the backend expects.
let frame = self
.rescaler
.process(&frame)
.context("failed to color-convert frame")?
.clone();
let frame = match self.backend {
HwBackend::Vaapi => {
let va = self
.vaapi
.as_ref()
.ok_or_else(|| anyhow!("no vaapi state"))?;
let hw_frame = va.transfer_nv12_to_hw(&frame)?;
hw_frame
}
// Other backends accept SW frames directly
_ => frame,
};
self.encoder
.send_frame(&frame)
.map_err(|e| anyhow!("send_frame failed: {e:?}"))?;
Ok(())
}
pub fn flush(&mut self) -> Result<()> {
self.encoder.send_eof()?;
Ok(())
}
}
impl av::VideoEncoder for H264Encoder {
fn with_preset(preset: av::VideoPreset) -> Result<Self>
where
Self: Sized,
{
Self::new(preset.width(), preset.height(), preset.fps())
}
}
impl av::VideoEncoderInner for H264Encoder {
fn name(&self) -> &str {
self.encoder.id().name()
}
fn config(&self) -> hang::catalog::VideoConfig {
self.video_config().expect("video_config available")
}
fn push_frame(&mut self, frame: av::VideoFrame) -> anyhow::Result<()> {
trace!(len = frame.raw.len(), format=?frame.format, "push frame");
let frame = frame.to_ffmpeg();
self.encode_frame(frame)
}
fn pop_packet(&mut self) -> anyhow::Result<Option<hang::Frame>> {
match self.receive_packet()? {
std::task::Poll::Ready(v) => Ok(v),
std::task::Poll::Pending => Ok(None),
}
}
}
struct VaapiState {
device_ctx: *mut ffmpeg::sys::AVBufferRef,
frames_ctx: *mut ffmpeg::sys::AVBufferRef,
}
unsafe impl Send for VaapiState {}
impl VaapiState {
/// Create VAAPI device + frames pool (NV12→VAAPI surfaces) for given size.
fn new(width: u32, height: u32, device_path: &str) -> Result<Self> {
// 1) Create VAAPI device
let cpath = CString::new(device_path)?;
let mut dev: *mut ffmpeg::sys::AVBufferRef = ptr::null_mut();
let ret = unsafe {
ffmpeg::sys::av_hwdevice_ctx_create(
&mut dev,
ffmpeg::sys::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
cpath.as_ptr(),
ptr::null_mut(),
0,
)
};
if ret < 0 || dev.is_null() {
unsafe { ffmpeg::sys::av_buffer_unref(&mut dev) };
return Err(anyhow!("vaapi device create failed: {ret}"));
}
// 2) Create frames pool for VAAPI with SW format NV12
let frames = unsafe { ffmpeg::sys::av_hwframe_ctx_alloc(dev) };
if frames.is_null() {
unsafe { ffmpeg::sys::av_buffer_unref(&mut dev) };
return Err(anyhow!("av_hwframe_ctx_alloc failed"));
}
let fc = unsafe { &mut *((*frames).data as *mut ffmpeg::sys::AVHWFramesContext) };
fc.format = ffmpeg::sys::AVPixelFormat::AV_PIX_FMT_VAAPI;
fc.sw_format = ffmpeg::sys::AVPixelFormat::AV_PIX_FMT_NV12;
fc.width = width as i32;
fc.height = height as i32;
fc.initial_pool_size = 32;
let ret = unsafe { ffmpeg::sys::av_hwframe_ctx_init(frames) };
if ret < 0 {
unsafe {
ffmpeg::sys::av_buffer_unref(&mut (frames as *mut _));
ffmpeg::sys::av_buffer_unref(&mut dev);
}
return Err(anyhow!("av_hwframe_ctx_init failed: {ret}"));
}
Ok(Self {
device_ctx: dev,
frames_ctx: frames,
})
}
/// Attach the frames context so the codec context expects VAAPI frames.
fn bind_to_context(&self, ctx: &mut codec::context::Context) {
unsafe {
let ctx = ctx.as_mut_ptr();
(*ctx).hw_frames_ctx = ffmpeg::sys::av_buffer_ref(self.frames_ctx);
(*ctx).pix_fmt = Pixel::VAAPI.into();
}
}
/// Transfer a SW NV12 frame into a VAAPI HW frame, preserving PTS.
/// Returns a new `VideoFrame` backed by a VAAPI surface.
fn transfer_nv12_to_hw(&self, sw_frame: &VideoFrame) -> Result<VideoFrame> {
unsafe {
// Allocate an empty HW frame from the pool
let mut hw = ffmpeg::frame::Video::empty();
let ret = ffmpeg::sys::av_hwframe_get_buffer(self.frames_ctx, hw.as_mut_ptr(), 0);
if ret < 0 {
return Err(anyhow!("av_hwframe_get_buffer failed: {ret}"));
}
// Keep PTS
(*hw.as_mut_ptr()).pts = sw_frame.pts().unwrap_or(0);
// Transfer SW NV12 → HW VAAPI surface
let ret = ffmpeg::sys::av_hwframe_transfer_data(hw.as_mut_ptr(), sw_frame.as_ptr(), 0);
if ret < 0 {
return Err(anyhow!("av_hwframe_transfer_data failed: {ret}"));
}
Ok(hw)
}
}
}
impl Drop for VaapiState {
fn drop(&mut self) {
unsafe {
if !self.frames_ctx.is_null() {
ffmpeg::sys::av_buffer_unref(&mut (self.frames_ctx as *mut _));
}
if !self.device_ctx.is_null() {
ffmpeg::sys::av_buffer_unref(&mut (self.device_ctx as *mut _));
}
}
}
}
// pub struct Av1FfmpegEncoder {
// encoder: ffmpeg::encoder::video::Encoder,
// rescaler: Rescaler,
// opts: EncoderOpts,
// frame_count: u64,
// }
// impl Av1FfmpegEncoder {
// pub fn new(width: u32, height: u32, framerate: u32) -> Result<Self> {
// info!("Initializing AV1 (FFmpeg) encoder: {width}x{height} @ {framerate}fps");
// ffmpeg::init()?;
// let pixels = width * height;
// let framerate_factor = 30.0 + (framerate as f32 - 30.) / 2.;
// let bitrate = (pixels as f32 * 0.05 * framerate_factor).round() as u64;
// let opts = EncoderOpts {
// width,
// height,
// framerate,
// bitrate,
// };
// let codec = ffmpeg::encoder::find(ffmpeg::codec::Id::AV1).context("AV1 codec not found")?;
// let mut ctx = codec::context::Context::new_with_codec(codec);
// unsafe {
// let ctx_mut = ctx.as_mut_ptr();
// (*ctx_mut).width = width as i32;
// (*ctx_mut).height = height as i32;
// (*ctx_mut).time_base.num = 1;
// (*ctx_mut).time_base.den = framerate as i32;
// (*ctx_mut).framerate.num = framerate as i32;
// (*ctx_mut).framerate.den = 1;
// (*ctx_mut).gop_size = framerate as i32;
// (*ctx_mut).bit_rate = bitrate as i64;
// (*ctx_mut).pix_fmt = Pixel::YUV420P.into();
// }
// // libaom options for realtime
// let enc_opts =
// ffmpeg::Dictionary::from_iter([("cpu-used", "8"), ("row-mt", "1"), ("tiles", "2x2")]);
// let encoder = ctx.encoder().video()?.open_as_with(
// ffmpeg::encoder::find(ffmpeg::codec::Id::AV1).unwrap(),
// enc_opts,
// )?;
// let rescaler = Rescaler::new(Pixel::YUV420P, Some((width, height)))?;
// Ok(Self {
// encoder,
// rescaler,
// opts,
// frame_count: 0,
// })
// }
// }
// impl av::VideoEncoder for Av1FfmpegEncoder {
// fn with_preset(preset: av::VideoPreset) -> Result<Self>
// where
// Self: Sized,
// {
// Self::new(preset.width(), preset.height(), preset.fps())
// }
// fn config(&self) -> hang::catalog::VideoConfig {
// hang::catalog::VideoConfig {
// codec: hang::catalog::VideoCodec::AV1(Default::default()),
// description: None,
// coded_width: Some(self.opts.width),
// coded_height: Some(self.opts.height),
// display_ratio_width: None,
// display_ratio_height: None,
// bitrate: Some(self.opts.bitrate),
// framerate: Some(self.opts.framerate as f64),
// optimize_for_latency: Some(true),
// }
// }
// fn push_frame(
// &mut self,
// format: &av::VideoFormat,
// frame: av::VideoFrame,
// ) -> anyhow::Result<()> {
// use ffmpeg_next::frame::Video as FfFrame;
// let pixel = match format.pixel_format {
// av::PixelFormat::Rgba => Pixel::RGBA,
// av::PixelFormat::Bgra => Pixel::BGRA,
// };
// let [w, h] = format.dimensions;
// let mut ff = FfFrame::new(pixel, w, h);
// let stride = ff.stride(0) as usize;
// let row_bytes = (w as usize) * 4;
// for y in 0..(h as usize) {
// let dst_off = y * stride;
// let src_off = y * row_bytes;
// ff.data_mut(0)[dst_off..dst_off + row_bytes]
// .copy_from_slice(&frame.raw[src_off..src_off + row_bytes]);
// }
// let sw = self
// .rescaler
// .process(&ff)
// .context("failed to color-convert frame")?
// .clone();
// let mut enc_frame = sw;
// enc_frame.set_pts(Some(self.frame_count as i64));
// self.frame_count += 1;
// self.encoder.send_frame(&enc_frame)?;
// Ok(())
// }
// fn pop_packet(&mut self) -> anyhow::Result<Option<hang::Frame>> {
// let mut packet = ffmpeg::packet::Packet::empty();
// match self.encoder.receive_packet(&mut packet) {
// Ok(()) => {
// let payload = packet.data().unwrap_or(&[]).to_vec();
// let hang_frame = hang::Frame {
// payload: payload.into(),
// timestamp: std::time::Duration::from_nanos(
// self.frame_count.saturating_sub(1) * 1_000_000_000
// / self.opts.framerate as u64,
// ),
// keyframe: packet.is_key(),
// };
// Ok(Some(hang_frame))
// }
// Err(ffmpeg::Error::Eof) => Ok(None),
// Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::util::error::EAGAIN => Ok(None),
// Err(e) => Err(e.into()),
// }
// }
// }

138
third_party/iroh-live/moq-media/src/ffmpeg/video/util.rs vendored Normal file
View file

@ -0,0 +1,138 @@
use std::time::Duration;
use bytes::{BufMut, BytesMut};
use ffmpeg_next::util::{format::pixel::Pixel, frame::video::Video as FfmpegFrame};
use hang::Timestamp;
use image::{Delay, RgbaImage};
pub(crate) use self::mjpg_decoder::MjpgDecoder;
pub(crate) use self::rescaler::Rescaler;
use crate::av::{self, DecodedFrame, PixelFormat, VideoFormat, VideoFrame};
mod mjpg_decoder;
mod rescaler;
#[derive(Default, Debug)]
pub(crate) struct StreamClock {
pub(crate) last_timestamp: Option<hang::Timestamp>,
}
impl StreamClock {
pub(crate) fn frame_delay(&mut self, timestamp: &hang::Timestamp) -> Duration {
// Compute interframe delay from provided timestamps
let delay = match self.last_timestamp {
None => Duration::ZERO,
Some(last_timestamp) => timestamp
.checked_sub(last_timestamp)
.unwrap_or(Timestamp::ZERO)
.into(),
};
self.last_timestamp = Some(*timestamp);
delay
}
}
impl av::VideoFrame {
pub fn to_ffmpeg(&self) -> FfmpegFrame {
// Wrap raw RGBA/BGRA data into an ffmpeg frame and encode
let pixel = match self.format.pixel_format {
av::PixelFormat::Rgba => Pixel::RGBA,
av::PixelFormat::Bgra => Pixel::BGRA,
};
let [w, h] = self.format.dimensions;
let mut ff = FfmpegFrame::new(pixel, w, h);
let stride = ff.stride(0) as usize;
let row_bytes = (w as usize) * 4;
for y in 0..(h as usize) {
let dst_off = y * stride;
let src_off = y * row_bytes;
ff.data_mut(0)[dst_off..dst_off + row_bytes]
.copy_from_slice(&self.raw[src_off..src_off + row_bytes]);
}
ff
}
}
impl av::DecodedFrame {
pub fn from_ffmpeg(frame: &FfmpegFrame, delay: Duration, timestamp: Duration) -> Self {
let image = ffmpeg_frame_to_image(frame);
// Compute interframe delay from provided timestamps
let delay = Delay::from_saturating_duration(delay);
DecodedFrame {
frame: image::Frame::from_parts(image, 0, 0, delay),
timestamp,
}
}
}
/// Convert the ffmpeg frame into an [image] frame.
///
/// Note: This does not do any color conversion. Make sure the frame is in the correct color format before.
///
/// This allocates the full frame into a vec, which we need anyway to cross the thread boundary.
pub(crate) fn ffmpeg_frame_to_image(frame: &ffmpeg_next::util::frame::Video) -> image::RgbaImage {
let width = frame.width();
let height = frame.height();
let bytes_per_pixel = 4usize; // BGRA
let src = frame.data(0);
// ffmpeg frames may have padding at end of each line; copy row-by-row.
let stride = frame.stride(0) as usize;
let row_bytes = (width as usize) * bytes_per_pixel;
let mut out = vec![0u8; row_bytes * (height as usize)];
for y in 0..(height as usize) {
let src_off = y * stride;
let dst_off = y * row_bytes;
out[dst_off..dst_off + row_bytes].copy_from_slice(&src[src_off..src_off + row_bytes]);
}
RgbaImage::from_raw(width, height, out).expect("valid image buffer")
}
impl PixelFormat {
pub fn to_ffmpeg(&self) -> Pixel {
match self {
PixelFormat::Rgba => Pixel::RGBA,
PixelFormat::Bgra => Pixel::BGRA,
}
}
pub fn from_ffmpeg(value: Pixel) -> Option<Self> {
match value {
Pixel::RGBA => Some(PixelFormat::Rgba),
Pixel::BGRA => Some(PixelFormat::Bgra),
_ => None,
}
}
}
/// Convert the ffmpeg frame into a [`VideoFrame`]
///
/// Returns `None` if the frame has an unsupported pixel format.
///
/// This allocates the full frame into a vec, which we need anyway to cross the thread boundary.
pub(crate) fn ffmpeg_frame_to_video_frame(
frame: &ffmpeg_next::util::frame::Video,
) -> Option<VideoFrame> {
let pixel_format = PixelFormat::from_ffmpeg(frame.format())?;
let width = frame.width();
let height = frame.height();
let bytes_per_pixel = 4usize; // RGBA/BGRA
let src = frame.data(0);
// ffmpeg frames may have padding at end of each line; copy row-by-row.
let stride = frame.stride(0) as usize;
let row_bytes = (width as usize) * bytes_per_pixel;
let mut out = BytesMut::with_capacity(row_bytes * (height as usize));
// let mut out = vec![0u8; row_bytes * (height as usize)];
for y in 0..(height as usize) {
let src_off = y * stride;
// let dst_off = y * row_bytes;
out.put(&src[src_off..src_off + row_bytes]);
// out[dst_off..dst_off + row_bytes].copy_from_slice(&src[src_off..src_off + row_bytes]);
}
Some(VideoFrame {
format: VideoFormat {
dimensions: [width, height],
pixel_format,
},
raw: out.freeze(),
})
}

74
third_party/iroh-live/moq-media/src/ffmpeg/video/util/mjpg_decoder.rs vendored Normal file
View file

@ -0,0 +1,74 @@
use std::time::Instant;
use ffmpeg_next::{
self as ffmpeg, Error, Packet, codec::Id, format::Pixel, frame::Video as FfmpegVideoFrame,
};
use tracing::trace;
use crate::{
av::VideoFrame,
ffmpeg::util::{Rescaler, ffmpeg_frame_to_video_frame},
};
pub struct MjpgDecoder {
dec: ffmpeg::decoder::Video,
rescaler: Rescaler,
}
impl MjpgDecoder {
/// Initialize FFmpeg and create a Video decoder for MJPEG.
pub fn new() -> anyhow::Result<Self> {
ffmpeg::init()?;
// Find the MJPEG decoder and create a context bound to it.
let mjpeg = ffmpeg::decoder::find(Id::MJPEG).ok_or(Error::DecoderNotFound)?;
// Create a codec::Context that's pre-bound to this decoder codec,
// then get a video decoder out of it.
let ctx = ffmpeg::codec::context::Context::new_with_codec(mjpeg);
let dec = ctx.decoder().video()?; // has send_packet/receive_frame
let rescaler = Rescaler::new(Pixel::RGBA, None)?;
Ok(Self { dec, rescaler })
}
/// Decode one complete MJPEG/JPEG frame from `mjpg_frame`.
pub fn decode_frame(&mut self, mjpg_frame: &[u8]) -> Result<VideoFrame, Error> {
let now = Instant::now();
// Make a packet that borrows/copies the data.
let packet = Packet::borrow(mjpg_frame);
// Feed & drain once — MJPEG is intra-only (one picture per packet).
self.dec.send_packet(&packet)?;
trace!(t=?now.elapsed(), "decode ffmpeg: send packet");
let mut frame = FfmpegVideoFrame::empty();
self.dec.receive_frame(&mut frame)?;
trace!(t=?now.elapsed(), "decode ffmpeg: receive frame");
// MJPEG may output deprecated YUVJ* formats. Replace them with
// the non-deprecated equivalents and mark full range to keep semantics.
// This avoids ffmpeg warning: "deprecated pixel format used, make sure you did set range correctly".
use ffmpeg_next::util::color::Range;
match frame.format() {
Pixel::YUVJ420P => {
frame.set_color_range(Range::JPEG);
frame.set_format(Pixel::YUV420P);
}
Pixel::YUVJ422P => {
frame.set_color_range(Range::JPEG);
frame.set_format(Pixel::YUV422P);
}
Pixel::YUVJ444P => {
frame.set_color_range(Range::JPEG);
frame.set_format(Pixel::YUV444P);
}
_ => {}
}
trace!(t=?now.elapsed(), "decode ffmpeg: color");
let frame = self.rescaler.process(&frame)?;
trace!(t=?now.elapsed(), "decode ffmpeg: rescale");
let frame = ffmpeg_frame_to_video_frame(frame).expect("valid pixel format set in rescaler");
trace!(t=?now.elapsed(), "decode ffmpeg: convert");
Ok(frame)
}
}

77
third_party/iroh-live/moq-media/src/ffmpeg/video/util/rescaler.rs vendored Normal file
View file

@ -0,0 +1,77 @@
use anyhow::Result;
use ffmpeg_next::software::scaling::Flags;
use ffmpeg_next::{
self as ffmpeg,
software::scaling::{self},
util::{format::pixel::Pixel, frame::video::Video as FfmpegFrame},
};
pub(crate) struct Rescaler {
pub(crate) target_format: Pixel,
pub(crate) target_width_height: Option<(u32, u32)>,
pub(crate) ctx: Option<scaling::Context>,
pub(crate) out_frame: FfmpegFrame,
}
// I think the ffmpeg structs are send-safe.
// We want to create the encoder before moving it to a thread.
unsafe impl Send for Rescaler {}
impl Rescaler {
pub fn new(target_format: Pixel, target_width_height: Option<(u32, u32)>) -> Result<Self> {
Ok(Self {
target_format,
ctx: None,
target_width_height,
out_frame: FfmpegFrame::empty(),
})
}
pub fn set_target_dimensions(&mut self, w: u32, h: u32) {
self.target_width_height = Some((w, h));
}
pub fn process<'a: 'b, 'b>(
&'a mut self,
frame: &'b FfmpegFrame,
) -> Result<&'b FfmpegFrame, ffmpeg::Error> {
// Short-circuit if possible.
if self.target_width_height.is_none() && self.target_format == frame.format() {
return Ok(frame);
}
let (target_width, target_height) = self
.target_width_height
.unwrap_or_else(|| (frame.width(), frame.height()));
let out_frame_needs_reset = self.out_frame.width() != target_width
|| self.out_frame.height() != target_height
|| self.out_frame.format() != self.target_format;
if out_frame_needs_reset {
self.out_frame = FfmpegFrame::new(self.target_format, target_width, target_height);
}
let ctx = match self.ctx {
None => self.ctx.insert(scaling::Context::get(
frame.format(),
frame.width(),
frame.height(),
self.out_frame.format(),
self.out_frame.width(),
self.out_frame.height(),
Flags::BILINEAR,
)?),
Some(ref mut ctx) => ctx,
};
// This resets the context if any parameters changed.
ctx.cached(
frame.format(),
frame.width(),
frame.height(),
self.out_frame.format(),
self.out_frame.width(),
self.out_frame.height(),
Flags::BILINEAR,
);
ctx.run(&frame, &mut self.out_frame)?;
Ok(&self.out_frame)
}
}

9
third_party/iroh-live/moq-media/src/lib.rs vendored Normal file
View file

@ -0,0 +1,9 @@
pub mod audio;
pub mod av;
pub mod capture;
pub mod ffmpeg;
pub mod publish;
pub mod subscribe;
mod util;
pub use audio::AudioBackend;

594
third_party/iroh-live/moq-media/src/publish.rs vendored Normal file
View file

@ -0,0 +1,594 @@
use std::{
collections::{BTreeMap, HashMap},
sync::{
Arc, Mutex,
atomic::{AtomicBool, AtomicU32, Ordering},
},
time::{Duration, Instant},
};
use anyhow::Context;
use hang::catalog::{AudioConfig, Catalog, CatalogProducer, VideoConfig};
use moq_lite::BroadcastProducer;
use n0_error::Result;
use n0_future::task::AbortOnDropHandle;
use tokio_util::sync::{CancellationToken, DropGuard};
use tracing::{debug, error, info, info_span, trace, warn};
use crate::{
av::{
AudioEncoder, AudioEncoderInner, AudioPreset, AudioSource, DecodeConfig, VideoEncoder,
VideoEncoderInner, VideoPreset, VideoSource,
},
subscribe::WatchTrack,
util::spawn_thread,
};
pub struct PublishBroadcast {
producer: BroadcastProducer,
catalog: CatalogProducer,
state: Arc<Mutex<State>>,
_task: Arc<AbortOnDropHandle<()>>,
}
impl PublishBroadcast {
pub fn new() -> Self {
let mut producer = BroadcastProducer::default();
let catalog = Catalog::default().produce();
producer.insert_track(catalog.consumer.track);
let catalog = catalog.producer;
let state = Arc::new(Mutex::new(State::default()));
let task_handle = tokio::spawn(Self::run(state.clone(), producer.clone()));
Self {
producer,
catalog,
state,
_task: Arc::new(AbortOnDropHandle::new(task_handle)),
}
}
pub fn producer(&self) -> BroadcastProducer {
self.producer.clone()
}
async fn run(state: Arc<Mutex<State>>, mut producer: BroadcastProducer) {
while let Some(track) = producer.requested_track().await {
let name = track.info.name.clone();
if state
.lock()
.expect("poisoned")
.start_track(track.clone())
.inspect_err(|err| warn!(%name, "failed to start requested track: {err:#}"))
.is_ok()
{
info!("started track: {name}");
tokio::spawn({
let state = state.clone();
async move {
track.unused().await;
info!("stopping track: {name}");
state.lock().expect("poisoned").stop_track(&name);
}
});
}
}
}
/// Create a local WatchTrack from the current video source, if present.
pub fn watch_local(&self, decode_config: DecodeConfig) -> Option<WatchTrack> {
let (source, shutdown) = {
let state = self.state.lock().expect("poisoned");
let source = state
.available_video
.as_ref()
.map(|video| video.source.clone())?;
Some((source, state.shutdown_token.child_token()))
}?;
Some(WatchTrack::from_video_source(
"local".to_string(),
shutdown,
source,
decode_config,
))
}
pub fn set_video(&mut self, renditions: Option<VideoRenditions>) -> Result<()> {
match renditions {
Some(renditions) => {
let priority = 1u8;
let configs = renditions.available_renditions()?;
let video = hang::catalog::Video {
renditions: configs,
priority,
display: None,
rotation: None,
flip: None,
};
{
let mut catalog = self.catalog.lock();
catalog.video = Some(video);
}
self.state.lock().expect("poisoned").available_video = Some(renditions);
// TODO: Drop active encodings if their rendition is no longer available?
}
None => {
// Clear catalog and stop any active video encoders
self.state.lock().expect("poisoned").remove_video();
{
let mut catalog = self.catalog.lock();
catalog.video = None;
}
}
}
Ok(())
}
pub fn set_audio(&mut self, renditions: Option<AudioRenditions>) -> Result<()> {
match renditions {
Some(renditions) => {
let priority = 2u8;
let configs = renditions.available_renditions()?;
let audio = hang::catalog::Audio {
renditions: configs,
priority,
};
{
let mut catalog = self.catalog.lock();
catalog.audio = Some(audio);
}
self.state.lock().expect("poisoned").available_audio = Some(renditions);
}
None => {
// Clear catalog and stop any active audio encoders
self.state.lock().expect("poisoned").remove_audio();
{
let mut catalog = self.catalog.lock();
catalog.audio = None;
}
}
}
Ok(())
}
}
impl Drop for PublishBroadcast {
fn drop(&mut self) {
self.state.lock().expect("poisoned").shutdown_token.cancel();
self.producer.close();
}
}
#[derive(Default)]
struct State {
shutdown_token: CancellationToken,
available_video: Option<VideoRenditions>,
available_audio: Option<AudioRenditions>,
active_video: HashMap<String, EncoderThread>,
active_audio: HashMap<String, EncoderThread>,
}
impl State {
fn stop_track(&mut self, name: &str) {
let thread = self
.active_video
.remove(name)
.or_else(|| self.active_audio.remove(name));
if let Some(thread) = thread {
thread.shutdown.cancel();
}
}
fn remove_audio(&mut self) {
for (_name, thread) in self.active_audio.drain() {
thread.shutdown.cancel();
}
self.available_audio = None;
}
fn remove_video(&mut self) {
for (_name, thread) in self.active_video.drain() {
thread.shutdown.cancel();
}
self.available_video = None;
}
fn start_track(&mut self, track: moq_lite::TrackProducer) -> Result<()> {
let name = track.info.name.clone();
let track = hang::TrackProducer::new(track);
let shutdown_token = self.shutdown_token.child_token();
if let Some(video) = self.available_video.as_mut()
&& video.contains_rendition(&name)
{
let thread = video.start_encoder(&name, track, shutdown_token)?;
self.active_video.insert(name, thread);
Ok(())
} else if let Some(audio) = self.available_audio.as_mut()
&& audio.contains_rendition(&name)
{
let thread = audio.start_encoder(&name, track, shutdown_token)?;
self.active_audio.insert(name, thread);
Ok(())
} else {
info!("ignoring track request {name}: rendition not available");
Err(n0_error::anyerr!("rendition not available"))
}
}
}
pub struct AudioRenditions {
make_encoder: Box<dyn Fn(AudioPreset) -> Result<Box<dyn AudioEncoder>> + Send>,
source: Box<dyn AudioSource>,
renditions: HashMap<String, AudioPreset>,
}
impl AudioRenditions {
pub fn new<E: AudioEncoder>(
source: impl AudioSource,
presets: impl IntoIterator<Item = AudioPreset>,
) -> Self {
let renditions = presets
.into_iter()
.map(|preset| (format!("audio-{preset}"), preset))
.collect();
let format = source.format();
Self {
make_encoder: Box::new(move |preset| Ok(Box::new(E::with_preset(format, preset)?))),
renditions,
source: Box::new(source),
}
}
pub fn available_renditions(&self) -> Result<BTreeMap<String, AudioConfig>> {
let mut renditions = BTreeMap::new();
for (name, preset) in self.renditions.iter() {
// We need to create the encoder to get the config, even though we drop it
// again (it will be created on deman). Not ideal, but works for now.
let config = (self.make_encoder)(*preset)?.config();
renditions.insert(name.clone(), config);
}
Ok(renditions)
}
pub fn encoder(&mut self, name: &str) -> Option<Result<Box<dyn AudioEncoder>>> {
let preset = self.renditions.get(name)?;
Some((self.make_encoder)(*preset))
}
pub fn contains_rendition(&self, name: &str) -> bool {
self.renditions.contains_key(name)
}
pub fn start_encoder(
&mut self,
name: &str,
producer: hang::TrackProducer,
shutdown_token: CancellationToken,
) -> Result<EncoderThread> {
let preset = self
.renditions
.get(name)
.context("rendition not available")?;
let encoder = (self.make_encoder)(*preset)?;
let thread = EncoderThread::spawn_audio(
self.source.cloned_boxed(),
encoder,
producer,
shutdown_token,
);
Ok(thread)
}
}
pub struct VideoRenditions {
make_encoder: Box<dyn Fn(VideoPreset) -> Result<Box<dyn VideoEncoder>> + Send>,
source: SharedVideoSource,
renditions: HashMap<String, VideoPreset>,
_shared_source_cancel_guard: DropGuard,
}
impl VideoRenditions {
pub fn new<E: VideoEncoder>(
source: impl VideoSource,
presets: impl IntoIterator<Item = VideoPreset>,
) -> Self {
let shutdown_token = CancellationToken::new();
let source = SharedVideoSource::new(source, shutdown_token.clone());
let renditions = presets
.into_iter()
.map(|preset| (format!("video-{preset}"), preset))
.collect();
Self {
make_encoder: Box::new(|preset| Ok(Box::new(E::with_preset(preset)?))),
renditions,
source,
_shared_source_cancel_guard: shutdown_token.drop_guard(),
}
}
pub fn available_renditions(&self) -> Result<BTreeMap<String, VideoConfig>> {
let mut renditions = BTreeMap::new();
for (name, preset) in self.renditions.iter() {
// We need to create the encoder to get the config, even though we drop it
// again (it will be created on deman). Not ideal, but works for now.
let config = (self.make_encoder)(*preset)?.config();
renditions.insert(name.clone(), config);
}
Ok(renditions)
}
pub fn contains_rendition(&self, name: &str) -> bool {
self.renditions.contains_key(name)
}
pub fn start_encoder(
&mut self,
name: &str,
producer: hang::TrackProducer,
shutdown_token: CancellationToken,
) -> Result<EncoderThread> {
let preset = self
.renditions
.get(name)
.context("rendition not available")?;
let encoder = (self.make_encoder)(*preset)?;
let thread =
EncoderThread::spawn_video(self.source.clone(), encoder, producer, shutdown_token);
Ok(thread)
}
}
#[derive(Debug, Clone)]
pub(crate) struct SharedVideoSource {
name: String,
frames_rx: tokio::sync::watch::Receiver<Option<crate::av::VideoFrame>>,
format: crate::av::VideoFormat,
running: Arc<AtomicBool>,
thread: Arc<std::thread::JoinHandle<()>>,
subscriber_count: Arc<AtomicU32>,
}
impl SharedVideoSource {
fn new(mut source: impl VideoSource, shutdown: CancellationToken) -> Self {
let name = source.name().to_string();
let format = source.format();
let (tx, rx) = tokio::sync::watch::channel(None);
let running = Arc::new(AtomicBool::new(false));
let thread = spawn_thread(format!("vshr-{}", source.name()), {
let shutdown = shutdown.clone();
let running = running.clone();
move || {
let frame_time = Duration::from_secs_f32(1. / 30.);
let start = Instant::now();
for i in 0.. {
if shutdown.is_cancelled() {
break;
}
loop {
if running.load(Ordering::Relaxed) {
break;
}
if let Err(err) = source.stop() {
warn!("Failed to stop video source: {err:#}");
}
std::thread::park();
if let Err(err) = source.start() {
warn!("Failed to stop video source: {err:#}");
}
}
match source.pop_frame() {
Ok(Some(frame)) => {
let _ = tx.send(Some(frame));
}
Ok(None) => {}
Err(_) => break,
}
let expected = frame_time * i;
let actual = start.elapsed();
if actual < expected {
std::thread::sleep(expected - actual);
}
}
}
});
Self {
name,
format,
frames_rx: rx,
thread: Arc::new(thread),
running,
subscriber_count: Default::default(),
}
}
}
impl VideoSource for SharedVideoSource {
fn name(&self) -> &str {
&self.name
}
fn format(&self) -> crate::av::VideoFormat {
self.format.clone()
}
fn start(&mut self) -> anyhow::Result<()> {
let prev_count = self.subscriber_count.fetch_add(1, Ordering::Relaxed);
if prev_count == 0 {
self.running.store(true, Ordering::Relaxed);
self.thread.thread().unpark();
}
Ok(())
}
fn stop(&mut self) -> anyhow::Result<()> {
if self
.subscriber_count
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |val| {
Some(val.saturating_sub(1))
})
.expect("always returns Some")
== 1
{
self.running.store(false, Ordering::Relaxed);
}
Ok(())
}
fn pop_frame(&mut self) -> anyhow::Result<Option<crate::av::VideoFrame>> {
let frame = self.frames_rx.borrow_and_update().clone();
Ok(frame)
}
}
pub struct EncoderThread {
_thread_handle: std::thread::JoinHandle<()>,
shutdown: CancellationToken,
}
impl EncoderThread {
pub fn spawn_video(
mut source: impl VideoSource,
mut encoder: impl VideoEncoderInner,
mut producer: hang::TrackProducer,
shutdown: CancellationToken,
) -> Self {
let thread_name = format!("venc-{:<4}-{:<4}", source.name(), encoder.name());
let span = info_span!("videoenc", source = source.name(), encoder = encoder.name());
let handle = spawn_thread(thread_name, {
let shutdown = shutdown.clone();
move || {
let _guard = span.enter();
if let Err(err) = source.start() {
warn!("video source failed to start: {err:#}");
return;
}
let format = source.format();
tracing::debug!(
src_format = ?format,
dst_config = ?encoder.config(),
"video encoder thread start"
);
let framerate = encoder.config().framerate.unwrap_or(30.0);
let interval = Duration::from_secs_f64(1. / framerate);
loop {
let start = Instant::now();
if shutdown.is_cancelled() {
debug!("stop video encoder: cancelled");
break;
}
let frame = match source.pop_frame() {
Ok(frame) => frame,
Err(err) => {
warn!("video encoder failed: {err:#}");
break;
}
};
if let Some(frame) = frame {
if let Err(err) = encoder.push_frame(frame) {
warn!("video encoder failed: {err:#}");
break;
};
while let Ok(Some(pkt)) = encoder.pop_packet() {
if let Err(err) = producer.write(pkt) {
warn!("failed to write frame to producer: {err:#}");
}
}
}
std::thread::sleep(interval.saturating_sub(start.elapsed()));
}
producer.inner.close();
if let Err(err) = source.stop() {
warn!("video source failed to stop: {err:#}");
}
tracing::debug!("video encoder thread stop");
}
});
Self {
_thread_handle: handle,
shutdown,
}
}
pub fn spawn_audio(
mut source: Box<dyn AudioSource>,
mut encoder: impl AudioEncoderInner,
mut producer: hang::TrackProducer,
shutdown: CancellationToken,
) -> Self {
let sd = shutdown.clone();
let name = encoder.name();
let thread_name = format!("aenc-{:<4}", name);
let span = info_span!("audioenc", %name);
let handle = spawn_thread(thread_name, move || {
let _guard = span.enter();
tracing::debug!(config=?encoder.config(), "audio encoder thread start");
let shutdown = sd;
// 20ms framing to align with typical Opus config (48kHz → 960 samples/ch)
const INTERVAL: Duration = Duration::from_millis(20);
let format = source.format();
let samples_per_frame = (format.sample_rate / 1000) * INTERVAL.as_millis() as u32;
let mut buf = vec![0.0f32; samples_per_frame as usize * format.channel_count as usize];
let start = Instant::now();
for tick in 0.. {
trace!("tick");
if shutdown.is_cancelled() {
break;
}
match source.pop_samples(&mut buf) {
Ok(Some(_n)) => {
// Expect a full frame; if shorter, zero-pad via slice len
if let Err(err) = encoder.push_samples(&buf) {
error!(buf_len = buf.len(), "audio push_samples failed: {err:#}");
break;
}
while let Ok(Some(pkt)) = encoder
.pop_packet()
.inspect_err(|err| warn!("encoder error: {err:#}"))
{
if let Err(err) = producer.write(pkt) {
warn!("failed to write frame to producer: {err:#}");
}
}
}
Ok(None) => {
// keep pacing
}
Err(err) => {
error!("audio source failed: {err:#}");
break;
}
}
let expected_time = (tick + 1) * INTERVAL;
let actual_time = start.elapsed();
if actual_time > expected_time {
warn!("audio thread too slow by {:?}", actual_time - expected_time);
}
let sleep = expected_time.saturating_sub(start.elapsed());
if sleep > Duration::ZERO {
std::thread::sleep(sleep);
}
}
// drain
while let Ok(Some(pkt)) = encoder.pop_packet() {
if let Err(err) = producer.write(pkt) {
warn!("failed to write frame to producer: {err:#}");
}
}
producer.inner.close();
tracing::debug!("audio encoder thread stop");
});
Self {
_thread_handle: handle,
shutdown,
}
}
}
impl Drop for EncoderThread {
fn drop(&mut self) {
self.shutdown.cancel();
}
}

712
third_party/iroh-live/moq-media/src/subscribe.rs vendored Normal file
View file

@ -0,0 +1,712 @@
use std::{collections::BTreeMap, sync::Arc, time::Duration};
use hang::{
Timestamp, TrackConsumer,
catalog::{AudioConfig, Catalog, CatalogConsumer, VideoConfig},
};
use moq_lite::{BroadcastConsumer, Track};
use n0_error::{Result, StackResultExt, StdResultExt};
use n0_future::task::AbortOnDropHandle;
use n0_watcher::{Watchable, Watcher};
use tokio::{
sync::mpsc::{self, error::TryRecvError},
time::Instant,
};
use tokio_util::sync::{CancellationToken, DropGuard};
use tracing::{Span, debug, error, info, info_span, trace, warn};
use crate::{
av::{
AudioDecoder, AudioSink, AudioSinkHandle, DecodeConfig, DecodedFrame, Decoders,
PlaybackConfig, Quality, VideoDecoder, VideoSource,
},
ffmpeg::util::Rescaler,
util::spawn_thread,
};
const DEFAULT_MAX_LATENCY: Duration = Duration::from_millis(150);
#[derive(derive_more::Debug, Clone)]
pub struct SubscribeBroadcast {
broadcast_name: String,
#[debug("BroadcastConsumer")]
broadcast: BroadcastConsumer,
// catalog_watcher: n0_watcher::Direct<CatalogWrapper>,
catalog_watchable: Watchable<CatalogWrapper>,
shutdown: CancellationToken,
_catalog_task: Arc<AbortOnDropHandle<()>>,
}
#[derive(Debug, derive_more::PartialEq, derive_more::Eq, Default, Clone, derive_more::Deref)]
pub struct CatalogWrapper {
#[eq(skip)]
#[deref]
inner: Arc<Catalog>,
seq: usize,
}
impl CatalogWrapper {
fn new(inner: Catalog, seq: usize) -> Self {
Self {
inner: Arc::new(inner),
seq,
}
}
pub fn video_renditions(&self) -> impl Iterator<Item = &str> {
let mut renditions: Vec<_> = self
.inner
.video
.as_ref()
.iter()
.map(|v| v.renditions.iter())
.flatten()
.map(|(name, config)| (name.as_str(), config.coded_width))
.collect();
renditions.sort_by(|a, b| a.1.cmp(&b.1));
renditions.into_iter().map(|(name, _w)| name)
}
pub fn audio_renditions(&self) -> impl Iterator<Item = &str> + '_ {
self.inner
.audio
.as_ref()
.into_iter()
.map(|v| v.renditions.iter())
.flatten()
.map(|(name, _config)| name.as_str())
}
pub fn select_video_rendition(&self, quality: Quality) -> Result<String> {
let video = self.inner.video.as_ref().context("no video published")?;
let track_name =
select_video_rendition(&video.renditions, quality).context("no video renditions")?;
Ok(track_name)
}
pub fn select_audio_rendition(&self, quality: Quality) -> Result<String> {
let audio = self.inner.audio.as_ref().context("no video published")?;
let track_name =
select_audio_rendition(&audio.renditions, quality).context("no video renditions")?;
Ok(track_name)
}
}
impl CatalogWrapper {
pub fn into_inner(self) -> Arc<Catalog> {
self.inner
}
}
impl SubscribeBroadcast {
pub async fn new(broadcast_name: String, broadcast: BroadcastConsumer) -> Result<Self> {
let shutdown = CancellationToken::new();
let (catalog_watchable, catalog_task) = {
let track = broadcast.subscribe_track(&Catalog::default_track());
let mut consumer = CatalogConsumer::new(track);
let initial_catalog = consumer
.next()
.await
.std_context("Broadcast closed before receiving catalog")?
.context("Catalog track closed before receiving catalog")?;
let watchable = Watchable::new(CatalogWrapper::new(initial_catalog, 0));
let task = tokio::spawn({
let shutdown = shutdown.clone();
let watchable = watchable.clone();
async move {
for seq in 1.. {
match consumer.next().await {
Ok(Some(catalog)) => {
watchable.set(CatalogWrapper::new(catalog, seq)).ok();
}
Ok(None) => {
debug!("subscribed broadcast catalog track ended");
break;
}
Err(err) => {
debug!("subscribed broadcast closed: {err:#}");
break;
}
}
}
shutdown.cancel();
}
});
(watchable, task)
};
Ok(Self {
broadcast_name,
broadcast,
catalog_watchable,
_catalog_task: Arc::new(AbortOnDropHandle::new(catalog_task)),
shutdown: CancellationToken::new(),
})
}
pub fn broadcast_name(&self) -> &str {
&self.broadcast_name
}
pub fn catalog_watcher(&mut self) -> n0_watcher::Direct<CatalogWrapper> {
self.catalog_watchable.watch()
}
pub fn catalog(&self) -> CatalogWrapper {
self.catalog_watchable.get()
}
pub fn watch_and_listen<D: Decoders>(
self,
audio_out: impl AudioSink,
playback_config: PlaybackConfig,
) -> Result<AvRemoteTrack> {
AvRemoteTrack::new::<D>(self, audio_out, playback_config)
}
pub fn watch<D: VideoDecoder>(&self) -> Result<WatchTrack> {
self.watch_with::<D>(&Default::default(), Quality::Highest)
}
pub fn watch_with<D: VideoDecoder>(
&self,
playback_config: &DecodeConfig,
quality: Quality,
) -> Result<WatchTrack> {
let track_name = self.catalog().select_video_rendition(quality)?;
self.watch_rendition::<D>(playback_config, &track_name)
}
pub fn watch_rendition<D: VideoDecoder>(
&self,
playback_config: &DecodeConfig,
track_name: &str,
) -> Result<WatchTrack> {
let catalog = self.catalog();
let video = catalog.video.as_ref().context("no video published")?;
let config = video
.renditions
.get(track_name)
.context("rendition not found")?;
let consumer = TrackConsumer::new(
self.broadcast.subscribe_track(&Track {
name: track_name.to_string(),
priority: video.priority,
}),
DEFAULT_MAX_LATENCY,
);
let span = info_span!("videodec", %track_name);
WatchTrack::from_consumer::<D>(
track_name.to_string(),
consumer,
&config,
playback_config,
self.shutdown.child_token(),
span,
)
}
pub fn listen<D: AudioDecoder>(&self, output: impl AudioSink) -> Result<AudioTrack> {
self.listen_with::<D>(Quality::Highest, output)
}
pub fn listen_with<D: AudioDecoder>(
&self,
quality: Quality,
output: impl AudioSink,
) -> Result<AudioTrack> {
let track_name = self.catalog().select_audio_rendition(quality)?;
self.listen_rendition::<D>(&track_name, output)
}
pub fn listen_rendition<D: AudioDecoder>(
&self,
name: &str,
output: impl AudioSink,
) -> Result<AudioTrack> {
let catalog = self.catalog();
let audio = catalog.audio.as_ref().context("no audio published")?;
let config = audio.renditions.get(name).context("rendition not found")?;
let consumer = TrackConsumer::new(
self.broadcast.subscribe_track(&Track {
name: name.to_string(),
priority: audio.priority,
}),
DEFAULT_MAX_LATENCY,
);
let span = info_span!("audiodec", %name);
AudioTrack::spawn::<D>(
name.to_string(),
consumer,
config.clone(),
output,
self.shutdown.child_token(),
span,
)
}
pub fn closed(&self) -> impl Future<Output = ()> + 'static {
self.broadcast.closed()
}
pub fn shutdown(&self) {
self.shutdown.cancel();
}
}
fn select_rendition<T, P: ToString>(
renditions: &BTreeMap<String, T>,
order: &[P],
) -> Option<String> {
order
.iter()
.map(ToString::to_string)
.find(|k| renditions.contains_key(k.as_str()))
.or_else(|| renditions.keys().next().cloned())
}
fn select_video_rendition<'a, T>(
renditions: &'a BTreeMap<String, T>,
q: Quality,
) -> Option<String> {
use crate::av::VideoPreset::*;
let order = match q {
Quality::Highest => [P1080, P720, P360, P180],
Quality::High => [P720, P360, P180, P1080],
Quality::Mid => [P360, P180, P720, P1080],
Quality::Low => [P180, P360, P720, P1080],
};
select_rendition(renditions, &order)
}
fn select_audio_rendition<'a, T>(
renditions: &'a BTreeMap<String, T>,
q: Quality,
) -> Option<String> {
use crate::av::AudioPreset::*;
let order = match q {
Quality::Highest | Quality::High => [Hq, Lq],
Quality::Mid | Quality::Low => [Lq, Hq],
};
select_rendition(renditions, &order)
}
pub struct AudioTrack {
name: String,
handle: Box<dyn AudioSinkHandle>,
shutdown_token: CancellationToken,
_task_handle: AbortOnDropHandle<()>,
_thread_handle: std::thread::JoinHandle<()>,
}
impl AudioTrack {
pub(crate) fn spawn<D: AudioDecoder>(
name: String,
consumer: TrackConsumer,
config: AudioConfig,
output: impl AudioSink,
shutdown: CancellationToken,
span: Span,
) -> Result<Self> {
let _guard = span.enter();
let (packet_tx, packet_rx) = mpsc::channel(32);
let output_format = output.format()?;
info!(?config, "audio thread start");
let decoder = D::new(&config, output_format)?;
let handle = output.handle();
let thread_name = format!("adec-{}", name);
let thread = spawn_thread(thread_name, {
let shutdown = shutdown.clone();
let span = span.clone();
move || {
let _guard = span.enter();
if let Err(err) = Self::run_loop(decoder, packet_rx, output, &shutdown) {
error!("audio decoder failed: {err:#}");
}
info!("audio decoder thread stop");
}
});
let task = tokio::spawn(forward_frames(consumer, packet_tx));
Ok(Self {
name,
handle,
shutdown_token: shutdown,
_task_handle: AbortOnDropHandle::new(task),
_thread_handle: thread,
})
}
pub fn stopped(&self) -> impl Future<Output = ()> + 'static {
let shutdown_token = self.shutdown_token.clone();
async move { shutdown_token.cancelled().await }
}
pub fn rendition(&self) -> &str {
&self.name
}
pub fn handle(&self) -> &dyn AudioSinkHandle {
self.handle.as_ref()
}
pub(crate) fn run_loop(
mut decoder: impl AudioDecoder,
mut packet_rx: mpsc::Receiver<hang::Frame>,
mut sink: impl AudioSink,
shutdown: &CancellationToken,
) -> Result<()> {
const INTERVAL: Duration = Duration::from_millis(10);
let mut remote_start = None;
let loop_start = Instant::now();
'main: for i in 0.. {
let tick = Instant::now();
if shutdown.is_cancelled() {
debug!("stop audio thread: cancelled");
break;
}
loop {
match packet_rx.try_recv() {
Ok(packet) => {
let remote_start = *remote_start.get_or_insert_with(|| packet.timestamp);
if tracing::enabled!(tracing::Level::TRACE) {
let loop_elapsed = tick.duration_since(loop_start);
let remote_elapsed: Duration = packet
.timestamp
.checked_sub(remote_start)
.unwrap_or(Timestamp::ZERO)
.into();
let diff_ms =
(loop_elapsed.as_secs_f32() - remote_elapsed.as_secs_f32()) * 1000.;
trace!(len = packet.payload.num_bytes(), ts=?packet.timestamp, ?loop_elapsed, ?remote_elapsed, ?diff_ms, "recv packet");
}
// TODO: Skip outdated packets?
if !sink.is_paused() {
decoder.push_packet(packet)?;
if let Some(samples) = decoder.pop_samples()? {
sink.push_samples(samples)?;
}
}
}
Err(TryRecvError::Disconnected) => {
debug!("stop audio thread: packet_rx disconnected");
break 'main;
}
Err(TryRecvError::Empty) => {
trace!("no packet to recv");
break;
}
}
}
let target_time = i * INTERVAL;
let real_time = Instant::now().duration_since(loop_start);
let sleep = target_time.saturating_sub(real_time);
if !sleep.is_zero() {
std::thread::sleep(sleep);
}
}
shutdown.cancel();
Ok(())
}
}
impl Drop for AudioTrack {
fn drop(&mut self) {
self.shutdown_token.cancel();
}
}
pub struct WatchTrack {
video_frames: WatchTrackFrames,
handle: WatchTrackHandle,
}
pub struct WatchTrackHandle {
rendition: String,
viewport: Watchable<(u32, u32)>,
_guard: WatchTrackGuard,
}
impl WatchTrackHandle {
pub fn set_viewport(&self, w: u32, h: u32) {
self.viewport.set((w, h)).ok();
}
pub fn rendition(&self) -> &str {
&self.rendition
}
}
pub struct WatchTrackFrames {
rx: mpsc::Receiver<DecodedFrame>,
}
impl WatchTrackFrames {
pub fn current_frame(&mut self) -> Option<DecodedFrame> {
let mut out = None;
while let Ok(item) = self.rx.try_recv() {
out = Some(item);
}
out
}
pub async fn next_frame(&mut self) -> Option<DecodedFrame> {
if let Some(frame) = self.current_frame() {
Some(frame)
} else {
self.rx.recv().await
}
}
}
struct WatchTrackGuard {
_shutdown_token_guard: DropGuard,
_task_handle: Option<AbortOnDropHandle<()>>,
_thread_handle: Option<std::thread::JoinHandle<()>>,
}
impl WatchTrack {
pub fn empty(rendition: impl ToString) -> Self {
let (tx, rx) = mpsc::channel(1);
let task = tokio::task::spawn(async move {
std::future::pending::<()>().await;
let _ = tx;
});
let guard = WatchTrackGuard {
_shutdown_token_guard: CancellationToken::new().drop_guard(),
_task_handle: Some(AbortOnDropHandle::new(task)),
_thread_handle: None,
};
Self {
video_frames: WatchTrackFrames { rx },
handle: WatchTrackHandle {
rendition: rendition.to_string(),
viewport: Default::default(),
_guard: guard,
},
}
}
pub fn from_video_source(
rendition: String,
shutdown: CancellationToken,
mut source: impl VideoSource,
decode_config: DecodeConfig,
) -> Self {
let viewport = Watchable::new((1u32, 1u32));
let (frame_tx, frame_rx) = tokio::sync::mpsc::channel::<DecodedFrame>(2);
let thread_name = format!("vpr-{:>4}-{:>4}", source.name(), rendition);
let thread = spawn_thread(thread_name, {
let mut viewport = viewport.watch();
let shutdown = shutdown.clone();
move || {
// TODO: Make configurable.
let fps = 30;
let mut rescaler = Rescaler::new(decode_config.pixel_format.to_ffmpeg(), None)
.expect("failed to create rescaler");
let frame_duration = Duration::from_secs_f32(1. / fps as f32);
if let Err(err) = source.start() {
warn!("Video source failed to start: {err:?}");
return;
}
let start = Instant::now();
for i in 1.. {
// let t = Instant::now();
if shutdown.is_cancelled() {
break;
}
if viewport.update() {
let (w, h) = viewport.peek();
rescaler.set_target_dimensions(*w, *h);
}
match source.pop_frame() {
Ok(Some(frame)) => {
// trace!(t=?t.elapsed(), "pop");
let frame = frame.to_ffmpeg();
let frame = rescaler.process(&frame).expect("rescaler failed");
let frame =
DecodedFrame::from_ffmpeg(frame, frame_duration, start.elapsed());
// trace!(t=?t.elapsed(), "convert");
let _ = frame_tx.blocking_send(frame);
// trace!(t=?t.elapsed(), "send");
}
Ok(None) => {}
Err(_) => break,
}
let expected_time = i * frame_duration;
let actual_time = start.elapsed();
if expected_time > actual_time {
std::thread::sleep(expected_time - actual_time);
// trace!(t=?t.elapsed(), slept=?(actual_time - expected_time), ?expected_time, ?actual_time, "done");
}
}
if let Err(err) = source.stop() {
warn!("Video source failed to stop: {err:?}");
return;
}
}
});
let guard = WatchTrackGuard {
_shutdown_token_guard: shutdown.drop_guard(),
_task_handle: None,
_thread_handle: Some(thread),
};
WatchTrack {
video_frames: WatchTrackFrames { rx: frame_rx },
handle: WatchTrackHandle {
rendition,
viewport,
_guard: guard,
},
}
}
pub(crate) fn from_consumer<D: VideoDecoder>(
rendition: String,
consumer: TrackConsumer,
config: &VideoConfig,
playback_config: &DecodeConfig,
shutdown: CancellationToken,
span: Span,
) -> Result<Self> {
let (packet_tx, packet_rx) = mpsc::channel(32);
let (frame_tx, frame_rx) = mpsc::channel(32);
let viewport = Watchable::new((1u32, 1u32));
let viewport_watcher = viewport.watch();
let _guard = span.enter();
debug!(?config, "video decoder start");
let decoder = D::new(config, playback_config)?;
let thread_name = format!("vdec-{}", rendition);
let thread = spawn_thread(thread_name, {
let shutdown = shutdown.clone();
let span = span.clone();
move || {
let _guard = span.enter();
if let Err(err) =
Self::run_loop(&shutdown, packet_rx, frame_tx, viewport_watcher, decoder)
{
error!("video decoder failed: {err:#}");
}
shutdown.cancel();
}
});
let task = tokio::task::spawn(forward_frames(consumer, packet_tx));
let guard = WatchTrackGuard {
_shutdown_token_guard: shutdown.drop_guard(),
_task_handle: Some(AbortOnDropHandle::new(task)),
_thread_handle: Some(thread),
};
Ok(WatchTrack {
video_frames: WatchTrackFrames { rx: frame_rx },
handle: WatchTrackHandle {
rendition,
viewport,
_guard: guard,
},
})
}
pub fn split(self) -> (WatchTrackFrames, WatchTrackHandle) {
(self.video_frames, self.handle)
}
pub fn set_viewport(&self, w: u32, h: u32) {
self.handle.set_viewport(w, h);
}
pub fn rendition(&self) -> &str {
self.handle.rendition()
}
pub fn current_frame(&mut self) -> Option<DecodedFrame> {
self.video_frames.current_frame()
}
pub(crate) fn run_loop(
shutdown: &CancellationToken,
mut input_rx: mpsc::Receiver<hang::Frame>,
output_tx: mpsc::Sender<DecodedFrame>,
mut viewport_watcher: n0_watcher::Direct<(u32, u32)>,
mut decoder: impl VideoDecoder,
) -> Result<(), anyhow::Error> {
loop {
if shutdown.is_cancelled() {
break;
}
let Some(packet) = input_rx.blocking_recv() else {
break;
};
if viewport_watcher.update() {
let (w, h) = viewport_watcher.peek();
decoder.set_viewport(*w, *h);
}
let t = Instant::now();
decoder
.push_packet(packet)
.context("failed to push packet")?;
trace!(t=?t.elapsed(), "videodec: push_packet");
while let Some(frame) = decoder.pop_frame().context("failed to pop frame")? {
trace!(t=?t.elapsed(), "videodec: pop frame");
if output_tx.blocking_send(frame).is_err() {
break;
}
trace!(t=?t.elapsed(), "videodec: tx");
}
}
Ok(())
}
}
async fn forward_frames(mut track: hang::TrackConsumer, sender: mpsc::Sender<hang::Frame>) {
loop {
let frame = track.read_frame().await;
match frame {
Ok(Some(frame)) => {
if sender.send(frame).await.is_err() {
break;
}
}
Ok(None) => break,
Err(err) => {
warn!("failed to read frame: {err:?}");
break;
}
}
}
}
pub struct AvRemoteTrack {
pub broadcast: SubscribeBroadcast,
pub video: Option<WatchTrack>,
pub audio: Option<AudioTrack>,
}
impl AvRemoteTrack {
pub fn new<D: Decoders>(
broadcast: SubscribeBroadcast,
audio_out: impl AudioSink,
playback_config: PlaybackConfig,
) -> Result<Self> {
let audio = broadcast
.listen_with::<D::Audio>(playback_config.quality, audio_out)
.inspect_err(|err| tracing::warn!("no audio track: {err}"))
.ok();
let video = broadcast
.watch_with::<D::Video>(&playback_config.decode_config, playback_config.quality)
.inspect_err(|err| tracing::warn!("no video track: {err}"))
.ok();
Ok(Self {
broadcast,
audio,
video,
})
}
}

12
third_party/iroh-live/moq-media/src/util.rs vendored Normal file
View file

@ -0,0 +1,12 @@
/// Spawn a named OS thread and panic if spawning fails.
pub fn spawn_thread<F, T>(name: impl ToString, f: F) -> std::thread::JoinHandle<T>
where
F: FnOnce() -> T + Send + 'static,
T: Send + 'static,
{
let name_str = name.to_string();
std::thread::Builder::new()
.name(name_str.clone())
.spawn(f)
.expect(&format!("failed to spawn thread: {}", name_str))
}