every.channel: sanitized baseline

This commit is contained in:
every.channel 2026-02-15 16:17:27 -05:00
commit 897e556bea
No known key found for this signature in database
258 changed files with 74298 additions and 0 deletions

10
crates/ec-ts/Cargo.toml Normal file
View file

@ -0,0 +1,10 @@
[package]
name = "ec-ts"
version = "0.0.0"
edition.workspace = true
license.workspace = true
[dependencies]
anyhow.workspace = true
serde.workspace = true
serde-big-array = "0.5"

648
crates/ec-ts/src/lib.rs Normal file
View file

@ -0,0 +1,648 @@
//! Minimal MPEG-TS parsing for timing and table extraction.
use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};
use serde_big_array::BigArray;
use std::collections::HashMap;
use std::io::Read;
pub const TS_PACKET_SIZE: usize = 188;
pub const PID_ATSC_PSIP: u16 = 0x1FFB;
pub const PID_DVB_TDT_TOT: u16 = 0x0014;
const SYNC_BYTE: u8 = 0x47;
const TABLE_ID_ATSC_STT: u8 = 0xCD;
const TABLE_ID_DVB_TDT: u8 = 0x70;
const TABLE_ID_DVB_TOT: u8 = 0x73;
const GPS_EPOCH_TO_UNIX: i64 = 315964800;
const MJD_UNIX_EPOCH: i64 = 40587;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TsPacket {
#[serde(with = "BigArray")]
data: [u8; TS_PACKET_SIZE],
pub pid: u16,
pub payload_unit_start: bool,
pub continuity_counter: u8,
pub discontinuity: bool,
pub pcr_27mhz: Option<u64>,
payload_offset: usize,
payload_len: usize,
}
impl TsPacket {
pub fn payload(&self) -> &[u8] {
&self.data[self.payload_offset..self.payload_offset + self.payload_len]
}
pub fn as_bytes(&self) -> &[u8; TS_PACKET_SIZE] {
&self.data
}
}
pub struct TsReader<R> {
reader: R,
}
impl<R: Read> TsReader<R> {
pub fn new(reader: R) -> Self {
Self { reader }
}
pub fn read_packet(&mut self) -> Result<Option<TsPacket>> {
let mut data = [0u8; TS_PACKET_SIZE];
let mut read = 0usize;
while read < TS_PACKET_SIZE {
let n = self.reader.read(&mut data[read..])?;
if n == 0 {
if read == 0 {
return Ok(None);
}
return Err(anyhow!("truncated TS packet"));
}
read += n;
}
let packet = parse_packet(data)?;
Ok(Some(packet))
}
}
pub fn parse_packet(data: [u8; TS_PACKET_SIZE]) -> Result<TsPacket> {
if data[0] != SYNC_BYTE {
return Err(anyhow!("missing sync byte"));
}
let payload_unit_start = (data[1] & 0x40) != 0;
let pid = ((data[1] as u16 & 0x1F) << 8) | data[2] as u16;
let continuity_counter = data[3] & 0x0F;
let adaptation_control = (data[3] >> 4) & 0x03;
let has_adaptation = adaptation_control == 2 || adaptation_control == 3;
let has_payload = adaptation_control == 1 || adaptation_control == 3;
let mut offset = 4usize;
let mut discontinuity = false;
let mut pcr_27mhz = None;
if has_adaptation {
let length = data[offset] as usize;
offset += 1;
if offset + length > TS_PACKET_SIZE {
return Err(anyhow!("invalid adaptation field length"));
}
if length > 0 {
let flags = data[offset];
discontinuity = (flags & 0x80) != 0;
let pcr_flag = (flags & 0x10) != 0;
if pcr_flag && length >= 7 {
let pcr_bytes = &data[offset + 1..offset + 7];
pcr_27mhz = Some(parse_pcr_27mhz(pcr_bytes));
}
}
offset += length;
}
let payload_len = if has_payload && offset <= TS_PACKET_SIZE {
TS_PACKET_SIZE - offset
} else {
0
};
Ok(TsPacket {
data,
pid,
payload_unit_start,
continuity_counter,
discontinuity,
pcr_27mhz,
payload_offset: offset,
payload_len,
})
}
fn parse_pcr_27mhz(data: &[u8]) -> u64 {
let base = ((data[0] as u64) << 25)
| ((data[1] as u64) << 17)
| ((data[2] as u64) << 9)
| ((data[3] as u64) << 1)
| ((data[4] as u64) >> 7);
let ext = (((data[4] as u64) & 0x01) << 8) | data[5] as u64;
base * 300 + ext
}
pub fn parse_pts_90khz(packet: &TsPacket) -> Option<u64> {
if !packet.payload_unit_start {
return None;
}
let payload = packet.payload();
if payload.len() < 14 {
return None;
}
if payload[0] != 0 || payload[1] != 0 || payload[2] != 1 {
return None;
}
let flags = payload[7];
let pts_dts_flags = (flags >> 6) & 0x03;
if pts_dts_flags == 0 {
return None;
}
let header_length = payload[8] as usize;
let pts_start = 9usize;
if header_length < 5 || payload.len() < pts_start + 5 {
return None;
}
let b = &payload[pts_start..pts_start + 5];
let pts = ((b[0] as u64 & 0x0E) << 29)
| ((b[1] as u64) << 22)
| ((b[2] as u64 & 0xFE) << 14)
| ((b[3] as u64) << 7)
| ((b[4] as u64) >> 1);
Some(pts)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Section {
pub pid: u16,
pub table_id: u8,
pub data: Vec<u8>,
}
#[derive(Debug, Default)]
pub struct SectionAssembler {
buffers: HashMap<u16, SectionBuffer>,
}
#[derive(Debug)]
struct SectionBuffer {
expected_len: usize,
data: Vec<u8>,
}
impl SectionAssembler {
pub fn push_packet(&mut self, packet: &TsPacket) -> Vec<Section> {
let mut sections = Vec::new();
let payload = packet.payload();
if payload.is_empty() {
return sections;
}
if packet.payload_unit_start {
let pointer = payload[0] as usize;
if pointer + 1 > payload.len() {
return sections;
}
let mut idx = 1 + pointer;
while idx + 3 <= payload.len() {
let table_id = payload[idx];
let section_length =
(((payload[idx + 1] & 0x0F) as usize) << 8) | payload[idx + 2] as usize;
let total_len = 3 + section_length;
if idx + total_len <= payload.len() {
let data = payload[idx..idx + total_len].to_vec();
sections.push(Section {
pid: packet.pid,
table_id,
data,
});
idx += total_len;
} else {
let data = payload[idx..].to_vec();
self.buffers.insert(
packet.pid,
SectionBuffer {
expected_len: total_len,
data,
},
);
break;
}
}
} else if let Some(buffer) = self.buffers.get_mut(&packet.pid) {
buffer.data.extend_from_slice(payload);
if buffer.data.len() >= buffer.expected_len {
let data = buffer.data[..buffer.expected_len].to_vec();
let table_id = data[0];
sections.push(Section {
pid: packet.pid,
table_id,
data,
});
self.buffers.remove(&packet.pid);
}
}
sections
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum TimeSource {
AtscStt,
DvbTdt,
DvbTot,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BroadcastUtc {
pub unix_seconds: i64,
pub source: TimeSource,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TimeSyncUpdate {
pub pcr_27mhz: Option<u64>,
pub utc_unix_seconds: Option<i64>,
pub chunk_index: Option<u64>,
pub chunk_start_27mhz: Option<u64>,
pub utc_start_unix: Option<i64>,
pub synced: bool,
pub discontinuity: bool,
}
#[derive(Debug)]
pub struct TimeSyncEngine {
chunk_ticks: u64,
last_pcr: Option<u64>,
utc_offset_ticks: Option<i64>,
synced: bool,
last_chunk_index: Option<u64>,
}
impl TimeSyncEngine {
pub fn new(chunk_duration_ms: u64) -> Self {
let chunk_ticks = chunk_duration_ms * 27_000;
Self {
chunk_ticks,
last_pcr: None,
utc_offset_ticks: None,
synced: false,
last_chunk_index: None,
}
}
pub fn ingest_packet(
&mut self,
packet: &TsPacket,
assembler: &mut SectionAssembler,
) -> Vec<TimeSyncUpdate> {
let mut updates = Vec::new();
if packet.discontinuity {
self.last_pcr = None;
self.utc_offset_ticks = None;
self.synced = false;
self.last_chunk_index = None;
updates.push(TimeSyncUpdate {
pcr_27mhz: packet.pcr_27mhz,
utc_unix_seconds: None,
chunk_index: None,
chunk_start_27mhz: None,
utc_start_unix: None,
synced: false,
discontinuity: true,
});
}
for section in assembler.push_packet(packet) {
if let Some(utc) = parse_time_section(&section) {
if let Some(pcr) = self.last_pcr {
let utc_ticks = utc.unix_seconds.saturating_mul(27_000_000);
let offset = utc_ticks - pcr as i64;
self.utc_offset_ticks = Some(offset);
self.synced = true;
}
updates.push(TimeSyncUpdate {
pcr_27mhz: self.last_pcr,
utc_unix_seconds: Some(utc.unix_seconds),
chunk_index: self.current_chunk_index(),
chunk_start_27mhz: self.current_chunk_start_27mhz(),
utc_start_unix: self.current_chunk_utc_start(),
synced: self.synced,
discontinuity: false,
});
}
}
if let Some(pcr) = packet.pcr_27mhz {
self.last_pcr = Some(pcr);
let chunk_index = self.current_chunk_index();
if chunk_index != self.last_chunk_index {
self.last_chunk_index = chunk_index;
updates.push(TimeSyncUpdate {
pcr_27mhz: Some(pcr),
utc_unix_seconds: self.current_utc_seconds(),
chunk_index,
chunk_start_27mhz: self.current_chunk_start_27mhz(),
utc_start_unix: self.current_chunk_utc_start(),
synced: self.synced,
discontinuity: false,
});
}
}
updates
}
fn current_utc_seconds(&self) -> Option<i64> {
let pcr = self.last_pcr? as i64;
let offset = self.utc_offset_ticks?;
Some((pcr + offset) / 27_000_000)
}
fn current_chunk_index(&self) -> Option<u64> {
let pcr = self.last_pcr? as i128;
let offset = self.utc_offset_ticks.unwrap_or(0) as i128;
let t = pcr + offset;
if t < 0 {
return None;
}
Some((t as u128 / self.chunk_ticks as u128) as u64)
}
fn current_chunk_start_27mhz(&self) -> Option<u64> {
let chunk_index = self.current_chunk_index()? as i128;
let offset = self.utc_offset_ticks.unwrap_or(0) as i128;
let anchored = chunk_index * self.chunk_ticks as i128;
let pcr = anchored - offset;
if pcr < 0 {
return None;
}
Some(pcr as u64)
}
fn current_chunk_utc_start(&self) -> Option<i64> {
let _ = self.utc_offset_ticks?;
let chunk_index = self.current_chunk_index()? as i128;
let anchored = chunk_index * self.chunk_ticks as i128;
Some((anchored / 27_000_000) as i64)
}
}
pub fn parse_time_section(section: &Section) -> Option<BroadcastUtc> {
match section.table_id {
TABLE_ID_ATSC_STT if section.pid == PID_ATSC_PSIP => {
parse_atsc_stt(&section.data).map(|utc| BroadcastUtc {
unix_seconds: utc,
source: TimeSource::AtscStt,
})
}
TABLE_ID_DVB_TDT if section.pid == PID_DVB_TDT_TOT => {
parse_dvb_time(&section.data).map(|utc| BroadcastUtc {
unix_seconds: utc,
source: TimeSource::DvbTdt,
})
}
TABLE_ID_DVB_TOT if section.pid == PID_DVB_TDT_TOT => {
parse_dvb_time(&section.data).map(|utc| BroadcastUtc {
unix_seconds: utc,
source: TimeSource::DvbTot,
})
}
_ => None,
}
}
fn parse_atsc_stt(data: &[u8]) -> Option<i64> {
if data.len() < 3 + 1 + 4 + 1 {
return None;
}
let system_time = u32::from_be_bytes([data[4], data[5], data[6], data[7]]) as i64;
let gps_utc_offset = data[8] as i64;
let utc_since_1980 = system_time - gps_utc_offset;
Some(utc_since_1980 - GPS_EPOCH_TO_UNIX)
}
fn parse_dvb_time(data: &[u8]) -> Option<i64> {
if data.len() < 8 {
return None;
}
let mjd = u16::from_be_bytes([data[3], data[4]]);
let hour = bcd_to_dec(data[5])?;
let minute = bcd_to_dec(data[6])?;
let second = bcd_to_dec(data[7])?;
let days = mjd as i64 - MJD_UNIX_EPOCH;
Some(days * 86_400 + hour as i64 * 3_600 + minute as i64 * 60 + second as i64)
}
fn bcd_to_dec(value: u8) -> Option<u32> {
let high = (value >> 4) & 0x0F;
let low = value & 0x0F;
if high > 9 || low > 9 {
return None;
}
Some((high as u32) * 10 + low as u32)
}
#[cfg(test)]
mod tests {
use super::*;
fn build_ts_packet_with_adaptation_pcr(
pid: u16,
continuity_counter: u8,
pcr_27mhz: u64,
) -> [u8; TS_PACKET_SIZE] {
// Encode PCR into base (90kHz) and extension (27MHz remainder).
let base = pcr_27mhz / 300;
let ext = pcr_27mhz % 300;
let mut pcr = [0u8; 6];
pcr[0] = ((base >> 25) & 0xFF) as u8;
pcr[1] = ((base >> 17) & 0xFF) as u8;
pcr[2] = ((base >> 9) & 0xFF) as u8;
pcr[3] = ((base >> 1) & 0xFF) as u8;
pcr[4] = (((base & 0x1) << 7) as u8) | 0x7E | (((ext >> 8) & 0x1) as u8);
pcr[5] = (ext & 0xFF) as u8;
let mut data = [0u8; TS_PACKET_SIZE];
data[0] = SYNC_BYTE;
data[1] = ((pid >> 8) as u8) & 0x1F;
data[2] = (pid & 0xFF) as u8;
// adaptation only (no payload): adaptation_control=2
data[3] = (2 << 4) | (continuity_counter & 0x0F);
// adaptation length: 1 byte flags + 6 bytes PCR = 7
data[4] = 7;
// flags: PCR flag
data[5] = 0x10;
data[6..12].copy_from_slice(&pcr);
data
}
fn encode_pts_90khz(pts: u64) -> [u8; 5] {
let mut b = [0u8; 5];
b[0] = 0x20 | ((((pts >> 30) & 0x07) as u8) << 1) | 1;
b[1] = ((pts >> 22) & 0xFF) as u8;
b[2] = ((((pts >> 15) & 0x7F) as u8) << 1) | 1;
b[3] = ((pts >> 7) & 0xFF) as u8;
b[4] = (((pts & 0x7F) as u8) << 1) | 1;
b
}
#[test]
fn parse_packet_extracts_pid_and_payload() {
let pid = 0x0033u16;
let mut data = [0u8; TS_PACKET_SIZE];
data[0] = SYNC_BYTE;
data[1] = 0x40 | (((pid >> 8) as u8) & 0x1F); // payload_unit_start
data[2] = (pid & 0xFF) as u8;
data[3] = (1 << 4) | 0x0A; // payload only
data[4] = 0xAA;
let pkt = parse_packet(data).unwrap();
assert_eq!(pkt.pid, pid);
assert!(pkt.payload_unit_start);
assert_eq!(pkt.continuity_counter, 0x0A);
assert_eq!(pkt.payload()[0], 0xAA);
}
#[test]
fn parse_packet_rejects_bad_sync() {
let mut data = [0u8; TS_PACKET_SIZE];
data[0] = 0;
assert!(parse_packet(data).is_err());
}
#[test]
fn parse_packet_rejects_invalid_adaptation_length() {
let pid = 0x0011u16;
let mut data = [0u8; TS_PACKET_SIZE];
data[0] = SYNC_BYTE;
data[1] = ((pid >> 8) as u8) & 0x1F;
data[2] = (pid & 0xFF) as u8;
data[3] = (3 << 4) | 0x00; // adaptation + payload
data[4] = 250; // too large
assert!(parse_packet(data).is_err());
}
#[test]
fn parse_packet_reads_pcr_27mhz() {
let pcr = 54_000_123u64;
let data = build_ts_packet_with_adaptation_pcr(0x0100, 0, pcr);
let pkt = parse_packet(data).unwrap();
assert_eq!(pkt.pcr_27mhz, Some(pcr));
}
#[test]
fn parse_pts_extracts_expected_value() {
let pid = 0x0020u16;
let pts = 90_000u64 * 3;
let pts_bytes = encode_pts_90khz(pts);
let mut data = [0u8; TS_PACKET_SIZE];
data[0] = SYNC_BYTE;
data[1] = 0x40 | (((pid >> 8) as u8) & 0x1F);
data[2] = (pid & 0xFF) as u8;
data[3] = (1 << 4) | 0x00; // payload only
// Minimal PES header with PTS.
let payload = &mut data[4..];
payload[0..3].copy_from_slice(&[0, 0, 1]);
payload[3] = 0xE0;
payload[7] = 0x80; // pts_dts_flags = 2
payload[8] = 5; // header length
payload[9..14].copy_from_slice(&pts_bytes);
let pkt = parse_packet(data).unwrap();
let parsed = parse_pts_90khz(&pkt).unwrap();
assert_eq!(parsed, pts);
}
#[test]
fn section_assembler_reassembles_across_packets() {
let pid = 0x0014u16;
let table_id = TABLE_ID_DVB_TDT;
// A tiny "section" with declared length 10 (3 + 10 = 13 bytes total).
let total_len = 13usize;
let section_length = (total_len - 3) as u16;
let mut section = vec![0u8; total_len];
section[0] = table_id;
section[1] = 0x00 | (((section_length >> 8) as u8) & 0x0F);
section[2] = (section_length & 0xFF) as u8;
for i in 3..total_len {
section[i] = i as u8;
}
// Packet 1: payload is intentionally short (via a large adaptation field) so the assembler
// must buffer until packet 2 arrives.
let mut pkt1 = [0u8; TS_PACKET_SIZE];
pkt1[0] = SYNC_BYTE;
pkt1[1] = 0x40 | (((pid >> 8) as u8) & 0x1F);
pkt1[2] = (pid & 0xFF) as u8;
pkt1[3] = (3 << 4) | 0; // adaptation + payload
let payload_len_1 = 8usize; // 1 pointer + 7 bytes of section
let adaptation_len_1 = (TS_PACKET_SIZE - 5) - payload_len_1;
pkt1[4] = adaptation_len_1 as u8;
// adaptation flags at pkt1[5] left as 0; rest is stuffing 0.
let payload_start_1 = 4 + 1 + adaptation_len_1;
pkt1[payload_start_1] = 0; // pointer = 0
pkt1[payload_start_1 + 1..payload_start_1 + 1 + 7].copy_from_slice(&section[..7]);
let mut pkt2 = [0u8; TS_PACKET_SIZE];
pkt2[0] = SYNC_BYTE;
pkt2[1] = ((pid >> 8) as u8) & 0x1F;
pkt2[2] = (pid & 0xFF) as u8;
pkt2[3] = (1 << 4) | 1; // payload only
pkt2[4..4 + (total_len - 7)].copy_from_slice(&section[7..]);
let p1 = parse_packet(pkt1).unwrap();
let p2 = parse_packet(pkt2).unwrap();
let mut asm = SectionAssembler::default();
assert!(asm.push_packet(&p1).is_empty());
let out = asm.push_packet(&p2);
assert_eq!(out.len(), 1);
assert_eq!(out[0].pid, pid);
assert_eq!(out[0].table_id, table_id);
assert_eq!(out[0].data.len(), total_len);
assert_eq!(out[0].data[3], 3u8);
}
#[test]
fn parse_time_sections_for_dvb_and_atsc_epoch() {
// DVB TDT at UNIX epoch.
let mut dvb = vec![0u8; 8];
dvb[0] = TABLE_ID_DVB_TDT;
dvb[3] = 0x9E;
dvb[4] = 0x8B; // MJD 40587
dvb[5] = 0x00;
dvb[6] = 0x00;
dvb[7] = 0x00;
let section = Section {
pid: PID_DVB_TDT_TOT,
table_id: TABLE_ID_DVB_TDT,
data: dvb,
};
let utc = parse_time_section(&section).unwrap();
assert_eq!(utc.unix_seconds, 0);
// ATSC STT at UNIX epoch according to our parser logic.
let mut atsc = vec![0u8; 9];
atsc[0] = TABLE_ID_ATSC_STT;
let system_time = GPS_EPOCH_TO_UNIX as u32;
atsc[4..8].copy_from_slice(&system_time.to_be_bytes());
atsc[8] = 0;
let section = Section {
pid: PID_ATSC_PSIP,
table_id: TABLE_ID_ATSC_STT,
data: atsc,
};
let utc = parse_time_section(&section).unwrap();
assert_eq!(utc.unix_seconds, 0);
}
#[test]
fn time_sync_engine_emits_chunk_boundaries_from_pcr() {
let mut engine = TimeSyncEngine::new(1000);
let mut asm = SectionAssembler::default();
let p0 = parse_packet(build_ts_packet_with_adaptation_pcr(0x0100, 0, 0)).unwrap();
let p1 = parse_packet(build_ts_packet_with_adaptation_pcr(0x0100, 1, 27_000_000)).unwrap();
let u0 = engine.ingest_packet(&p0, &mut asm);
assert!(u0.iter().any(|u| u.chunk_index == Some(0)));
let u1 = engine.ingest_packet(&p1, &mut asm);
assert!(u1.iter().any(|u| u.chunk_index == Some(1)));
}
}