From 4b4b174751e88103a4f44e33aa188a7784633801 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Wed, 15 Jul 2026 00:58:41 +0800 Subject: [PATCH 1/5] fix(player): normalize TS timestamp continuity --- tools/devlab/README.md | 36 +++++ tools/devlab/devlab.py | 151 ++++++++++++++++-- .../src/playback-engine/demux/ts-demuxer.ts | 10 +- web-ui/src/playback-engine/hls/hls-source.ts | 4 +- .../src/playback-engine/remux/mp4-remuxer.ts | 73 +++++---- .../playback-engine/timeline/ts-continuity.ts | 45 ++++++ web-ui/src/playback-engine/worker/pipeline.ts | 70 ++++---- .../playback-engine/worker/segment-source.ts | 6 +- 8 files changed, 306 insertions(+), 89 deletions(-) create mode 100644 web-ui/src/playback-engine/timeline/ts-continuity.ts diff --git a/tools/devlab/README.md b/tools/devlab/README.md index 7ed77683..9938b940 100644 --- a/tools/devlab/README.md +++ b/tools/devlab/README.md @@ -10,9 +10,12 @@ needs to support: | HLS-fMP4 live | HTTP `.m3u8` + `.m4s` | `/http` | `h264-aac`, `hevc-aac` | | HLS alternate audio | HTTP master + split A/V playlists | `/http` | AAC / MP2 / AC-3 (TS), AAC (fMP4) | | HLS internal multi-audio | HTTP `.m3u8` + multi-PID `.ts` | `/http` | H.264 + AAC / MP2 / AC-3 | +| HLS timestamp continuity | HTTP VOD with crafted `.ts` PTS | `/http` | H.264 + MP2 | +| HLS live packet loss | HTTP `.m3u8` + damaged `.ts` | `/http` | H.264 + MP2 | | HTTP internal multi-audio | continuous multi-PID MPEG-TS | `/http` | H.264 + AAC / MP2 / AC-3 | | HLS catchup | HTTP HLS VOD (`playseek`) | `/http` | all of the above | | mpegts (RTSP) | RTSP TS live + catchup | `/rtsp` | `h264-mp2`, `hevc-aac` | +| mpegts continuity faults | RTSP catchup + RTP packet loss | `/rtsp` | H.264 + MP2 | | mpegts (multicast) | RTP multicast live | `/rtp` | `h264-mp2`, `hevc-ac3`, `hevc-eac3` | | mpegts (scan) | RTP multicast live | `/rtp` | 1080i / 1080p / 2160p (see below) | | external file | RTP multicast (looped) | `/rtp` | whatever the `.ts` file contains | @@ -38,6 +41,24 @@ tones as separate PIDs in one MPEG-TS program. One is segmented as HLS-TS and the other is a continuous HTTP TS response, covering both inputs accepted by the custom demux/transmux pipeline. +The **HLS-TS continuity overlap gap restart** VOD has four six-second TS +segments with deterministic timestamp starts: 0s, 5s, 13s, then 0s. This +creates a one-second overlap, a two-second gap, and finally a greater-than-ten +second timestamp restart. An `EXT-X-DISCONTINUITY` is also placed before the +gap segment to prove that HLS-TS follows the timestamp policy instead of +resetting its output timeline from playlist metadata. + +The two live packet-loss channels are deterministic fault injectors. HLS drops +40 complete TS packets from every third segment; RTSP skips six consecutive RTP +packets roughly every five seconds. The player should recover at the next valid +sample/keyframe and keep its output timeline continuous. + +The **mpegts catchup gap** channel terminates every requested RTSP catchup +window and advances the source timestamp offset by an extra two seconds for +each subsequent request. Together with the normal RTSP catchup channel's +one-second request overlap, these cover both overlap and gap handling across +plain MPEG-TS URL boundaries. + **HLS catchup** is a real HLS VOD: each `playseek` window returns an `index.m3u8` (`#EXT-X-PLAYLIST-TYPE:VOD`) listing fixed-duration `.ts` slices. The playlist is produced instantly; each slice is encoded lazily on first @@ -105,6 +126,21 @@ uv run tools/devlab/devlab.py # http://127.0.0.1:5140/player ``` +## Browser autoplay and audio prompts + +Browsers may show a **Click to Play** prompt because unmuted audio playback +requires a trusted user gesture. Browser automation must handle this prompt +according to what the scenario is testing: + +- If the test does not depend on audible output, mute the player and then + reload the page. The reload is required so playback starts in the muted + state and the automation can continue without the prompt. +- If the test validates audio or audible track switching, stop the automation, + show the player to the user, and ask the user to click **Click to Play**. + Resume only after the user confirms the interaction. Do not attempt to clear + the prompt with a programmatic click: it does not provide the trusted user + gesture required to enable audio playback. + Requires `ffmpeg` on PATH with `drawtext`, `libx264`, `libx265`, `aac`, and `mp2`. On macOS with Homebrew, install a full build: diff --git a/tools/devlab/devlab.py b/tools/devlab/devlab.py index 6dbb926e..2ed14d2c 100644 --- a/tools/devlab/devlab.py +++ b/tools/devlab/devlab.py @@ -87,6 +87,7 @@ # Tuple: (channel_label, profile, seg_type, live_key). HLS_CHANNELS = ( ("HLS-TS (h264-mp2)", "h264-mp2", "mpegts", "h264-mp2-ts"), + ("HLS-TS live packet loss", "h264-mp2", "mpegts", "h264-mp2-ts-loss"), ("HLS-TS (hevc-aac)", "hevc-aac", "mpegts", "hevc-aac-ts"), ("HLS-fMP4 (h264-aac)", "h264-aac", "fmp4", "h264-aac-fmp4"), ("HLS-fMP4 (hevc-aac)", "hevc-aac", "fmp4", "hevc-aac-fmp4"), @@ -121,11 +122,19 @@ class AudioRendition(NamedTuple): AudioRendition("audio-es", "Español", "es", "spa", 1320, "h264-ac3", "ac3", "192k"), ) +MPEG_TS_PACKET_SIZE = 188 +RTP_TS_PAYLOAD_SIZE = 7 * MPEG_TS_PACKET_SIZE + # --------------------------------------------------------------------------- # ffmpeg argument helpers # --------------------------------------------------------------------------- +def mpegts_timestamp_args(offset: int) -> list[str]: + """Return ffmpeg arguments for deterministic zero-delay MPEG-TS timestamps.""" + return ["-output_ts_offset", str(offset), "-muxdelay", "0", "-muxpreload", "0"] + + def resolve_font(path: str | None) -> str: if path: if os.path.isfile(path): @@ -319,16 +328,11 @@ def _parse_time_token(tok: str) -> int: return int(time.time()) -def parse_playseek_begin(playseek: str) -> int: - """Return the begin time of a playseek value as a UTC epoch (seconds).""" - return _parse_time_token(playseek.split("-")[0]) if playseek else int(time.time()) - - def parse_playseek_range(playseek: str) -> tuple[int, int]: """Return ``(begin, end)`` UTC epochs for a ``BEGIN-END`` playseek value. rtp2httpd forwards compact ``yyyyMMddHHmmss`` times (no internal dashes) so a - plain ``-`` split is safe. A missing/!invalid end defaults to begin + 60s. + plain ``-`` split is safe. A missing or invalid end defaults to begin + 60s. """ toks = [t for t in playseek.split("-") if t.strip()] if playseek else [] begin = _parse_time_token(toks[0]) if toks else int(time.time()) @@ -588,12 +592,65 @@ def segment_cmd(self, profile: str, begin: int, idx: int) -> list[str]: ] +class ContinuityHLS: + """Deterministic HLS-TS VOD with overlap, gap and a large timestamp restart.""" + + seg_dur = 6 + timestamp_offsets = (0, 5, 13, 0) + + def __init__(self, ffmpeg: str): + self.ffmpeg = ffmpeg + + def playlist(self, base: str) -> str: + lines = [ + "#EXTM3U", + "#EXT-X-VERSION:3", + f"#EXT-X-TARGETDURATION:{self.seg_dur}", + "#EXT-X-MEDIA-SEQUENCE:0", + "#EXT-X-PLAYLIST-TYPE:VOD", + ] + for idx in range(len(self.timestamp_offsets)): + if idx == 2: + # The TS path deliberately ignores this tag and follows the + # actual timestamp continuity policy instead. + lines.append("#EXT-X-DISCONTINUITY") + lines += [f"#EXTINF:{self.seg_dur}.000,", f"{base}/continuity-hls/{idx}.ts"] + lines.append("#EXT-X-ENDLIST") + return "\n".join(lines) + "\n" + + def segment_cmd(self, idx: int) -> list[str]: + offset = self.timestamp_offsets[idx] + return [ + self.ffmpeg, + "-hide_banner", + "-loglevel", + "error", + *lavfi_inputs(), + "-t", + str(self.seg_dur), + "-vf", + f"{live_filter('HLS continuity')},{_drawtext(f'SEG {idx} PTS {offset}s', 152, expansion=False)}", + *video_args("h264-mp2"), + *audio_args("h264-mp2"), + *mpegts_timestamp_args(offset), + "-f", + "mpegts", + "-", + ] + + # --------------------------------------------------------------------------- # HTTP origin server # --------------------------------------------------------------------------- -def make_http_handler(host: str, port: int, live_root: str, catchup: CatchupHLS) -> type[BaseHTTPRequestHandler]: +def make_http_handler( + host: str, + port: int, + live_root: str, + catchup: CatchupHLS, + continuity: ContinuityHLS, +) -> type[BaseHTTPRequestHandler]: base = f"http://{host}:{port}" class Handler(BaseHTTPRequestHandler): @@ -621,7 +678,18 @@ def _serve_file(self, path: str) -> None: else: ctype = "video/mp2t" with open(path, "rb") as fh: - self._send(200, ctype, fh.read()) + body = fh.read() + if "h264-mp2-ts-loss" in path and path.endswith(".ts"): + try: + segment_index = int(os.path.basename(path).removeprefix("seg_").removesuffix(".ts")) + except ValueError: + segment_index = 0 + if segment_index % 3 == 1: + packet_count = len(body) // MPEG_TS_PACKET_SIZE + drop_start = max(1, packet_count // 2) + drop_end = min(packet_count, drop_start + 40) + body = body[: drop_start * MPEG_TS_PACKET_SIZE] + body[drop_end * MPEG_TS_PACKET_SIZE :] + self._send(200, ctype, body) def _live(self, parts: list[str]) -> None: # /live// @@ -657,6 +725,26 @@ def _catchup_seg(self, profile: str, begin: int, idx: int) -> None: if proc.poll() is None: proc.terminate() + def _continuity_playlist(self) -> None: + text = continuity.playlist(base) + self._send(200, "application/vnd.apple.mpegurl", text.encode()) + + def _continuity_seg(self, idx: int) -> None: + if idx < 0 or idx >= len(continuity.timestamp_offsets): + self._send(404, "text/plain", b"not found") + return + proc = subprocess.run( + continuity.segment_cmd(idx), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=30, + check=False, + ) + if proc.returncode != 0: + self._send(500, "text/plain", proc.stderr[-4096:] or b"ffmpeg failed") + return + self._send(200, "video/mp2t", proc.stdout) + def _multi_ts(self) -> None: self.send_response(200) self.send_header("Content-Type", "video/mp2t") @@ -702,6 +790,10 @@ def do_GET(self) -> None: # noqa: N802 (http.server API) elif len(parts) == 4 and parts[0] == "catchup-seg": # /catchup-seg///.ts self._catchup_seg(parts[1], int(parts[2]), int(parts[3].removesuffix(".ts"))) + elif parts == ["continuity-hls", "index.m3u8"]: + self._continuity_playlist() + elif len(parts) == 2 and parts[0] == "continuity-hls": + self._continuity_seg(int(parts[1].removesuffix(".ts"))) elif parts == ["multi-audio.ts"]: self._multi_ts() else: @@ -731,6 +823,8 @@ def __init__(self, ffmpeg: str, host: str, port: int): self._sock: socket.socket | None = None self._thread: threading.Thread | None = None self._stop = threading.Event() + self._catchup_gap_requests = 0 + self._catchup_gap_lock = threading.Lock() def start(self) -> None: self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -765,10 +859,22 @@ def _profile_from_uri(uri: str) -> str: def _ffmpeg_cmd(self, uri: str) -> list[str]: profile = self._profile_from_uri(uri) - qs = parse_qs(urlparse(uri).query) + parsed_uri = urlparse(uri) + qs = parse_qs(parsed_uri.query) playseek = (qs.get("playseek") or qs.get("tvdr") or [""])[0] + duration_args: list[str] = [] + timestamp_args: list[str] = [] if playseek: - vf = catchup_filter(profile, parse_playseek_begin(playseek)) + begin, end = parse_playseek_range(playseek) + vf = catchup_filter(profile, begin) + duration_args = ["-t", str(end - begin)] + timeline_offset = begin % 86400 + if "/catchup-gap/" in parsed_uri.path: + with self._catchup_gap_lock: + request_index = self._catchup_gap_requests + self._catchup_gap_requests += 1 + timeline_offset += request_index * 2 + timestamp_args = mpegts_timestamp_args(timeline_offset) else: vf = live_filter(profile) return [ @@ -778,10 +884,12 @@ def _ffmpeg_cmd(self, uri: str) -> list[str]: "error", "-re", *lavfi_inputs(), + *duration_args, "-vf", vf, *video_args(profile), *audio_args(profile), + *timestamp_args, "-f", "mpegts", "-", @@ -850,7 +958,8 @@ def _pump(self, conn: socket.socket, uri: str) -> None: seq = 0 ts = 0 buf = b"" - payload = 1316 # 7 * 188-byte TS packets per RTP datagram + payload = RTP_TS_PAYLOAD_SIZE + inject_packet_loss = "/live-loss/" in urlparse(uri).path try: while not self._stop.is_set(): chunk = proc.stdout.read(payload) @@ -861,7 +970,9 @@ def _pump(self, conn: socket.socket, uri: str) -> None: pkt, buf = buf[:payload], buf[payload:] header = struct.pack("!BBHII", 0x80, 33, seq & 0xFFFF, ts & 0xFFFFFFFF, 0x0DEFACED) rtp = header + pkt - conn.sendall(b"\x24" + struct.pack("!BH", 0, len(rtp)) + rtp) + drop_for_fault = inject_packet_loss and 500 <= seq % 1000 < 506 + if not drop_for_fault: + conn.sendall(b"\x24" + struct.pack("!BH", 0, len(rtp)) + rtp) seq += 1 ts += 3600 except BrokenPipeError, ConnectionError, OSError: @@ -908,7 +1019,7 @@ def url(self) -> str: def _cmd(self) -> list[str]: common = [self.ffmpeg, "-hide_banner", "-loglevel", "error"] - out = f"{self.url()}?ttl=1&pkt_size=1316" + out = f"{self.url()}?ttl=1&pkt_size={RTP_TS_PAYLOAD_SIZE}" if self.ts_file: # Stream-copy the original bitstream so the exact codecs are relayed. return [*common, "-re", "-stream_loop", "-1", "-i", self.ts_file, "-c", "copy", "-f", "rtp_mpegts", out] @@ -976,6 +1087,10 @@ def build_services_m3u(http_hostport: str, rtsp_hostport: str, mcast_channels: l f'#EXTINF:-1 group-title="HLS" catchup="default" catchup-source="{src}",{label}', f"http://{http_hostport}/live/{key}/index.m3u8", ] + lines += [ + '#EXTINF:-1 group-title="HLS",HLS-TS continuity overlap gap restart', + f"http://{http_hostport}/continuity-hls/index.m3u8", + ] for label, _seg_type, key in ALT_AUDIO_HLS_CHANNELS: lines += [ f'#EXTINF:-1 group-title="HLS",{label}', @@ -1000,6 +1115,13 @@ def build_services_m3u(http_hostport: str, rtsp_hostport: str, mcast_channels: l extinf, f"rtsp://{rtsp_hostport}/live/{prof}", ] + gap_src = f"rtsp://{rtsp_hostport}/catchup-gap/h264-mp2?{tpl}" + lines += [ + f'#EXTINF:-1 group-title="mpegts (RTSP)" catchup="default" catchup-source="{gap_src}",mpegts catchup gap', + f"rtsp://{rtsp_hostport}/live/h264-mp2", + '#EXTINF:-1 group-title="mpegts (RTSP)",mpegts live packet loss', + f"rtsp://{rtsp_hostport}/live-loss/h264-mp2", + ] for group_title, name, rtp_url in mcast_channels: lines += [f'#EXTINF:-1 group-title="{group_title}",{name}', rtp_url] return "\n".join(lines) + "\n" @@ -1093,7 +1215,8 @@ def main() -> int: return 1 catchup = CatchupHLS(args.ffmpeg) - handler = make_http_handler(args.host, args.http_port, live_root, catchup) + continuity = ContinuityHLS(args.ffmpeg) + handler = make_http_handler(args.host, args.http_port, live_root, catchup, continuity) httpd = ThreadingHTTPServer((args.host, args.http_port), handler) threading.Thread(target=httpd.serve_forever, daemon=True).start() diff --git a/web-ui/src/playback-engine/demux/ts-demuxer.ts b/web-ui/src/playback-engine/demux/ts-demuxer.ts index cee7725e..f2220ab5 100644 --- a/web-ui/src/playback-engine/demux/ts-demuxer.ts +++ b/web-ui/src/playback-engine/demux/ts-demuxer.ts @@ -184,6 +184,12 @@ export interface TSAudioTrackInfo { export type OnAudioTracksCallback = (tracks: TSAudioTrackInfo[], selectedPid: number | undefined) => void; +export interface RawMp2AudioFrame { + codec: "mp2"; + data: Uint8Array; + pts?: number; +} + class TSDemuxer { private readonly TAG: string = "TSDemuxer"; @@ -194,7 +200,7 @@ class TSDemuxer { public onPcr: OnPcrCallback | null = null; public onAudioTracks: OnAudioTracksCallback | null = null; /** Software audio decode support (MP2) */ - public onRawAudioData: ((frame: { codec: "mp2"; data: Uint8Array; pts: number }) => void) | null = null; + public onRawAudioData: ((frame: RawMp2AudioFrame) => void) | null = null; private ts_packet_size_: number; private sync_offset_: number; @@ -1940,7 +1946,7 @@ class TSDemuxer { // Dispatch the raw payload for software decoding (the WASM decoder // splits it into frames and carries partial frames across payloads) - const pts_ms = (pts ?? 0) / this.timescale_; + const pts_ms = pts === undefined ? undefined : pts / this.timescale_; this.onRawAudioData({ codec: "mp2", data, pts: pts_ms }); // Don't push samples to audio track — the remuxer generates diff --git a/web-ui/src/playback-engine/hls/hls-source.ts b/web-ui/src/playback-engine/hls/hls-source.ts index 1804e394..b31906c0 100644 --- a/web-ui/src/playback-engine/hls/hls-source.ts +++ b/web-ui/src/playback-engine/hls/hls-source.ts @@ -145,7 +145,7 @@ export class HlsSource implements SegmentSource { const meta = this.segments[this.nextIndex++]; if (this.resetPending) { this.resetPending = false; - return { ...meta, resetRemuxer: true }; + return { ...meta, resetReason: "initial" }; } return meta; } @@ -248,7 +248,7 @@ export class HlsSource implements SegmentSource { url: seg.url, start: this.timelinePos, duration: seg.duration, - resetRemuxer: seg.discontinuity || skipped, + resetReason: seg.discontinuity || skipped ? "playlist-reset" : undefined, initUrl: seg.initUrl, }); this.timelinePos += seg.duration; diff --git a/web-ui/src/playback-engine/remux/mp4-remuxer.ts b/web-ui/src/playback-engine/remux/mp4-remuxer.ts index 06570dde..13e50fcb 100644 --- a/web-ui/src/playback-engine/remux/mp4-remuxer.ts +++ b/web-ui/src/playback-engine/remux/mp4-remuxer.ts @@ -1,3 +1,4 @@ +import { classifyTimestampOverlap, mapContinuousPcmRange } from "../timeline/ts-continuity"; import { isFirefox } from "../utils/browser"; import { IllegalStateException } from "../utils/exception"; import Log from "../utils/logger"; @@ -97,7 +98,7 @@ interface TrackTimingState { durationResidual: number; } -type PcmTimestampMapping = { action: "drop" } | { action: "emit"; time: number; trimStartMs: number }; +type PcmTimestampMapping = { action: "drop" } | { action: "emit"; time: number; trimStartFrames: number }; type InitSegmentCallback = (type: string, segment: InitSegment) => void; type MediaSegmentCallback = (type: string, segment: MediaSegment) => void; @@ -136,6 +137,7 @@ class MP4Remuxer { private _audioTiming: TrackTimingState; private _videoTiming: TrackTimingState; private _pcmTiming: TrackTimingState; + private _pcmNextDts: number | undefined; private _videoPresentationOffset: number | undefined; private _videoInitialPresentationOffset: number | undefined; private _videoInitialOutputTime: number | undefined; @@ -151,7 +153,6 @@ class MP4Remuxer { private _silentAudioMode: boolean; private _silentAudioLastDts: number | undefined; private _silentAudioDurationResidual: number; - private _tsSegmentContinuityNormalization: boolean; private _mediaSegmentBatchDurationMs: number; private _mediaSegmentBatchMaxBytes: number; private _audioMediaSegmentEmitted: boolean; @@ -173,6 +174,7 @@ class MP4Remuxer { this._audioTiming = this._createTrackTimingState(); this._videoTiming = this._createTrackTimingState(); this._pcmTiming = this._createTrackTimingState(); + this._pcmNextDts = undefined; this._videoPresentationOffset = undefined; this._videoInitialPresentationOffset = undefined; this._videoInitialOutputTime = undefined; @@ -189,7 +191,6 @@ class MP4Remuxer { this._silentAudioMode = false; this._silentAudioLastDts = undefined; this._silentAudioDurationResidual = 0; - this._tsSegmentContinuityNormalization = false; this._mediaSegmentBatchDurationMs = normalizeMediaBatchLimit( options.mediaSegmentBatchDurationMs, DEFAULT_MEDIA_SEGMENT_BATCH_DURATION_MS, @@ -209,12 +210,12 @@ class MP4Remuxer { this._silentAudioMode = false; this._silentAudioLastDts = undefined; this._silentAudioDurationResidual = 0; - this._tsSegmentContinuityNormalization = false; this._audioMediaSegmentEmitted = false; this._videoMediaSegmentEmitted = false; this._audioTiming = this._createTrackTimingState(); this._videoTiming = this._createTrackTimingState(); this._pcmTiming = this._createTrackTimingState(); + this._pcmNextDts = undefined; this._videoPresentationOffset = undefined; this._videoInitialPresentationOffset = undefined; this._videoInitialOutputTime = undefined; @@ -256,7 +257,6 @@ class MP4Remuxer { } insertDiscontinuity(): void { - this._audioNextDts = this._videoNextDts = undefined; this._silentAudioLastDts = undefined; this._silentAudioDurationResidual = 0; this._videoPresentationOffset = undefined; @@ -272,6 +272,7 @@ class MP4Remuxer { this._audioStashedLastSample = null; this._audioTiming = this._createTrackTimingState(); this._pcmTiming = this._createTrackTimingState(); + this._pcmNextDts = nextDtsMs; this._audioMeta = null; this._silentAudioMode = false; this._silentAudioLastDts = undefined; @@ -304,10 +305,6 @@ class MP4Remuxer { ]); } - setTsSegmentContinuityNormalization(enabled: boolean): void { - this._tsSegmentContinuityNormalization = enabled; - } - private _createTrackTimingState(): TrackTimingState { return { lastOriginalDts: undefined, @@ -359,20 +356,25 @@ class MP4Remuxer { timing: TrackTimingState, mdatBytes: number, ): number { - if (!this._tsSegmentContinuityNormalization) { + const lastOriginalDts = timing.lastOriginalDts; + if (lastOriginalDts === undefined) { return mdatBytes; } - const lastOriginalDts = timing.lastOriginalDts; - if (lastOriginalDts === undefined) { + const firstOriginalDts = samples[0].dts - this._dtsBase; + const decision = classifyTimestampOverlap(firstOriginalDts, lastOriginalDts); + if (decision === "restart") { + Log.w(this.TAG, `Keeping samples after ${Math.round(lastOriginalDts - firstOriginalDts)}ms timestamp restart`); + timing.lastOriginalDts = undefined; + timing.lastOriginalEndDts = undefined; + timing.durationResidual = 0; return mdatBytes; } + if (decision !== "skip") return mdatBytes; while (samples.length > 0) { const originalDts = samples[0].dts - this._dtsBase; - if (originalDts > lastOriginalDts) { - break; - } + if (originalDts > lastOriginalDts) break; const sample = samples.shift() as T; mdatBytes -= sample.length; @@ -640,17 +642,21 @@ class MP4Remuxer { return this._videoInitialOutputTime ?? 0; } - mapPcmTimestamp(ptsMs: number, durationMs: number): PcmTimestampMapping | undefined { + mapPcmTimestamp(ptsMs: number, frameCount: number, sampleRate: number): PcmTimestampMapping | undefined { if (!this._dtsBaseInited) { return undefined; } const originalTime = ptsMs - this._dtsBase; - const duration = Math.max(0, durationMs); + const totalFrames = Math.max(0, frameCount); + const duration = (totalFrames / sampleRate) * 1000; let outputTime: number; - let trimStartMs = 0; + let trimStartFrames = 0; - if (this._pcmTiming.lastOriginalEndDts === undefined || this._pcmTiming.lastOutputEndDts === undefined) { + if (this._pcmNextDts !== undefined) { + outputTime = this._pcmNextDts; + this._pcmNextDts = undefined; + } else if (this._pcmTiming.lastOriginalEndDts === undefined || this._pcmTiming.lastOutputEndDts === undefined) { if ( this._videoDtsBase !== Infinity && (this._videoInitialPresentationOffset === undefined || this._videoInitialOutputTime === undefined) @@ -666,17 +672,25 @@ class MP4Remuxer { return { action: "drop" }; } - trimStartMs = Math.max(0, presentationFloor - presentationStart); + trimStartFrames = Math.min( + totalFrames, + Math.ceil((Math.max(0, presentationFloor - presentationStart) * sampleRate) / 1000 - 1e-9), + ); outputTime = Math.max(presentationFloor, presentationStart); } else { - const distance = originalTime - this._pcmTiming.lastOriginalEndDts; - outputTime = this._pcmTiming.lastOutputEndDts + (distance > 0 ? 0 : distance); - if (distance > 0) { - Log.v(this.TAG, `PCM: bridging ${Math.round(distance)}ms timestamp hole`); - } + const mapping = mapContinuousPcmRange({ + originalStartMs: originalTime, + frameCount: totalFrames, + sampleRate, + lastOriginalEndMs: this._pcmTiming.lastOriginalEndDts, + lastOutputEndMs: this._pcmTiming.lastOutputEndDts, + }); + if (mapping.action === "drop") return mapping; + outputTime = mapping.outputStartMs; + trimStartFrames = mapping.trimStartFrames; } - const emittedDuration = duration - trimStartMs; + const emittedDuration = ((totalFrames - trimStartFrames) / sampleRate) * 1000; if (emittedDuration <= 0) { return { action: "drop" }; } @@ -684,7 +698,7 @@ class MP4Remuxer { this._pcmTiming.lastOriginalEndDts = originalTime + duration; this._pcmTiming.lastOutputEndDts = outputTime + emittedDuration; this._pcmTiming.lastOutputDuration = emittedDuration; - return { action: "emit", time: outputTime / 1000, trimStartMs }; + return { action: "emit", time: outputTime / 1000, trimStartFrames }; } flushStashedSamples(): void { @@ -974,8 +988,9 @@ class MP4Remuxer { const isKeyframe = sample.isKeyframe; const dts = nextOutputDts; - const originalPts = originalDts + sample.cts; - const correctedPtsBase = this._tsSegmentContinuityNormalization ? originalPts - dtsCorrection : originalPts; + // Apply the exact DTS translation to PTS as well. This keeps CTS stable + // when a TS segment gap or overlap is collapsed onto the output timeline. + const correctedPtsBase = dts + sample.cts; if (this._videoPresentationOffset === undefined) { this._videoPresentationOffset = correctedPtsBase - dts; if (this._videoInitialPresentationOffset === undefined) { diff --git a/web-ui/src/playback-engine/timeline/ts-continuity.ts b/web-ui/src/playback-engine/timeline/ts-continuity.ts new file mode 100644 index 00000000..20ac707c --- /dev/null +++ b/web-ui/src/playback-engine/timeline/ts-continuity.ts @@ -0,0 +1,45 @@ +const OVERLAP_SKIP_LIMIT_MS = 10_000; + +type TimestampOverlapDecision = "none" | "skip" | "restart"; + +export function classifyTimestampOverlap(currentMs: number, previousMs: number): TimestampOverlapDecision { + if (currentMs > previousMs) return "none"; + return previousMs - currentMs > OVERLAP_SKIP_LIMIT_MS ? "restart" : "skip"; +} + +interface ContinuousPcmRange { + originalStartMs: number; + frameCount: number; + sampleRate: number; + lastOriginalEndMs: number; + lastOutputEndMs: number; +} + +type ContinuousPcmMapping = + | { action: "drop" } + | { + action: "emit"; + outputStartMs: number; + trimStartFrames: number; + }; + +/** Map one decoded PCM range after the first range has established both clocks. */ +export function mapContinuousPcmRange(input: ContinuousPcmRange): ContinuousPcmMapping { + const { originalStartMs, frameCount, sampleRate, lastOriginalEndMs, lastOutputEndMs } = input; + const overlapMs = lastOriginalEndMs - originalStartMs; + const overlapDecision = classifyTimestampOverlap(originalStartMs, lastOriginalEndMs); + let trimStartFrames = 0; + + if (overlapDecision === "skip" && overlapMs > 0) { + // Ceil so the first retained frame is never earlier than the source range + // already emitted. The epsilon avoids rounding an exact frame boundary up. + trimStartFrames = Math.min(frameCount, Math.ceil((overlapMs * sampleRate) / 1000 - 1e-9)); + if (trimStartFrames >= frameCount) return { action: "drop" }; + } + + return { + action: "emit", + outputStartMs: lastOutputEndMs, + trimStartFrames, + }; +} diff --git a/web-ui/src/playback-engine/worker/pipeline.ts b/web-ui/src/playback-engine/worker/pipeline.ts index b0d12268..8e8ad650 100644 --- a/web-ui/src/playback-engine/worker/pipeline.ts +++ b/web-ui/src/playback-engine/worker/pipeline.ts @@ -1,7 +1,7 @@ import type { PlayerConfig } from "../config"; import { createDefaultConfig } from "../config"; import { WorkerAudioDecoder } from "../decoder/worker-audio-decoder"; -import TSDemuxer, { type TSAudioTrackInfo } from "../demux/ts-demuxer"; +import TSDemuxer, { type RawMp2AudioFrame, type TSAudioTrackInfo } from "../demux/ts-demuxer"; import { type DemuxErrorDetail, type LoaderErrorDetail, PlayerErrors } from "../errors"; import { containsMoov, @@ -171,6 +171,7 @@ class Pipeline { private _fmp4Timescales = new Map(); private _fmp4TimestampOffsetWarningLogged = false; private _timelineAnchorSent = false; + private _resetAudioParserAtNextTsBoundary = false; // --- player-facing media metadata --- private _mediaInfo: PlayerMediaInfo = {}; @@ -199,7 +200,6 @@ class Pipeline { channels: number; sampleRate: number; ptsMs: number; - durationMs: number; }> = []; /** Incremented on audio timing resets to invalidate decode callbacks queued before the reset. */ private _audioGen = 0; @@ -559,6 +559,7 @@ class Pipeline { this._fmp4Timescales = new Map(); this._fmp4TimestampOffsetWarningLogged = false; this._timelineAnchorSent = false; + this._resetAudioParserAtNextTsBoundary = false; this._paused = false; this._sourceMode = "static-ts-list"; this._resumeGate?.(); @@ -625,7 +626,7 @@ class Pipeline { } try { - if (meta.resetRemuxer) { + if (this._shouldResetTransmux(meta)) { const initSegmentChanged = meta.initUrl !== undefined && meta.initUrl !== this._lastInitUrl; this._resetTransmux(meta.start, !this._fmp4Mode || initSegmentChanged); } @@ -687,8 +688,12 @@ class Pipeline { this._resetAudioDecodeState(); } + private _shouldResetTransmux(meta: SegmentMeta): boolean { + return meta.resetReason === "initial" || (meta.resetReason === "playlist-reset" && this._fmp4Mode); + } + private _shouldAnchorSegment(meta: SegmentMeta): boolean { - return meta.resetRemuxer || !this._hlsSource; + return this._shouldResetTransmux(meta) || !this._hlsSource; } private _finishTsInputBoundary(): void { @@ -696,7 +701,6 @@ class Pipeline { // remux batch is not mixed with the previous segment's tail. this._demuxer?.flushSegmentBoundary(); this._remuxer?.flushStashedSamples(); - this._resetAudioDecodeState(); } private _prepareContinuousLiveTsRestart(meta: SegmentMeta, ioctl: FetchLoader): void { @@ -705,6 +709,8 @@ class Pipeline { } this._finishTsInputBoundary(); + this._resetAudioParserAtNextTsBoundary = true; + this._resetAudioDecodeState(); this._fmp4Mode = false; this._fmp4Chunks = []; ioctl.onDataArrival = (data, byteStart) => this._onInputChunk(meta, data, byteStart); @@ -819,11 +825,12 @@ class Pipeline { const canReuseHls = this._hlsSource !== null && !shouldAnchor && this._demuxer !== null && this._remuxer !== null; const canReuseTsInputBoundary = this._sourceMode !== "hls" && this._demuxer !== null && this._remuxer !== null; const canReuse = canReuseHls || canReuseTsInputBoundary; + const resetAudioParserState = this._resetAudioParserAtNextTsBoundary; + this._resetAudioParserAtNextTsBoundary = false; if (canReuse) { this._demuxer?.resetSegmentBoundary(probeData as ConstructorParameters[0], { - resetAudioParserState: canReuseTsInputBoundary, + resetAudioParserState, }); - this._remuxer?.setTsSegmentContinuityNormalization(canReuseTsInputBoundary); return; } @@ -848,8 +855,6 @@ class Pipeline { } this._pendingDtsOffsetMs = 0; } - this._remuxer.setTsSegmentContinuityNormalization(false); - demuxer.onError = this._onDemuxException.bind(this); demuxer.timestampBase = 0; demuxer.onTrackDiscontinuity = (track) => { @@ -1051,10 +1056,7 @@ class Pipeline { // ---- MP2 software audio decode ---- - private _handleRawAudioFrame( - frame: { codec: "mp2"; data: Uint8Array; pts: number }, - needsOwnTimelineBase: boolean, - ): void { + private _handleRawAudioFrame(frame: RawMp2AudioFrame, needsOwnTimelineBase: boolean): void { // Lazily create WorkerAudioDecoder on first raw audio frame if (!this._workerAudioDecoder) { const mp2Url = this._config.wasmDecoders.mp2; @@ -1071,27 +1073,23 @@ class Pipeline { const result = this._workerAudioDecoder.decode(frame.data); if (!result) return; - // PTS extrapolation: anchor on the PES PTS, advance by decoded sample count. - // This gives every decoded chunk a jitter-free timestamp even when frames - // straddle PES boundaries or a PES contains multiple frames. Re-anchor only - // on genuine discontinuities (> 100ms deviation). + // A real PES PTS always wins so the continuity mapper can observe source + // gaps and overlaps. Sample-count extrapolation is used only when the PES + // genuinely carries no timestamp. const sr = result.sampleRate; const carriedSamples = Math.min(Math.max(0, result.samplesBeforeInput), result.samplesPerChannel); - const decodedStartPts = frame.pts - (carriedSamples / sr) * 1000; + const decodedStartPts = frame.pts === undefined ? undefined : frame.pts - (carriedSamples / sr) * 1000; if (this._audioAnchorPtsMs === null || this._audioSampleRate !== sr) { + if (decodedStartPts === undefined) { + Log.w(this.TAG, "Dropping decoded MP2 audio until the first timestamped PES"); + return; + } this._audioAnchorPtsMs = decodedStartPts; this._audioSamplesSinceAnchor = 0; this._audioSampleRate = sr; - } else { - const extrapolatedMs = this._audioAnchorPtsMs + (this._audioSamplesSinceAnchor / sr) * 1000; - if (Math.abs(decodedStartPts - extrapolatedMs) > 100) { - Log.v( - this.TAG, - `Audio PTS discontinuity: decoded=${decodedStartPts.toFixed(1)}ms extrap=${extrapolatedMs.toFixed(1)}ms`, - ); - this._audioAnchorPtsMs = decodedStartPts; - this._audioSamplesSinceAnchor = 0; - } + } else if (decodedStartPts !== undefined) { + this._audioAnchorPtsMs = decodedStartPts; + this._audioSamplesSinceAnchor = 0; } const ptsMs = this._audioAnchorPtsMs + (this._audioSamplesSinceAnchor / sr) * 1000; this._audioSamplesSinceAnchor += result.samplesPerChannel; @@ -1112,9 +1110,8 @@ class Pipeline { ptsMs: number, needsOwnTimelineBase: boolean, ): void { - const durationMs = (Math.floor(pcm.length / channels) / sampleRate) * 1000; if (needsOwnTimelineBase) this._remuxer?.ensureAudioTimestampBase(ptsMs); - this._pendingPcm.push({ pcm, channels, sampleRate, ptsMs, durationMs }); + this._pendingPcm.push({ pcm, channels, sampleRate, ptsMs }); if (this._remuxer?.getTimestampBase() === undefined) { // Bound the queue: ~25s of audio at one payload per ~72ms is plenty @@ -1129,7 +1126,8 @@ class Pipeline { for (let i = 0; i < pending.length; i++) { const item = pending[i]; - const mapping = this._remuxer?.mapPcmTimestamp(item.ptsMs, item.durationMs); + const totalFrames = Math.floor(item.pcm.length / item.channels); + const mapping = this._remuxer?.mapPcmTimestamp(item.ptsMs, totalFrames, item.sampleRate); if (mapping === undefined) { this._pendingPcm.push(...pending.slice(i)); if (this._pendingPcm.length > 512) { @@ -1142,15 +1140,11 @@ class Pipeline { } let pcm = item.pcm; - if (mapping.trimStartMs > 0) { - const cutFrames = Math.round((mapping.trimStartMs / 1000) * item.sampleRate); - const totalFrames = Math.floor(pcm.length / item.channels); - if (cutFrames >= totalFrames) { + if (mapping.trimStartFrames > 0) { + if (mapping.trimStartFrames >= totalFrames) { continue; } - if (cutFrames > 0) { - pcm = pcm.slice(cutFrames * item.channels); - } + pcm = pcm.slice(mapping.trimStartFrames * item.channels); } if (this._forcedTrack === "audio") { this._remuxer?.remuxSilentAudioRange( diff --git a/web-ui/src/playback-engine/worker/segment-source.ts b/web-ui/src/playback-engine/worker/segment-source.ts index 49266e93..19f007c1 100644 --- a/web-ui/src/playback-engine/worker/segment-source.ts +++ b/web-ui/src/playback-engine/worker/segment-source.ts @@ -6,8 +6,8 @@ export interface SegmentMeta { start: number; /** Segment duration in seconds (0 if unknown / live). */ duration: number; - /** Destroy and recreate the remuxer before this segment, re-anchoring output at `start` (HLS discontinuity / seek). */ - resetRemuxer: boolean; + /** Why the transmuxer may need resetting. MPEG-TS ignores playlist resets; fMP4 keeps its period reset. */ + resetReason?: "initial" | "playlist-reset"; /** fMP4 initialization segment URL (HLS EXT-X-MAP), if any. */ initUrl?: string; } @@ -30,7 +30,6 @@ export class StaticSegmentSource implements SegmentSource { url: seg.url, start, duration: seg.duration ?? 0, - resetRemuxer: false, }; start += seg.duration ?? 0; return meta; @@ -56,7 +55,6 @@ export class ContinuousLiveSegmentSource implements SegmentSource { url: segment.url, start: 0, duration: 0, - resetRemuxer: false, }; } From abc1b0a6d409776eedd738bbb9a5f0a672faa99e Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Wed, 15 Jul 2026 02:50:51 +0800 Subject: [PATCH 2/5] fix(player): stabilize multi-audio switching --- web-ui/src/components/player/video-player.tsx | 1 + .../playback-engine/audio/pcm-audio-player.ts | 144 +++++++++++----- .../src/playback-engine/demux/ts-demuxer.ts | 17 +- .../mse/playback-controller.ts | 51 +++++- .../src/playback-engine/remux/mp4-remuxer.ts | 35 ++++ web-ui/src/playback-engine/worker/messages.ts | 19 ++- web-ui/src/playback-engine/worker/pipeline.ts | 3 + .../playback-engine/worker/transmux-worker.ts | 160 ++++++++++++------ 8 files changed, 317 insertions(+), 113 deletions(-) diff --git a/web-ui/src/components/player/video-player.tsx b/web-ui/src/components/player/video-player.tsx index 5a0aee59..6c83096b 100644 --- a/web-ui/src/components/player/video-player.tsx +++ b/web-ui/src/components/player/video-player.tsx @@ -1012,6 +1012,7 @@ function VideoPlayerComponent({ const track = state.tracks.find((candidate) => candidate.id === trackId); const activePlayer = getActivePlayer(); if (!track || !channel || !activePlayer || trackId === state.selectedTrackId) return; + setWarning(null); pendingAudioPreferenceRef.current = { channelId: channel.id, trackId, preferenceKey: track.preferenceKey }; activePlayer.selectAudioTrack(trackId); }); diff --git a/web-ui/src/playback-engine/audio/pcm-audio-player.ts b/web-ui/src/playback-engine/audio/pcm-audio-player.ts index b2540ac2..190a84e6 100644 --- a/web-ui/src/playback-engine/audio/pcm-audio-player.ts +++ b/web-ui/src/playback-engine/audio/pcm-audio-player.ts @@ -33,7 +33,7 @@ const TAG = "PCMAudioPlayer"; * Page-level one-shot autoplay gate for Web Audio. * Set when playback has started (any codec), the click-to-resume prompt was * already shown, or AudioContext.resume() succeeded — suppresses re-prompting - * on later channel switches that create a new AudioContext. + * when a reused AudioContext resumes after a channel switch. */ let playbackUnlocked = false; @@ -129,6 +129,9 @@ export class PCMAudioPlayer { private config: PlayerConfig; private context: AudioContext | null = null; private gainNode: GainNode | null = null; + /** Invalidates asynchronous work started for an earlier PCM timeline. */ + private pcmGeneration = 0; + private destroyed = false; private volume: number = 1.0; private muted: boolean = false; @@ -215,7 +218,7 @@ export class PCMAudioPlayer { } async init(): Promise { - if (this.context) { + if (this.destroyed || this.context) { return; } @@ -245,6 +248,7 @@ export class PCMAudioPlayer { this.updateGain(); this.context.onstatechange = () => { + if (this.destroyed) return; const state = this.context?.state as string | undefined; Log.v(TAG, `AudioContext state changed to: ${state}`); @@ -300,6 +304,7 @@ export class PCMAudioPlayer { } attachVideo(video: HTMLVideoElement): void { + if (this.destroyed) return; this.videoElement = video; // Sync initial volume state @@ -321,6 +326,15 @@ export class PCMAudioPlayer { this.completeRecovery("timeupdate"); return; } + // During a MediaSource replacement, events from the old and new + // pipelines can interleave. A late `waiting` after the new stream's + // `playing`/`canplay` leaves isBuffering stuck even though the video + // clock is advancing. Treat timeupdate as the authoritative recovery + // signal so PCM scheduling cannot remain permanently disabled. + if (this.isBuffering) { + this.maybeExitBuffering(); + if (!this.isBuffering) return; + } this.controlAndPump(); }; this.boundOnRateChange = () => { @@ -405,6 +419,7 @@ export class PCMAudioPlayer { /** `time` is normalized to the MSE timeline (same space as video.currentTime). */ feed(samples: Float32Array, channels: number, sampleRate: number, time: number): void { + if (this.destroyed) return; if (!this.context || !this.gainNode) { Log.w(TAG, "AudioContext not initialized, dropping audio"); return; @@ -435,33 +450,10 @@ export class PCMAudioPlayer { * track may use either Web Audio or MSE, so this must be applied even when no * new PCM chunks are expected. */ - replaceFrom(time: number): void { - const trimChunks = (chunks: AudioChunk[]): AudioChunk[] => { - const kept: AudioChunk[] = []; - for (const chunk of chunks) { - if (chunk.time >= time) break; - if (chunk.endTime <= time) { - kept.push(chunk); - continue; - } - - const frames = Math.max(0, Math.floor((time - chunk.time) * chunk.sampleRate)); - if (frames > 0) { - const endTime = chunk.time + frames / chunk.sampleRate; - kept.push({ - ...chunk, - samples: chunk.samples.subarray(0, frames * chunk.channels), - duration: endTime - chunk.time, - endTime, - }); - } - break; - } - return kept; - }; - - this.audioBuffer = trimChunks(this.audioBuffer); - this.pendingChunks = trimChunks(this.pendingChunks); + replaceFrom(time: number, expectsPCM: boolean): void { + if (this.destroyed) return; + this.pcmGeneration++; + this.restoreStartupPlaybackRate(); const ctx = this.context; const now = ctx?.currentTime ?? 0; @@ -484,15 +476,15 @@ export class PCMAudioPlayer { this.scheduledSpans = remaining; this.nextStartTime = remaining.length > 0 ? remaining[remaining.length - 1].ctxEnd : 0; - this.inputCursor = time; - this.stretcher?.reset(); - this.resetDriftState(); + this.resetPCMTimeline(time, expectsPCM ? "waiting" : "disabled"); + Log.v(TAG, `Audio track switch re-anchored timeline at ${time.toFixed(3)}s; target=${expectsPCM ? "PCM" : "MSE"}`); } // ==================== Stretcher ==================== /** Returns the stretcher if ready for this chunk's format, else kicks off (re)creation. */ private ensureStretcher(chunk: AudioChunk): Stretcher | null { + if (this.destroyed) return null; if (this.stretcher) { if (this.stretcher.sampleRate === chunk.sampleRate && this.stretcher.channels === chunk.channels) { return this.stretcher; @@ -513,6 +505,7 @@ export class PCMAudioPlayer { return null; } this.stretcherLoading = true; + const generation = this.pcmGeneration; const wasmUrl = this.config.wasmDecoders.mp2; const promise = wasmUrl @@ -522,8 +515,11 @@ export class PCMAudioPlayer { promise .then((stretcher) => { this.stretcherLoading = false; - if (!this.context) { + if (this.destroyed || generation !== this.pcmGeneration || !this.context) { stretcher.destroy(); + if (!this.destroyed && generation !== this.pcmGeneration) { + this.pump(); + } return; } this.stretcher = stretcher; @@ -531,6 +527,11 @@ export class PCMAudioPlayer { }) .catch((err) => { this.stretcherLoading = false; + if (this.destroyed) return; + if (generation !== this.pcmGeneration) { + this.pump(); + return; + } this.stretcherFailed = true; this.pendingChunks = []; Log.e(TAG, `WASM stretcher unavailable; cannot play software-decoded audio: ${err}`); @@ -549,6 +550,7 @@ export class PCMAudioPlayer { * back-to-back on the AudioContext clock. */ private pump(): void { + if (this.destroyed) return; const ctx = this.context; if (!ctx || !this.gainNode || !this.canScheduleAudio()) { return; @@ -570,6 +572,7 @@ export class PCMAudioPlayer { void ctx .resume() .then(() => { + if (this.destroyed) return; if ((ctx.state as string) === "running") { playbackUnlocked = true; this.pump(); @@ -578,6 +581,7 @@ export class PCMAudioPlayer { } }) .catch(() => { + if (this.destroyed) return; if (!playbackUnlocked) { this.notifyAutoplayBlocked(); } @@ -1004,12 +1008,22 @@ export class PCMAudioPlayer { } } if (chainRestart) { + // A previous stream reset may have left a short fade automation queued + // while the AudioContext was suspended. Normalize the persistent graph + // before starting a new chain so stale automation cannot keep it silent. + this.updateGain(); this.applyFadeIn(buffer); } const source = ctx.createBufferSource(); source.buffer = buffer; source.connect(this.gainNode); + source.onended = () => { + try { + source.disconnect(); + } catch (_e) {} + source.onended = null; + }; source.start(this.nextStartTime); this.scheduledSpans.push({ @@ -1398,12 +1412,15 @@ export class PCMAudioPlayer { // ==================== Playback Control ==================== async play(): Promise { + if (this.destroyed) return; if (this.context && this.context.state !== "running") { try { await this.context.resume(); + if (this.destroyed) return; playbackUnlocked = true; // onstatechange drives the rest (background free-run or recovery anchor) } catch (_e) { + if (this.destroyed) return; Log.w(TAG, "Failed to resume AudioContext on play()"); } } else { @@ -1421,6 +1438,7 @@ export class PCMAudioPlayer { try { await this.audioElement.play(); } catch (_e) { + if (this.destroyed) return; Log.w(TAG, "Failed to play audio element"); } } @@ -1442,26 +1460,44 @@ export class PCMAudioPlayer { } } - stop(): void { - this.cancelChain(); - this.restoreStartupPlaybackRate(); - + private resetPCMTimeline(anchor: number | null, startupSyncState: StartupSyncState): void { this.pendingChunks = []; this.audioBuffer = []; - this.startupSyncState = "waiting"; + this.startupSyncState = startupSyncState; this.startupSyncWaitStartedAt = null; this.startupWaitLogged = false; - - this.isBuffering = false; - this.isSeeking = false; - this.inputCursor = null; + this.inputCursor = anchor; + this.stretcherBase = anchor ?? 0; + this.outputStreamCursor = anchor ?? 0; this.stretcher?.reset(); this.stretcherFailed = false; this.softSyncUntil = 0; + this.driftLogCounter = 0; this.resetDriftState(); + } + + private resetStreamState(smooth: boolean): void { + this.restoreStartupPlaybackRate(); + this.cancelChain(smooth); + this.resetPCMTimeline(null, "waiting"); + + this.isBuffering = false; + this.isSeeking = false; + if (this.recoveryTimer) { + clearTimeout(this.recoveryTimer); + this.recoveryTimer = null; + } this.setSyncState(document.visibilityState === "hidden" ? "background" : "active"); } + /** Reset all stream-specific state while retaining the AudioContext and fixed output graph. */ + resetForNewStream(): void { + if (this.destroyed) return; + this.pcmGeneration++; + this.resetStreamState(true); + Log.v(TAG, "Reset for new stream; reusing AudioContext and audio graph"); + } + setVolume(volume: number): void { this.volume = Math.max(0, Math.min(1, volume)); this.updateGain(); @@ -1484,7 +1520,15 @@ export class PCMAudioPlayer { } async destroy(): Promise { - this.stop(); + if (this.destroyed) return; + this.destroyed = true; + this.pcmGeneration++; + this.onSuspended = null; + this.onResyncFailed = null; + this.onStartupSyncFailed = null; + this.onStartupRateControlChange = null; + + this.resetStreamState(false); this.detachVideo(); if (this.stretcher) { @@ -1508,10 +1552,16 @@ export class PCMAudioPlayer { this.gainNode = null; } - if (this.context) { - this.context.onstatechange = null; - await this.context.close(); - this.context = null; + const context = this.context; + this.context = null; + if (context) { + context.onstatechange = null; + try { + await context.close(); + Log.v(TAG, "AudioContext closed"); + } catch (error) { + Log.w(TAG, "Failed to close AudioContext", error); + } } } } diff --git a/web-ui/src/playback-engine/demux/ts-demuxer.ts b/web-ui/src/playback-engine/demux/ts-demuxer.ts index f2220ab5..4f5733e2 100644 --- a/web-ui/src/playback-engine/demux/ts-demuxer.ts +++ b/web-ui/src/playback-engine/demux/ts-demuxer.ts @@ -59,6 +59,8 @@ type CommonPidKey = keyof PMT["common_pids"]; type TSDemuxerOptions = { waitForInitialVideoKeyframe?: boolean; selectedAudioPid?: number; + /** Source codecs that should occupy MSE with silent AAC instead of their unsupported payload. */ + silentAudioCodecs?: readonly string[]; }; type TSSegmentBoundaryOptions = { resetAudioParserState?: boolean; @@ -251,6 +253,7 @@ class TSDemuxer { private loas_previous_frame: LOASAACFrame | null = null; private soft_decode_audio_codec_: "mp2" | null = null; + private readonly silent_audio_codecs_: Set; private audio_drop_until_sync_ = false; private drop_video_until_keyframe_ = true; private selected_audio_pid_: number | undefined; @@ -283,6 +286,7 @@ class TSDemuxer { this.ts_packet_size_ = probe_data.ts_packet_size as number; this.sync_offset_ = probe_data.sync_offset as number; this.selected_audio_pid_ = options.selectedAudioPid; + this.silent_audio_codecs_ = new Set(options.silentAudioCodecs?.map((codec) => codec.toLowerCase())); if (options.waitForInitialVideoKeyframe === false) { this.drop_video_until_keyframe_ = false; this.video_output_started_ = true; @@ -2187,9 +2191,14 @@ class TSDemuxer { Log.v(this.TAG, `Generated first AudioSpecificConfig for mimeType: ${meta.codec}`); } - // When software decoding, send a fake AAC-LC metadata with silentAudioMode - // so the remuxer creates an AAC SourceBuffer and generates silent frames - if (this.soft_decode_audio_codec_) { + const silentSourceCodec = + this.soft_decode_audio_codec_ ?? + (this.silent_audio_codecs_.has(this.audio_metadata_.codec) ? this.audio_metadata_.codec : null); + + // Software-decoded or browser-unsupported audio occupies MSE with silent + // AAC so the video clock keeps advancing while the real payload is handled + // elsewhere (PCM) or intentionally discarded (unsupported codec). + if (silentSourceCodec) { const sampleRate = (meta.audioSampleRate as number) || 48000; const channelCount = (meta.channelCount as number) || 2; // Find sampling frequency index for AAC config @@ -2204,7 +2213,7 @@ class TSDemuxer { channelCount: channelCount, codec: "mp4a.40.2", originalCodec: "mp4a.40.2", - sourceCodec: this.soft_decode_audio_codec_, + sourceCodec: silentSourceCodec, config: [(2 << 3) | ((freqIdx & 0x0f) >>> 1), ((freqIdx & 0x01) << 7) | ((channelCount & 0x0f) << 3)], refSampleDuration: (1024 / sampleRate) * 1000, silentAudioMode: true, diff --git a/web-ui/src/playback-engine/mse/playback-controller.ts b/web-ui/src/playback-engine/mse/playback-controller.ts index 240cd303..e379070d 100644 --- a/web-ui/src/playback-engine/mse/playback-controller.ts +++ b/web-ui/src/playback-engine/mse/playback-controller.ts @@ -27,6 +27,18 @@ const HLS_URL_RE = /\.m3u8?($|\?)/i; type SourceMode = "continuous-live-ts" | "static-ts-list" | "hls"; +const OPTIONAL_TS_AUDIO_CODECS = ["ac-3", "ec-3"] as const; + +function unsupportedTsAudioCodecs(): string[] { + const selfRecord = self as unknown as Record; + const mediaSourceClass = (selfRecord.MediaSource ?? selfRecord.ManagedMediaSource) as + | { isTypeSupported?: (type: string) => boolean } + | undefined; + return OPTIONAL_TS_AUDIO_CODECS.filter( + (codec) => mediaSourceClass?.isTypeSupported?.(`audio/mp4;codecs="${codec}"`) === false, + ); +} + export function createMSEPlaybackController( video: HTMLVideoElement, config: PlayerConfig, @@ -53,6 +65,7 @@ export function createMSEPlaybackController( let pcmPlayer: PCMAudioPlayer | null = null; let pcmPlayerInitPromise: Promise | null = null; let startupRateControlActive = false; + const unsupportedAudioCodecs = unsupportedTsAudioCodecs(); function ensurePCMPlayer(): PCMAudioPlayer { if (!pcmPlayer) { @@ -86,11 +99,17 @@ export function createMSEPlaybackController( } function destroyPCMPlayer(): void { - if (pcmPlayer) { - pcmPlayer.destroy(); - pcmPlayer = null; - pcmPlayerInitPromise = null; + const player = pcmPlayer; + pcmPlayer = null; + pcmPlayerInitPromise = null; + startupRateControlActive = false; + if (player) { + void player.destroy(); } + } + + function resetPCMPlayerForNewStream(): void { + pcmPlayer?.resetForNewStream(); startupRateControlActive = false; } @@ -170,6 +189,16 @@ export function createMSEPlaybackController( case "audio-tracks": impl.onAudioTracksChange?.(msg.state); break; + case "audio-codec-unsupported": + impl.onError?.({ + category: "media", + detail: PlayerErrors.CODEC_UNSUPPORTED, + info: `Unsupported MIME type: audio/mp4;codecs="${msg.codec}"`, + code: 0, + track: "audio", + codec: msg.codec, + }); + break; case "audio-track-switch": { const switchGeneration = mseGeneration; const switchWorker = worker; @@ -177,7 +206,8 @@ export function createMSEPlaybackController( if (switchGeneration !== mseGeneration || switchWorker !== worker) return; if (success) { mse?.replaceAudioFrom(msg.fromTime); - pcmPlayer?.replaceFrom(msg.pcmFromTime ?? msg.fromTime); + const expectsPCM = msg.audioMode === "pcm"; + pcmPlayer?.replaceFrom(expectsPCM ? msg.pcmFromTime : msg.fromTime, expectsPCM); } switchWorker?.postMessage({ type: "audio-track-switch-result", @@ -390,7 +420,14 @@ export function createMSEPlaybackController( function loadInWorker(segments: PlayerSegment[], options?: PlaybackLoadOptions): void { const w = ensureWorker(); if (!workerInitialized) { - const initCmd: WorkerCommand = { type: "init", segments, options, config, gen: mseGeneration }; + const initCmd: WorkerCommand = { + type: "init", + segments, + options, + config, + unsupportedTsAudioCodecs: unsupportedAudioCodecs, + gen: mseGeneration, + }; w.postMessage(initCmd); const startCmd: WorkerCommand = { type: "start" }; w.postMessage(startCmd); @@ -498,6 +535,7 @@ export function createMSEPlaybackController( loadSegments(segments: PlayerSegment[], options?: PlaybackLoadOptions) { mseGeneration++; + resetPCMPlayerForNewStream(); pendingInits = []; pendingLoad = { segments, options }; sourceMode = inferSourceMode(segments); @@ -508,7 +546,6 @@ export function createMSEPlaybackController( mse.destroy(); mse = null; } - destroyPCMPlayer(); initMSE(); initLiveHelpers(); }, diff --git a/web-ui/src/playback-engine/remux/mp4-remuxer.ts b/web-ui/src/playback-engine/remux/mp4-remuxer.ts index 13e50fcb..6aa506b5 100644 --- a/web-ui/src/playback-engine/remux/mp4-remuxer.ts +++ b/web-ui/src/playback-engine/remux/mp4-remuxer.ts @@ -418,6 +418,17 @@ class MP4Remuxer { this._calculateDtsBase(audioTrack, videoTrack); } + if (this._silentAudioMode && audioTrack?.samples.length) { + if (videoTrack?.samples.length) { + this._remuxVideo(videoTrack, force); + } else { + this._remuxSilentAudioTrack(audioTrack); + } + audioTrack.samples = []; + audioTrack.length = 0; + return; + } + const batchReady = force || isMediaBatchReady(audioTrack, videoTrack, this._mediaSegmentBatchDurationMs, this._mediaSegmentBatchMaxBytes); @@ -442,6 +453,30 @@ class MP4Remuxer { } } + /** Replace unsupported compressed audio samples with equivalent silent AAC occupancy. */ + private _remuxSilentAudioTrack(track: DemuxTrack): void { + const samples = track.samples as Array<{ dts?: number }>; + const firstDts = samples[0]?.dts; + const lastDts = samples[samples.length - 1]?.dts; + if (firstDts === undefined || lastDts === undefined) return; + + const frameDuration = this._audioMeta?.refSampleDuration ?? 32; + const start = firstDts - this._dtsBase; + const end = lastDts - this._dtsBase + frameDuration; + this._generateSilentAudio([ + { + dts: start, + pts: start, + cts: 0, + unit: new Uint8Array(), + size: 0, + duration: Math.max(1, end - start), + originalDts: firstDts, + flags: { isLeading: 0, dependsOn: 1, isDependedOn: 0, hasRedundancy: 0 }, + }, + ]); + } + /** * Generate silent AAC audio frames synced to video timestamps. * Used in soft decode mode to keep MSE audio track active (prevents diff --git a/web-ui/src/playback-engine/worker/messages.ts b/web-ui/src/playback-engine/worker/messages.ts index 78c91d37..3d44a341 100644 --- a/web-ui/src/playback-engine/worker/messages.ts +++ b/web-ui/src/playback-engine/worker/messages.ts @@ -2,8 +2,22 @@ import type { PlayerConfig } from "../config"; import type { DemuxErrorDetail, LoaderErrorDetail } from "../errors"; import type { PlaybackLoadOptions, PlayerAudioTrackState, PlayerMediaInfo, PlayerSegment } from "../types"; +type AudioTrackSwitchEvent = { + type: "audio-track-switch"; + trackId: string; + fromTime: number; + gen: number; +} & ({ audioMode: "mse" } | { audioMode: "pcm"; pcmFromTime: number }); + export type WorkerCommand = - | { type: "init"; segments: PlayerSegment[]; options?: PlaybackLoadOptions; config: PlayerConfig; gen: number } + | { + type: "init"; + segments: PlayerSegment[]; + options?: PlaybackLoadOptions; + config: PlayerConfig; + unsupportedTsAudioCodecs: string[]; + gen: number; + } | { type: "start" } | { type: "load-segments"; segments: PlayerSegment[]; options?: PlaybackLoadOptions; gen: number } | { type: "select-audio-track"; trackId: string; currentTime: number } @@ -30,7 +44,8 @@ export type WorkerEvent = } | { type: "hls-info"; live: boolean; totalDuration: number; gen: number } | { type: "audio-tracks"; state: PlayerAudioTrackState; gen: number } - | { type: "audio-track-switch"; trackId: string; fromTime: number; pcmFromTime?: number; gen: number } + | { type: "audio-codec-unsupported"; codec: string; gen: number } + | AudioTrackSwitchEvent | { type: "pcm-audio-data"; pcm: ArrayBuffer; diff --git a/web-ui/src/playback-engine/worker/pipeline.ts b/web-ui/src/playback-engine/worker/pipeline.ts index 8e8ad650..38d17294 100644 --- a/web-ui/src/playback-engine/worker/pipeline.ts +++ b/web-ui/src/playback-engine/worker/pipeline.ts @@ -68,6 +68,8 @@ export interface PipelineOptions extends HlsSourceOptions { forcedTrack?: "video" | "audio"; /** Select one elementary audio PID from an MPEG-TS program. */ selectedTsAudioPid?: number; + /** TS source codecs that should be remuxed as silent AAC for this browser. */ + silentTsAudioCodecs?: readonly string[]; /** Preserve the primary rendition's raw timestamp mapping in an alternate audio pipeline. */ mediaTimelineAnchor?: MediaTimelineAnchor; } @@ -840,6 +842,7 @@ class Pipeline { const demuxer = new TSDemuxer(probeData as ConstructorParameters[0], { waitForInitialVideoKeyframe: shouldAnchor || !this._demuxer || !this._remuxer, selectedAudioPid: this._options.selectedTsAudioPid, + silentAudioCodecs: this._options.silentTsAudioCodecs, }); this._demuxer = demuxer; diff --git a/web-ui/src/playback-engine/worker/transmux-worker.ts b/web-ui/src/playback-engine/worker/transmux-worker.ts index 9d989587..7f90ee12 100644 --- a/web-ui/src/playback-engine/worker/transmux-worker.ts +++ b/web-ui/src/playback-engine/worker/transmux-worker.ts @@ -17,10 +17,19 @@ type OutputEvent = | { kind: "init"; track: "video" | "audio"; data: ArrayBuffer; codec: string; container: string } | { kind: "media"; track: "video" | "audio"; data: ArrayBuffer; timestampOffset?: number }; +interface PendingAudioSwitchOutput { + queued: OutputEvent[]; + pcm: Array<{ pcm: Float32Array; channels: number; sampleRate: number; time: number }>; + audioMode: "pending" | "mse" | "pcm"; + mediaReady: boolean; + prepared: boolean; +} + let primaryPipeline: Pipeline | null = null; let audioPipeline: Pipeline | null = null; let pendingAudioPipeline: Pipeline | null = null; let config: PlayerConfig | null = null; +let unsupportedTsAudioCodecs = new Set(); let gen = 0; let started = false; @@ -36,15 +45,10 @@ let mediaTimelineAnchor: MediaTimelineAnchor | undefined; let pendingInitialAudioInfo: HlsInfo | undefined; let completePendingAudioSwitch: ((trackId: string, success: boolean, currentTime: number) => void) | null = null; -interface PendingInternalAudioSwitch { +interface PendingInternalAudioSwitch extends PendingAudioSwitchOutput { trackId: string; oldTrackId: string; oldPid: number; - queued: OutputEvent[]; - pcm: Array<{ pcm: Float32Array; channels: number; sampleRate: number; time: number }>; - audioMode: "pending" | "mse" | "pcm"; - mediaReady: boolean; - prepared: boolean; fromTime: number; } @@ -116,6 +120,60 @@ function postPCMAudio(pcm: Float32Array, channels: number, sampleRate: number, t post({ type: "pcm-audio-data", pcm: buffer, channels, sampleRate, time, gen }, [buffer]); } +function createPendingAudioSwitchOutput(): PendingAudioSwitchOutput { + return { + queued: [], + pcm: [], + audioMode: "pending", + mediaReady: false, + prepared: false, + }; +} + +function audioModeForTsCodec(codec: string): "mse" | "pcm" { + return codec.toLowerCase() === "mp2" ? "pcm" : "mse"; +} + +function reportUnsupportedTsAudioCodec(codec: string): void { + if (unsupportedTsAudioCodecs.has(codec.toLowerCase())) { + post({ type: "audio-codec-unsupported", codec, gen }); + } +} + +function preparePendingAudioSwitch(pending: PendingAudioSwitchOutput, trackId: string, fromTime: number): void { + if ( + pending.prepared || + !pending.mediaReady || + pending.audioMode === "pending" || + (pending.audioMode === "pcm" && pending.pcm.length === 0) + ) + return; + + pending.prepared = true; + postOutputEvents(pending.queued, "init"); + if (pending.audioMode === "pcm") { + post({ + type: "audio-track-switch", + trackId, + fromTime, + audioMode: "pcm", + pcmFromTime: pending.pcm[0].time, + gen, + }); + } else { + post({ type: "audio-track-switch", trackId, fromTime, audioMode: "mse", gen }); + } +} + +function commitPendingAudioSwitch(pending: PendingAudioSwitchOutput): void { + postOutputEvents(pending.queued, "media"); + for (const item of pending.pcm) { + postPCMAudio(item.pcm, item.channels, item.sampleRate, item.time); + } + pending.queued.length = 0; + pending.pcm.length = 0; +} + function flushInitialOutput(force = false): void { if (!initialOutputGated) return; if (!force && (!primaryInitSeen || !audioInitSeen)) return; @@ -235,7 +293,9 @@ function createAudioPipeline( onHlsInfo() {}, onTimelineAnchor() {}, onTsAudioTracks() {}, - onTsAudioSourceCodec() {}, + onTsAudioSourceCodec(codec) { + reportUnsupportedTsAudioCodec(codec); + }, onMediaInfo(info) { audioMediaInfo = info; emitMergedMediaInfo(); @@ -244,7 +304,11 @@ function createAudioPipeline( postPCMAudio(pcm, channels, sampleRate, time); }, }; - return new Pipeline([{ url, duration: 0 }], config, callbacks, { forcedTrack: "audio", ...options }); + return new Pipeline([{ url, duration: 0 }], config, callbacks, { + forcedTrack: "audio", + silentTsAudioCodecs: [...unsupportedTsAudioCodecs], + ...options, + }); } function startInitialAudio(info: HlsInfo, anchor: MediaTimelineAnchor): void { @@ -323,23 +387,7 @@ function handlePrimaryTsAudioTracks(tracks: TSAudioTrackInfo[], selectedPid: num function prepareInternalAudioSwitch(): void { const pending = pendingInternalAudioSwitch; - if ( - !pending || - pending.prepared || - !pending.mediaReady || - pending.audioMode === "pending" || - (pending.audioMode === "pcm" && pending.pcm.length === 0) - ) - return; - pending.prepared = true; - postOutputEvents(pending.queued, "init"); - post({ - type: "audio-track-switch", - trackId: pending.trackId, - fromTime: pending.fromTime, - pcmFromTime: pending.audioMode === "pcm" ? pending.pcm[0].time : pending.fromTime, - gen, - }); + if (pending) preparePendingAudioSwitch(pending, pending.trackId, pending.fromTime); } function selectInternalAudioTrack(trackId: string, pid: number, currentTime: number): void { @@ -350,14 +398,10 @@ function selectInternalAudioTrack(trackId: string, pid: number, currentTime: num pendingAudioTrackId = trackId; pendingInternalAudioSwitch = { + ...createPendingAudioSwitchOutput(), trackId, oldTrackId, oldPid, - queued: [], - pcm: [], - audioMode: "pending", - mediaReady: false, - prepared: false, fromTime: currentTime, }; emitAudioTrackState(); @@ -377,10 +421,7 @@ function selectInternalAudioTrack(trackId: string, pid: number, currentTime: num selectedAudioTrackId = pending.trackId; emitAudioTrackState(); - postOutputEvents(pending.queued, "media"); - for (const item of pending.pcm) { - postPCMAudio(item.pcm, item.channels, item.sampleRate, item.time); - } + commitPendingAudioSwitch(pending); }; if (!primaryPipeline.selectTsAudioPid(pid, currentTime)) { @@ -432,9 +473,11 @@ function createPrimaryPipeline(segments: PlayerSegment[], loadOptions: PlaybackL onTsAudioTracks: handlePrimaryTsAudioTracks, onTsAudioSourceCodec(codec) { const pending = pendingInternalAudioSwitch; - if (!pending) return; - pending.audioMode = codec.toLowerCase() === "mp2" ? "pcm" : "mse"; - prepareInternalAudioSwitch(); + reportUnsupportedTsAudioCodec(codec); + if (pending) { + pending.audioMode = audioModeForTsCodec(codec); + prepareInternalAudioSwitch(); + } }, onMediaInfo(info) { primaryMediaInfo = info; @@ -443,6 +486,7 @@ function createPrimaryPipeline(segments: PlayerSegment[], loadOptions: PlaybackL onPCMAudioData(pcm, channels, sampleRate, time) { if (pendingInternalAudioSwitch) { pendingInternalAudioSwitch.pcm.push({ pcm, channels, sampleRate, time }); + pendingInternalAudioSwitch.mediaReady = true; prepareInternalAudioSwitch(); return; } @@ -452,6 +496,7 @@ function createPrimaryPipeline(segments: PlayerSegment[], loadOptions: PlaybackL return new Pipeline(segments, config, callbacks, { preferredAudioTrackKey: loadOptions?.preferredAudioTrackKey, selectedTsAudioPid: preferredTsAudioPid(loadOptions?.preferredAudioTrackKey), + silentTsAudioCodecs: [...unsupportedTsAudioCodecs], }); } @@ -504,11 +549,9 @@ function selectAudioTrack(trackId: string, currentTime: number): void { pendingAudioTrackId = trackId; emitAudioTrackState(); - const queued: OutputEvent[] = []; + const pendingOutput = createPendingAudioSwitchOutput(); let candidate: Pipeline; let committed = false; - let prepared = false; - let hasMedia = false; let candidateComplete = false; let switchBoundary = currentTime; let candidateMediaInfo: PlayerMediaInfo = {}; @@ -528,10 +571,8 @@ function selectAudioTrack(trackId: string, currentTime: number): void { }; const prepare = () => { - if (prepared || pendingAudioPipeline !== candidate) return; - prepared = true; - postOutputEvents(queued, "init"); - post({ type: "audio-track-switch", trackId, fromTime: switchBoundary, gen }); + if (pendingAudioPipeline !== candidate) return; + preparePendingAudioSwitch(pendingOutput, trackId, switchBoundary); }; completePendingAudioSwitch = (resultTrackId, success) => { @@ -554,27 +595,28 @@ function selectAudioTrack(trackId: string, currentTime: number): void { audioMediaInfo = candidateMediaInfo; emitMergedMediaInfo(); emitAudioTrackState(); - postOutputEvents(queued, "media"); - queued.length = 0; + commitPendingAudioSwitch(pendingOutput); maybeComplete(); }; const callbacks: PipelineCallbacks = { onInitSegment(type, initSegment) { if (committed) postOutput(outputFromInit(type, initSegment)); - else queued.push(outputFromInit(type, initSegment)); + else pendingOutput.queued.push(outputFromInit(type, initSegment)); }, onMediaSegment(type, mediaSegment) { if (committed) postOutput(outputFromMedia(type, mediaSegment)); else { - hasMedia = true; - queued.push(outputFromMedia(type, mediaSegment)); + pendingOutput.queued.push(outputFromMedia(type, mediaSegment)); + pendingOutput.mediaReady = true; + if (pendingOutput.audioMode === "pending") pendingOutput.audioMode = "mse"; prepare(); } }, onLoadingComplete() { candidateComplete = true; - if (!hasMedia) fail("demux", "FormatError", { msg: "Selected audio rendition contains no media" }); + if (!pendingOutput.mediaReady) + fail("demux", "FormatError", { msg: "Selected audio rendition contains no media" }); else if (committed) { audioComplete = true; maybeComplete(); @@ -591,7 +633,11 @@ function selectAudioTrack(trackId: string, currentTime: number): void { }, onTimelineAnchor() {}, onTsAudioTracks() {}, - onTsAudioSourceCodec() {}, + onTsAudioSourceCodec(codec) { + reportUnsupportedTsAudioCodec(codec); + pendingOutput.audioMode = audioModeForTsCodec(codec); + prepare(); + }, onMediaInfo(info) { candidateMediaInfo = info; if (committed) { @@ -600,13 +646,20 @@ function selectAudioTrack(trackId: string, currentTime: number): void { } }, onPCMAudioData(pcm, channels, sampleRate, time) { - if (!committed) return; - postPCMAudio(pcm, channels, sampleRate, time); + if (committed) { + postPCMAudio(pcm, channels, sampleRate, time); + return; + } + pendingOutput.pcm.push({ pcm, channels, sampleRate, time }); + pendingOutput.audioMode = "pcm"; + pendingOutput.mediaReady = true; + prepare(); }, }; candidate = new Pipeline([{ url, duration: 0 }], config, callbacks, { forcedTrack: "audio", + silentTsAudioCodecs: [...unsupportedTsAudioCodecs], liveTimelineOffset: currentTime, startTime: currentTime, programDateTimeAnchor: @@ -624,6 +677,7 @@ self.addEventListener("message", (e: MessageEvent) => { case "init": gen = cmd.gen; config = cmd.config; + unsupportedTsAudioCodecs = new Set(cmd.unsupportedTsAudioCodecs.map((codec) => codec.toLowerCase())); Log.setLogLevel(cmd.config.logLevel); load(cmd.segments, cmd.options); break; From 6fc0deae5582699ac7003085273a45d91563fc56 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Wed, 15 Jul 2026 03:08:47 +0800 Subject: [PATCH 3/5] fix(player): preserve unsupported audio init batching --- .../src/playback-engine/mse/playback-controller.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/web-ui/src/playback-engine/mse/playback-controller.ts b/web-ui/src/playback-engine/mse/playback-controller.ts index e379070d..0a010f1f 100644 --- a/web-ui/src/playback-engine/mse/playback-controller.ts +++ b/web-ui/src/playback-engine/mse/playback-controller.ts @@ -146,9 +146,15 @@ export function createMSEPlaybackController( if (msg.type === "destroyed") return; // Discard stale messages from a previous load generation if (msg.gen !== mseGeneration) return; - // media-info can be posted right before an init-segment; flushing on it would - // split the batched init appends (see comment above pendingInits) - if (msg.type !== "init-segment" && msg.type !== "media-info" && msg.type !== "audio-track-switch") { + // Metadata notifications can be posted right before an init-segment; flushing + // on them would split the batched init appends (see comment above pendingInits). + // In particular, an unsupported TS codec warning precedes its silent AAC init. + if ( + msg.type !== "init-segment" && + msg.type !== "media-info" && + msg.type !== "audio-codec-unsupported" && + msg.type !== "audio-track-switch" + ) { void flushPendingInits(); } switch (msg.type) { From 6d14e3ad06b5766ef0529cece6a351b5e17d3144 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Wed, 15 Jul 2026 03:47:06 +0800 Subject: [PATCH 4/5] fix(player): preserve HEVC composition timing --- .../src/playback-engine/remux/mp4-remuxer.ts | 24 ++++++++++++++++--- web-ui/src/playback-engine/worker/pipeline.ts | 1 + 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/web-ui/src/playback-engine/remux/mp4-remuxer.ts b/web-ui/src/playback-engine/remux/mp4-remuxer.ts index 6aa506b5..5028e88a 100644 --- a/web-ui/src/playback-engine/remux/mp4-remuxer.ts +++ b/web-ui/src/playback-engine/remux/mp4-remuxer.ts @@ -138,6 +138,8 @@ class MP4Remuxer { private _videoTiming: TrackTimingState; private _pcmTiming: TrackTimingState; private _pcmNextDts: number | undefined; + private _videoPtsDtsOffset: number; + private _videoPtsOffsetUpdatePending: boolean; private _videoPresentationOffset: number | undefined; private _videoInitialPresentationOffset: number | undefined; private _videoInitialOutputTime: number | undefined; @@ -175,6 +177,8 @@ class MP4Remuxer { this._videoTiming = this._createTrackTimingState(); this._pcmTiming = this._createTrackTimingState(); this._pcmNextDts = undefined; + this._videoPtsDtsOffset = 0; + this._videoPtsOffsetUpdatePending = false; this._videoPresentationOffset = undefined; this._videoInitialPresentationOffset = undefined; this._videoInitialOutputTime = undefined; @@ -216,6 +220,8 @@ class MP4Remuxer { this._videoTiming = this._createTrackTimingState(); this._pcmTiming = this._createTrackTimingState(); this._pcmNextDts = undefined; + this._videoPtsDtsOffset = 0; + this._videoPtsOffsetUpdatePending = false; this._videoPresentationOffset = undefined; this._videoInitialPresentationOffset = undefined; this._videoInitialOutputTime = undefined; @@ -259,6 +265,7 @@ class MP4Remuxer { insertDiscontinuity(): void { this._silentAudioLastDts = undefined; this._silentAudioDurationResidual = 0; + this._videoPtsOffsetUpdatePending = true; this._videoPresentationOffset = undefined; // Resume quickly after a seek or stream discontinuity instead of waiting // for a complete steady-state batch. @@ -266,6 +273,11 @@ class MP4Remuxer { this._videoMediaSegmentEmitted = false; } + /** Re-anchor video PTS mapping on the first sample after a TS input boundary. */ + markTsInputBoundary(): void { + this._videoPtsOffsetUpdatePending = true; + } + /** Reset only audio state when switching an elementary TS audio PID. */ resetAudioTrackForSwitch(nextDtsMs?: number): void { this._audioNextDts = nextDtsMs; @@ -1014,6 +1026,10 @@ class MP4Remuxer { const mp4Samples: MP4Sample[] = []; let nextOutputDts = firstSampleOriginalDts - dtsCorrection; + if (this._videoPtsOffsetUpdatePending) { + this._videoPtsDtsOffset = firstSampleOriginalDts - nextOutputDts; + this._videoPtsOffsetUpdatePending = false; + } const presentationFloor = this._videoInitialOutputTime ?? this._dtsBaseOffset; // Correct dts for each sample, and calculate sample duration. Then output to mp4Samples @@ -1023,9 +1039,11 @@ class MP4Remuxer { const isKeyframe = sample.isKeyframe; const dts = nextOutputDts; - // Apply the exact DTS translation to PTS as well. This keeps CTS stable - // when a TS segment gap or overlap is collapsed onto the output timeline. - const correctedPtsBase = dts + sample.cts; + // Preserve source composition timing within a continuous TS input. Only + // explicit input boundaries update the cumulative PTS-to-DTS mapping; + // applying each fragment's small DTS correction to PTS can invalidate the + // first HEVC coded frame group when B-frames cross fragment boundaries. + const correctedPtsBase = originalDts + sample.cts - this._videoPtsDtsOffset; if (this._videoPresentationOffset === undefined) { this._videoPresentationOffset = correctedPtsBase - dts; if (this._videoInitialPresentationOffset === undefined) { diff --git a/web-ui/src/playback-engine/worker/pipeline.ts b/web-ui/src/playback-engine/worker/pipeline.ts index 38d17294..40abbf95 100644 --- a/web-ui/src/playback-engine/worker/pipeline.ts +++ b/web-ui/src/playback-engine/worker/pipeline.ts @@ -703,6 +703,7 @@ class Pipeline { // remux batch is not mixed with the previous segment's tail. this._demuxer?.flushSegmentBoundary(); this._remuxer?.flushStashedSamples(); + this._remuxer?.markTsInputBoundary(); } private _prepareContinuousLiveTsRestart(meta: SegmentMeta, ioctl: FetchLoader): void { From 068a589f73a7efd33f91579f8e4decd479246649 Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Wed, 15 Jul 2026 04:14:21 +0800 Subject: [PATCH 5/5] fix(player): prevent HLS PCM switch underruns --- .../playback-engine/audio/pcm-audio-player.ts | 19 +++++++++++++------ .../mse/playback-controller.ts | 13 ++++++++++++- web-ui/src/playback-engine/worker/messages.ts | 2 +- .../playback-engine/worker/transmux-worker.ts | 8 +++++++- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/web-ui/src/playback-engine/audio/pcm-audio-player.ts b/web-ui/src/playback-engine/audio/pcm-audio-player.ts index 190a84e6..9696a1db 100644 --- a/web-ui/src/playback-engine/audio/pcm-audio-player.ts +++ b/web-ui/src/playback-engine/audio/pcm-audio-player.ts @@ -177,6 +177,7 @@ export class PCMAudioPlayer { // This state is independent from lifecycle recovery: only first startup may // temporarily slow video while PCM establishes a shared anchor. private startupSyncState: StartupSyncState = "waiting"; + private startupMinimumLeadSec = STARTUP_MINIMUM_LEAD_SEC; private startupSyncWaitStartedAt: number | null = null; private startupWaitLogged = false; private startupOriginalPlaybackRate: number | null = null; @@ -448,9 +449,10 @@ export class PCMAudioPlayer { * Drop software-decoded audio at and after a track-switch boundary while * allowing already scheduled audio before the boundary to finish. The new * track may use either Web Audio or MSE, so this must be applied even when no - * new PCM chunks are expected. + * new PCM chunks are expected. Bursty inputs may request a larger startup + * lead so their scheduling chain cannot drain between deliveries. */ - replaceFrom(time: number, expectsPCM: boolean): void { + replaceFrom(time: number, expectsPCM: boolean, startupMinimumLeadSec = STARTUP_MINIMUM_LEAD_SEC): void { if (this.destroyed) return; this.pcmGeneration++; this.restoreStartupPlaybackRate(); @@ -477,6 +479,9 @@ export class PCMAudioPlayer { this.scheduledSpans = remaining; this.nextStartTime = remaining.length > 0 ? remaining[remaining.length - 1].ctxEnd : 0; this.resetPCMTimeline(time, expectsPCM ? "waiting" : "disabled"); + if (expectsPCM && Number.isFinite(startupMinimumLeadSec)) { + this.startupMinimumLeadSec = Math.max(STARTUP_MINIMUM_LEAD_SEC, startupMinimumLeadSec); + } Log.v(TAG, `Audio track switch re-anchored timeline at ${time.toFixed(3)}s; target=${expectsPCM ? "PCM" : "MSE"}`); } @@ -804,9 +809,9 @@ export class PCMAudioPlayer { const videoTime = video.currentTime; const audioStartsAfterVideo = audioRange.start > videoTime + GAP_SNAP; - const futureAudioHasLead = audioRange.end - audioRange.start >= STARTUP_MINIMUM_LEAD_SEC - GAP_SNAP; + const futureAudioHasLead = audioRange.end - audioRange.start >= this.startupMinimumLeadSec - GAP_SNAP; const targetHasLead = - videoTime >= audioRange.start - GAP_SNAP && videoTime + STARTUP_MINIMUM_LEAD_SEC <= audioRange.end + GAP_SNAP; + videoTime >= audioRange.start - GAP_SNAP && videoTime + this.startupMinimumLeadSec <= audioRange.end + GAP_SNAP; if (audioStartsAfterVideo && futureAudioHasLead) { this.startupSyncState = "anchoring"; @@ -843,7 +848,7 @@ export class PCMAudioPlayer { TAG, `Startup PCM trails video by ${(lagBehindVideo * 1000).toFixed(1)}ms; ` + `slowing video to ${STARTUP_VIDEO_PLAYBACK_RATE}x until PCM has ` + - `${Math.round(STARTUP_MINIMUM_LEAD_SEC * 1000)}ms lead`, + `${Math.round(this.startupMinimumLeadSec * 1000)}ms lead`, ); this.startupWaitLogged = true; } @@ -859,7 +864,8 @@ export class PCMAudioPlayer { video.playbackRate = STARTUP_VIDEO_PLAYBACK_RATE; } - if (now - this.startupSyncWaitStartedAt < STARTUP_SYNC_TIMEOUT_MS) return; + const timeoutMs = Math.max(STARTUP_SYNC_TIMEOUT_MS, (this.startupMinimumLeadSec + 2) * 1000); + if (now - this.startupSyncWaitStartedAt < timeoutMs) return; this.failStartupSync( `Startup sync failed: videoTime=${videoTime.toFixed(3)}s, ` + @@ -1464,6 +1470,7 @@ export class PCMAudioPlayer { this.pendingChunks = []; this.audioBuffer = []; this.startupSyncState = startupSyncState; + this.startupMinimumLeadSec = STARTUP_MINIMUM_LEAD_SEC; this.startupSyncWaitStartedAt = null; this.startupWaitLogged = false; this.inputCursor = anchor; diff --git a/web-ui/src/playback-engine/mse/playback-controller.ts b/web-ui/src/playback-engine/mse/playback-controller.ts index 0a010f1f..9a0528b2 100644 --- a/web-ui/src/playback-engine/mse/playback-controller.ts +++ b/web-ui/src/playback-engine/mse/playback-controller.ts @@ -28,6 +28,8 @@ const HLS_URL_RE = /\.m3u8?($|\?)/i; type SourceMode = "continuous-live-ts" | "static-ts-list" | "hls"; const OPTIONAL_TS_AUDIO_CODECS = ["ac-3", "ec-3"] as const; +/** HLS PCM arrives in segment-sized bursts; keep enough lead to bridge playlist publication jitter. */ +const HLS_PCM_SWITCH_SAFETY_SECONDS = 0.25; function unsupportedTsAudioCodecs(): string[] { const selfRecord = self as unknown as Record; @@ -55,6 +57,7 @@ export function createMSEPlaybackController( let liveSessionAnchor: LiveSessionAnchor | null = null; let sourceMode: SourceMode = "static-ts-list"; let hlsLive: boolean | null = null; + let hlsTargetDuration = 0; let lastLiveState: boolean | null = null; let hlsVodThrottleEnabled = false; @@ -183,6 +186,7 @@ export function createMSEPlaybackController( case "hls-info": sourceMode = "hls"; hlsLive = msg.live; + hlsTargetDuration = msg.targetDuration; updateLiveState(); if (msg.live) { mse?.setDuration(Infinity); @@ -213,7 +217,12 @@ export function createMSEPlaybackController( if (success) { mse?.replaceAudioFrom(msg.fromTime); const expectsPCM = msg.audioMode === "pcm"; - pcmPlayer?.replaceFrom(expectsPCM ? msg.pcmFromTime : msg.fromTime, expectsPCM); + const startupMinimumLead = + expectsPCM && hlsLive === true && Number.isFinite(hlsTargetDuration) + ? hlsTargetDuration + HLS_PCM_SWITCH_SAFETY_SECONDS + : undefined; + const player = expectsPCM ? ensurePCMPlayer() : pcmPlayer; + player?.replaceFrom(expectsPCM ? msg.pcmFromTime : msg.fromTime, expectsPCM, startupMinimumLead); } switchWorker?.postMessage({ type: "audio-track-switch-result", @@ -546,6 +555,7 @@ export function createMSEPlaybackController( pendingLoad = { segments, options }; sourceMode = inferSourceMode(segments); hlsLive = null; + hlsTargetDuration = 0; resetFetchBackpressure(); updateLiveState(); if (mse) { @@ -595,6 +605,7 @@ export function createMSEPlaybackController( pendingLoad = null; sourceMode = "static-ts-list"; hlsLive = null; + hlsTargetDuration = 0; updateLiveState(); if (worker) { const cmd: WorkerCommand = { type: "reset" }; diff --git a/web-ui/src/playback-engine/worker/messages.ts b/web-ui/src/playback-engine/worker/messages.ts index 3d44a341..151468f0 100644 --- a/web-ui/src/playback-engine/worker/messages.ts +++ b/web-ui/src/playback-engine/worker/messages.ts @@ -42,7 +42,7 @@ export type WorkerEvent = track?: "video" | "audio"; gen: number; } - | { type: "hls-info"; live: boolean; totalDuration: number; gen: number } + | { type: "hls-info"; live: boolean; targetDuration: number; totalDuration: number; gen: number } | { type: "audio-tracks"; state: PlayerAudioTrackState; gen: number } | { type: "audio-codec-unsupported"; codec: string; gen: number } | AudioTrackSwitchEvent diff --git a/web-ui/src/playback-engine/worker/transmux-worker.ts b/web-ui/src/playback-engine/worker/transmux-worker.ts index 7f90ee12..6d59a0bc 100644 --- a/web-ui/src/playback-engine/worker/transmux-worker.ts +++ b/web-ui/src/playback-engine/worker/transmux-worker.ts @@ -337,7 +337,13 @@ function handlePrimaryTimelineAnchor(anchor: MediaTimelineAnchor): void { } function handlePrimaryHlsInfo(info: HlsInfo): void { - post({ type: "hls-info", live: info.live, totalDuration: info.totalDuration, gen }); + post({ + type: "hls-info", + live: info.live, + targetDuration: info.targetDuration, + totalDuration: info.totalDuration, + gen, + }); audioTracks = info.audioTracks ?? []; audioTrackUrls = new Map(Object.entries(info.audioTrackUrls ?? {})); hasHlsAudioGroup = info.hasAudioGroup ?? false;