jellyfin: subtitle tracks, audio picker and real player controls

the sidecar now describes every audio and subtitle stream on the chosen source. a subtitle jellyfin can deliver as an external file gets a /_jf webvtt path with the embedded ApiKey stripped; one it cannot gets the reason instead, so the picker can show it disabled rather than pretend it does not exist. colliding labels get their stream index — two tracks called "English - ASS" are a coin flip otherwise. quality is a named rung rather than a number. a bitrate the owner picks maps onto the rung at or below it, which is what stops jellyfin's ResolutionNormalizer from inventing a resolution from an unrounded number. auto stays absent so a copyable source still copies. the control bar is ours in all three transports. progressive has no timeline the browser can render, and an audio-track or quality change is a re-negotiation with the server rather than something a <video> knows how to do — one bar for all three is what keeps those from being three different players. the menus are hand-rolled because a radix dropdown portals to document.body, which is outside the fullscreen element and would be invisible exactly when the player is most likely to be used. no scrubber thumbnails: they come from trickplay tiles and this server generates none. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-04 19:10:44 +00:00
parent 7732ca1da5
commit 7884bd3290
5 changed files with 842 additions and 81 deletions
@@ -190,12 +190,24 @@ async function item(cfg: UpstreamConfig, id: string): Promise<Response> {

 type SourceStream = {
  Type?: string;
+  Index?: number;
  Codec?: string;
  Profile?: string;
  Level?: number;
  Width?: number;
  Height?: number;
  BitRate?: number;
+  Channels?: number;
+  Language?: string | null;
+  DisplayTitle?: string | null;
+  Title?: string | null;
+  IsDefault?: boolean;
+  IsForced?: boolean;
+  IsExternal?: boolean;
+  IsTextSubtitleStream?: boolean;
+  /** Jellyfin's own verdict for THIS profile: External (a file we can fetch), Embed, Encode, or None. */
+  DeliveryMethod?: string | null;
+  DeliveryUrl?: string | null;
 };

 type MediaSource = {
@@ -238,10 +250,24 @@ const COPY_CEILING_BITRATE = 120_000_000;
 const ENCODE_BITRATE = 12_000_000;
 const ENCODE_MAX_WIDTH = 1920;

+/**
+ * The rungs the quality picker offers, widest first. A bitrate alone is not a quality: Jellyfin's own
+ * `ResolutionNormalizer` would pair a low one with a resolution of its choosing, and the whole 416-pixel
+ * incident was that mapping being applied to a number nobody meant. Naming both here is what makes
+ * "720p" mean 720p.
+ */
+export const QUALITY_RUNGS = [
+  { label: '1080p', videoBitrate: 12_000_000, maxWidth: 1920 },
+  { label: '720p', videoBitrate: 6_000_000, maxWidth: 1280 },
+  { label: '480p', videoBitrate: 2_500_000, maxWidth: 854 },
+  { label: '360p', videoBitrate: 1_000_000, maxWidth: 640 },
+] as const;
+
 /**
 * Decide the bitrate and resolution cap to ask for, from what is actually in the file.
 *
- * The two cases pull in opposite directions, which is why this is not one constant:
+ * With no `requested` ceiling — the "Auto" the player sends by default — the two cases pull in opposite
+ * directions, which is why this is not one constant:
 *
 *   video can be copied → any cap is a reason for Jellyfin to REFUSE the copy (it will not copy a stream
 *                         above the requested bitrate, or wider than a requested `maxWidth`), so ask for a
@@ -251,8 +277,18 @@ const ENCODE_MAX_WIDTH = 1920;
 *                         CPU; downscaling to 1080p is what makes that keep up with playback.
 *
 * The copy test mirrors the profile's own conditions rather than a new opinion: h264, not 10-bit, level ≤ 5.2.
+ *
+ * A `requested` ceiling overrides both, and deliberately defeats the copy: picking 480p on a file that would
+ * have passed through untouched is asking for a re-encode, and that is the entire point of picking it.
 */
-function transcodeQuality(source: MediaSource): { videoBitrate: number; maxWidth?: number } {
+export type TranscodeQuality = { videoBitrate: number; maxWidth?: number };
+
+function transcodeQuality(source: MediaSource, requested?: number): TranscodeQuality {
+  if (requested && requested < COPY_CEILING_BITRATE) {
+    const rung = QUALITY_RUNGS.find((r) => r.videoBitrate <= requested) ?? QUALITY_RUNGS[QUALITY_RUNGS.length - 1]!;
+    return { videoBitrate: rung.videoBitrate, maxWidth: rung.maxWidth };
+  }
+
  const video = source.MediaStreams?.find((stream) => stream.Type === 'Video');
  const copyable =
    video?.Codec?.toLowerCase() === 'h264' && video.Profile?.toLowerCase() !== 'high 10' && (video.Level ?? 0) <= 52;
@@ -261,6 +297,99 @@ function transcodeQuality(source: MediaSource): { videoBitrate: number; maxWidth
  return { videoBitrate: ENCODE_BITRATE, maxWidth: ENCODE_MAX_WIDTH };
 }

+/** `DisplayTitle` is what Jellyfin shows in its own picker; the rest is a legible fallback when it is absent. */
+function trackLabel(stream: SourceStream, fallback: string): string {
+  const parts = [stream.DisplayTitle || stream.Title, stream.Language, stream.Codec?.toUpperCase()].filter(Boolean);
+  return (parts[0] as string) || parts.join(' · ') || fallback;
+}
+
+export type AudioTrack = {
+  index: number;
+  label: string;
+  language: string | null;
+  codec: string | null;
+  channels: number | null;
+  isDefault: boolean;
+};
+
+export type SubtitleTrack = {
+  index: number;
+  label: string;
+  language: string | null;
+  codec: string | null;
+  isDefault: boolean;
+  isForced: boolean;
+  /** A `/_jf` path to a WebVTT file, or null when this track cannot be delivered as one. */
+  url: string | null;
+  /** Why `url` is null, in words the picker can show. Null when the track is fine. */
+  unavailable: string | null;
+};
+
+/**
+ * Turn the source's streams into the two pickers the player offers.
+ *
+ * Subtitles are the interesting half. Jellyfin has already decided, for OUR profile, how each one could be
+ * delivered, and `DeliveryMethod: 'External'` with a `DeliveryUrl` means "fetch this as a file" — which for
+ * the profile's declared formats is WebVTT the `<track>` element reads directly. That is the only method
+ * taken. `Encode` means burning the subtitle into the picture, which forces a full video re-encode of a file
+ * that may have needed nothing but a remux, so an image-based track (PGS, VOBSUB) is reported as unavailable
+ * WITH ITS REASON rather than quietly dropped — "why is this subtitle missing" should be answerable from the
+ * picker instead of from the server logs.
+ */
+function describeTracks(source: MediaSource): { audio: AudioTrack[]; subtitles: SubtitleTrack[] } {
+  const streams = source.MediaStreams ?? [];
+
+  const audio = streams
+    .filter((stream) => stream.Type === 'Audio' && stream.Index !== undefined)
+    .map((stream) => ({
+      index: stream.Index!,
+      label: trackLabel(stream, `Track ${stream.Index}`),
+      language: stream.Language ?? null,
+      codec: stream.Codec ?? null,
+      channels: stream.Channels ?? null,
+      isDefault: stream.IsDefault === true,
+    }));
+
+  const subtitles = streams
+    .filter((stream) => stream.Type === 'Subtitle' && stream.Index !== undefined)
+    .map((stream) => {
+      const external = stream.DeliveryMethod === 'External' && !!stream.DeliveryUrl;
+      const graphical = stream.IsTextSubtitleStream === false;
+      return {
+        index: stream.Index!,
+        label: trackLabel(stream, `Subtitle ${stream.Index}`),
+        language: stream.Language ?? null,
+        codec: stream.Codec ?? null,
+        isDefault: stream.IsDefault === true,
+        isForced: stream.IsForced === true,
+        url: external ? `/_jf${sanitizeUpstreamPath(stream.DeliveryUrl!)}` : null,
+        unavailable: external
+          ? null
+          : graphical
+            ? 'image-based subtitles can only be burned into the video, which forces a full re-encode'
+            : `the server offered no external file for this track (delivery: ${stream.DeliveryMethod ?? 'none'})`,
+      };
+    });
+
+  return { audio: disambiguate(audio), subtitles: disambiguate(subtitles) };
+}
+
+/**
+ * Make every label in a list distinguishable.
+ *
+ * Two subtitle tracks on the same file genuinely do come back as "English - ASS" and "English - ASS" — one is
+ * usually dialogue and the other signs, and Jellyfin's own `DisplayTitle` does not say which. A picker with
+ * two identical rows is a coin flip, so colliding labels get their stream index, which at least makes the
+ * choice repeatable.
+ */
+function disambiguate<T extends { index: number; label: string }>(tracks: T[]): T[] {
+  const counts = new Map<string, number>();
+  for (const track of tracks) counts.set(track.label, (counts.get(track.label) ?? 0) + 1);
+  return tracks.map((track) =>
+    counts.get(track.label)! > 1 ? { ...track, label: `${track.label} #${track.index}` } : track,
+  );
+}
+
 /**
 * Fix up the HLS URL Jellyfin handed back so it carries the same quality decision as the progressive one.
 *
@@ -275,11 +404,10 @@ function transcodeQuality(source: MediaSource): { videoBitrate: number; maxWidth
 * Verified against a real answer from this server: the URL comes back with `VideoBitrate=119616000` and no
 * width, on an item whose transcode reason was `ContainerNotSupported` alone.
 */
-function hlsPath(source: MediaSource): string {
+function hlsPath(source: MediaSource, quality: TranscodeQuality): string {
  const sanitized = sanitizeUpstreamPath(source.TranscodingUrl!);
  const [path, search = ''] = sanitized.split('?');
  const params = new URLSearchParams(search);
-  const quality = transcodeQuality(source);

  // Jellyfin's own casing here is PascalCase, and its query parsing is case-insensitive — but a duplicate key
  // under a different case would still be two values, so delete before setting rather than trusting that.
@@ -315,8 +443,15 @@ function hlsPath(source: MediaSource): string {
 * `scale=…min(max(iw,ih*a),416)…` with `-maxrate 0`, i.e. 416 pixels wide. That is the whole of the "why does
 * it look so bad" question, and it is a silent, plausible-looking picture rather than an error.
 */
-function progressivePath(id: string, source: MediaSource, playSessionId: string | null, startSeconds: number): string {
-  const quality = transcodeQuality(source);
+type ProgressiveParams = {
+  id: string;
+  source: MediaSource;
+  playSessionId: string | null;
+  startSeconds: number;
+  quality: TranscodeQuality;
+};
+
+function progressivePath({ id, source, playSessionId, startSeconds, quality }: ProgressiveParams): string {
  return `/Videos/${encodeURIComponent(id)}/stream.mp4${buildQuery({
    static: 'false',
    container: 'mp4',
@@ -389,6 +524,7 @@ async function playbackInfo(cfg: UpstreamConfig, id: string, req: Request, url:

  const playSessionId = info.PlaySessionId ?? null;
  const direct = !!(source.SupportsDirectPlay || source.SupportsDirectStream);
+  const quality = transcodeQuality(source, body.maxStreamingBitrate);

  const staticPath = `/Videos/${encodeURIComponent(id)}/stream${buildQuery({
    static: 'true',
@@ -403,7 +539,7 @@ async function playbackInfo(cfg: UpstreamConfig, id: string, req: Request, url:
    : // A `TranscodingUrl` is Jellyfin agreeing to transcode. We ask for the same thing progressively
      // instead of taking its HLS URL, but its absence still means "refused", so it stays the signal.
      source.TranscodingUrl
-      ? progressivePath(id, source, playSessionId, startSeconds)
+      ? progressivePath({ id, source, playSessionId, startSeconds, quality })
      : null;

  if (!path) {
@@ -419,12 +555,19 @@ async function playbackInfo(cfg: UpstreamConfig, id: string, req: Request, url:
    seekable: direct,
    // The seekable form of a transcode: the playlist spans the whole runtime, so a scrub is a segment request
    // rather than a new ffmpeg. Preferred by the player whenever it is present and MSE is available.
-    hlsUrl: source.TranscodingUrl ? `/_jf${hlsPath(source)}` : null,
+    hlsUrl: source.TranscodingUrl ? `/_jf${hlsPath(source, quality)}` : null,
    playSessionId,
    mediaSourceId: source.Id ?? null,
    mediaSource: source,
    startSeconds,
    runtimeSeconds: source.RunTimeTicks ? source.RunTimeTicks / TICKS_PER_SECOND : null,
+    // The pickers. Audio needs a re-negotiation to change (it is muxed into the stream); subtitles do not,
+    // because an external VTT is a separate file the <track> element fetches on its own.
+    ...describeTracks(source),
+    audioStreamIndex: body.audioStreamIndex ?? null,
+    /** Null means Auto — the copy-or-encode decision above, rather than a rung the owner picked. */
+    maxStreamingBitrate: body.maxStreamingBitrate ?? null,
+    qualityRungs: QUALITY_RUNGS,
  });
 }