From aa870e5574c40004a9462d3b942a904b3fde0e29 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 12 Feb 2026 16:56:36 +0100 Subject: [PATCH 1/7] Add typeguards for frame processors in order to avoid dual package hazard --- packages/livekit-rtc/src/audio_stream.ts | 9 +++-- packages/livekit-rtc/src/frame_processor.ts | 37 ++++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/packages/livekit-rtc/src/audio_stream.ts b/packages/livekit-rtc/src/audio_stream.ts index c136e098..984a86c1 100644 --- a/packages/livekit-rtc/src/audio_stream.ts +++ b/packages/livekit-rtc/src/audio_stream.ts @@ -5,7 +5,12 @@ import type { UnderlyingSource } from 'node:stream/web'; import { AudioFrame } from './audio_frame.js'; import type { FfiEvent } from './ffi_client.js'; import { FfiClient, FfiClientEvent, FfiHandle } from './ffi_client.js'; -import { FrameProcessor } from './frame_processor.js'; +import { + FrameProcessor, + FrameProcessorSymbol, + isAudioFrameProcessor, + isFrameProcessor, +} from './frame_processor.js'; import { log } from './log.js'; import type { NewAudioStreamResponse } from './proto/audio_frame_pb.js'; import { AudioStreamType, NewAudioStreamRequest } from './proto/audio_frame_pb.js'; @@ -41,7 +46,7 @@ class AudioStreamSource implements UnderlyingSource { if (sampleRateOrOptions !== undefined && typeof sampleRateOrOptions !== 'number') { this.sampleRate = sampleRateOrOptions.sampleRate ?? 48000; this.numChannels = sampleRateOrOptions.numChannels ?? 1; - if (sampleRateOrOptions.noiseCancellation instanceof FrameProcessor) { + if (isAudioFrameProcessor(sampleRateOrOptions.noiseCancellation)) { this.frameProcessor = sampleRateOrOptions.noiseCancellation; } else { this.legacyNcOptions = sampleRateOrOptions.noiseCancellation; diff --git a/packages/livekit-rtc/src/frame_processor.ts b/packages/livekit-rtc/src/frame_processor.ts index 70f6be11..9840796b 100644 --- a/packages/livekit-rtc/src/frame_processor.ts +++ b/packages/livekit-rtc/src/frame_processor.ts @@ -15,7 +15,42 @@ export type FrameProcessorCredentials = { url: string; }; -export abstract class FrameProcessor { +export const FrameProcessorSymbol = Symbol.for('lk.frame-processor'); + +export type FrameProcessorType = 'audio' | 'video'; + +export function isFrameProcessor( + maybeProcessor: unknown, + type?: Type, +): maybeProcessor is FrameProcessor< + Type extends 'audio' ? AudioFrame : Type extends 'video' ? VideoFrame : AudioFrame | VideoFrame +> { + return ( + maybeProcessor !== null && + typeof maybeProcessor === 'object' && + 'symbol' in maybeProcessor && + maybeProcessor.symbol === FrameProcessorSymbol && + (!type || ('type' in maybeProcessor && maybeProcessor.type === type)) + ); +} + +export function isAudioFrameProcessor( + maybeProcessor: unknown, +): maybeProcessor is FrameProcessor { + return isFrameProcessor(maybeProcessor, 'audio'); +} + +export function isVideoFrameProcessor( + maybeProcessor: unknown, +): maybeProcessor is FrameProcessor { + return isFrameProcessor(maybeProcessor, 'video'); +} + +export abstract class FrameProcessor< + Frame extends VideoFrame | AudioFrame = VideoFrame | AudioFrame, +> { + readonly symbol = FrameProcessorSymbol; + abstract readonly type: FrameProcessorType; abstract isEnabled(): boolean; abstract setEnabled(enabled: boolean): void; From eb188f1c36a97599df5fc512fc6e482b49b50839 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 12 Feb 2026 16:59:21 +0100 Subject: [PATCH 2/7] Create eleven-timers-kick.md --- .changeset/eleven-timers-kick.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/eleven-timers-kick.md diff --git a/.changeset/eleven-timers-kick.md b/.changeset/eleven-timers-kick.md new file mode 100644 index 00000000..8aa5ef38 --- /dev/null +++ b/.changeset/eleven-timers-kick.md @@ -0,0 +1,5 @@ +--- +"@livekit/rtc-node": patch +--- + +Add typeguards for frame processors in order to avoid dual package hazard From dbdc0e79ce62441ec5b65d00685703a1313171d2 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 12 Feb 2026 17:00:27 +0100 Subject: [PATCH 3/7] no default value for generic --- packages/livekit-rtc/src/frame_processor.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/livekit-rtc/src/frame_processor.ts b/packages/livekit-rtc/src/frame_processor.ts index 9840796b..1e6305c1 100644 --- a/packages/livekit-rtc/src/frame_processor.ts +++ b/packages/livekit-rtc/src/frame_processor.ts @@ -46,9 +46,7 @@ export function isVideoFrameProcessor( return isFrameProcessor(maybeProcessor, 'video'); } -export abstract class FrameProcessor< - Frame extends VideoFrame | AudioFrame = VideoFrame | AudioFrame, -> { +export abstract class FrameProcessor { readonly symbol = FrameProcessorSymbol; abstract readonly type: FrameProcessorType; abstract isEnabled(): boolean; From a76f56a1def852ee0c47edf30fe5b462a1db8f81 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 12 Feb 2026 17:03:48 +0100 Subject: [PATCH 4/7] simplify --- packages/livekit-rtc/src/frame_processor.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/livekit-rtc/src/frame_processor.ts b/packages/livekit-rtc/src/frame_processor.ts index 1e6305c1..d8f4b2f9 100644 --- a/packages/livekit-rtc/src/frame_processor.ts +++ b/packages/livekit-rtc/src/frame_processor.ts @@ -17,8 +17,6 @@ export type FrameProcessorCredentials = { export const FrameProcessorSymbol = Symbol.for('lk.frame-processor'); -export type FrameProcessorType = 'audio' | 'video'; - export function isFrameProcessor( maybeProcessor: unknown, type?: Type, @@ -48,7 +46,11 @@ export function isVideoFrameProcessor( export abstract class FrameProcessor { readonly symbol = FrameProcessorSymbol; - abstract readonly type: FrameProcessorType; + abstract readonly type: Frame extends VideoFrame + ? 'video' + : Frame extends AudioFrame + ? 'audio' + : never; abstract isEnabled(): boolean; abstract setEnabled(enabled: boolean): void; From 4c454fa8d1851af303def42075862d68ff0bed34 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 12 Feb 2026 17:18:45 +0100 Subject: [PATCH 5/7] Update eleven-timers-kick.md --- .changeset/eleven-timers-kick.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/eleven-timers-kick.md b/.changeset/eleven-timers-kick.md index 8aa5ef38..a84a957f 100644 --- a/.changeset/eleven-timers-kick.md +++ b/.changeset/eleven-timers-kick.md @@ -1,5 +1,5 @@ --- -"@livekit/rtc-node": patch +"@livekit/rtc-node": minor --- Add typeguards for frame processors in order to avoid dual package hazard From dabbf25348cf65047347a2282686f8e01dd23a6a Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 12 Feb 2026 17:22:29 +0100 Subject: [PATCH 6/7] lint --- packages/livekit-rtc/src/audio_stream.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/livekit-rtc/src/audio_stream.ts b/packages/livekit-rtc/src/audio_stream.ts index 984a86c1..94181cb1 100644 --- a/packages/livekit-rtc/src/audio_stream.ts +++ b/packages/livekit-rtc/src/audio_stream.ts @@ -5,12 +5,7 @@ import type { UnderlyingSource } from 'node:stream/web'; import { AudioFrame } from './audio_frame.js'; import type { FfiEvent } from './ffi_client.js'; import { FfiClient, FfiClientEvent, FfiHandle } from './ffi_client.js'; -import { - FrameProcessor, - FrameProcessorSymbol, - isAudioFrameProcessor, - isFrameProcessor, -} from './frame_processor.js'; +import { type FrameProcessor, isAudioFrameProcessor } from './frame_processor.js'; import { log } from './log.js'; import type { NewAudioStreamResponse } from './proto/audio_frame_pb.js'; import { AudioStreamType, NewAudioStreamRequest } from './proto/audio_frame_pb.js'; From 3209a2460272c4c682d5869935a2d17c764e9946 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 12 Feb 2026 20:09:04 +0100 Subject: [PATCH 7/7] ensure userdata is passed through when resampling (#609) --- .changeset/dry-buttons-help.md | 5 +++++ packages/livekit-rtc/src/audio_frame.ts | 18 +++++++++++++++--- packages/livekit-rtc/src/audio_resampler.ts | 2 ++ 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 .changeset/dry-buttons-help.md diff --git a/.changeset/dry-buttons-help.md b/.changeset/dry-buttons-help.md new file mode 100644 index 00000000..ec7844b9 --- /dev/null +++ b/.changeset/dry-buttons-help.md @@ -0,0 +1,5 @@ +--- +"@livekit/rtc-node": patch +--- + +ensure userdata is passed through when resampling diff --git a/packages/livekit-rtc/src/audio_frame.ts b/packages/livekit-rtc/src/audio_frame.ts index d8e7ab32..08c57d1b 100644 --- a/packages/livekit-rtc/src/audio_frame.ts +++ b/packages/livekit-rtc/src/audio_frame.ts @@ -32,9 +32,14 @@ export class AudioFrame { this._userdata = userdata; } - static create(sampleRate: number, channels: number, samplesPerChannel: number): AudioFrame { + static create( + sampleRate: number, + channels: number, + samplesPerChannel: number, + userdata?: Record, + ): AudioFrame { const data = new Int16Array(channels * samplesPerChannel); - return new AudioFrame(data, sampleRate, channels, samplesPerChannel); + return new AudioFrame(data, sampleRate, channels, samplesPerChannel, userdata); } /** @internal */ @@ -103,5 +108,12 @@ export const combineAudioFrames = (buffer: AudioFrame | AudioFrame[]): AudioFram } const data = new Int16Array(buffer.map((x) => [...x.data]).flat()); - return new AudioFrame(data, sampleRate, channels, totalSamplesPerChannel); + + // Merge userdata from all frames + const mergedUserdata: Record = {}; + for (const frame of buffer) { + Object.assign(mergedUserdata, frame.userdata); + } + + return new AudioFrame(data, sampleRate, channels, totalSamplesPerChannel, mergedUserdata); }; diff --git a/packages/livekit-rtc/src/audio_resampler.ts b/packages/livekit-rtc/src/audio_resampler.ts index 82dfe0fc..4468bea5 100644 --- a/packages/livekit-rtc/src/audio_resampler.ts +++ b/packages/livekit-rtc/src/audio_resampler.ts @@ -131,12 +131,14 @@ export class AudioResampler { } const outputData = FfiClient.instance.copyBuffer(res.outputPtr, res.size!); + return [ new AudioFrame( new Int16Array(outputData.buffer), this.#outputRate, this.#channels, Math.trunc(outputData.length / this.#channels / 2), + data.userdata, ), ]; }