Bring in sable-call!

.changeset/a-bring-in-sable-call.md

@@ -0,0 +1,5 @@
+---
+default: minor
+---
+
+Bring in Sable Call, our fork of element call, which introduces camera settings, screenshare settings, echo cancellation, noise suppression, automatic gain control, and avatars in calls.

.gitignore

@@ -19,6 +19,7 @@ devAssets
 *.tfbackend
 !*.tfbackend.example
 crash.log
+build.sh
 
 # the following line was added with the "git ignore" tool by itsrye.dev, version 0.1.0
 .lh

knip.json

@@ -7,7 +7,7 @@
   },
   "ignoreDependencies": [
     "buffer",
-    "@element-hq/element-call-embedded",
+    "@sableclient/sable-call-embedded",
     "@matrix-org/matrix-sdk-crypto-wasm",
     "@testing-library/user-event"
   ],

package.json

@@ -69,7 +69,7 @@
     "linkify-react": "^4.3.2",
     "linkifyjs": "^4.3.2",
     "matrix-js-sdk": "^38.4.0",
-    "matrix-widget-api": "1.13.0",
+    "matrix-widget-api": "^1.16.1",
     "millify": "^6.1.0",
     "pdfjs-dist": "^5.4.624",
     "prismjs": "^1.30.0",
@@ -93,7 +93,7 @@
   },
   "devDependencies": {
     "@cloudflare/vite-plugin": "^1.26.0",
-    "@element-hq/element-call-embedded": "0.16.3",
+    "@sableclient/sable-call-embedded": "v1.1.3",
     "@esbuild-plugins/node-globals-polyfill": "^0.2.3",
     "@eslint/compat": "2.0.2",
     "@eslint/js": "9.39.3",
@@ -134,4 +134,4 @@
     "vitest": "^4.1.0",
     "wrangler": "^4.70.0"
   }
-}
+}

pnpm-workspace.yaml

@@ -6,6 +6,8 @@ allowBuilds:
   workerd: true
 engineStrict: true
 minimumReleaseAge: 1440
+minimumReleaseAgeExclude:
+  - '@sableclient/sable-call-embedded'
 
 overrides:
   brace-expansion: '>=1.1.12'

src/app/components/CallEmbedProvider.tsx

@@ -9,8 +9,8 @@ import {
   useCallThemeSync,
   useCallMemberSoundSync,
 } from '$hooks/useCallEmbed';
+import { CallEmbed, useClientWidgetApiEvent, ElementWidgetActions } from '$plugins/call';
 import { callChatAtom, callEmbedAtom } from '$state/callEmbed';
-import { CallEmbed } from '$plugins/call';
 import { useSelectedRoom } from '$hooks/router/useSelectedRoom';
 import { ScreenSize, useScreenSizeContext } from '$hooks/useScreenSize';
 import { IncomingCallModal } from './IncomingCallModal';
@@ -20,12 +20,13 @@ function CallUtils({ embed }: { embed: CallEmbed }) {
 
   useCallMemberSoundSync(embed);
   useCallThemeSync(embed);
-  useCallHangupEvent(
-    embed,
-    useCallback(() => {
-      setCallEmbed(undefined);
-    }, [setCallEmbed])
-  );
+
+  const handleCallEnd = useCallback(() => {
+    setCallEmbed(undefined);
+  }, [setCallEmbed]);
+
+  useCallHangupEvent(embed, handleCallEnd);
+  useClientWidgetApiEvent(embed.call, ElementWidgetActions.Close, handleCallEnd);
 
   return null;
 }

src/app/features/widgets/GenericWidgetDriver.ts

@@ -30,7 +30,7 @@ import {
 
 export type CapabilityApprovalCallback = (requested: Set<Capability>) => Promise<Set<Capability>>;
 
-// Unlike SmallWidgetDriver which auto-grants all capabilities for Element Call,
+// Unlike CallWidgetDriver which auto-grants all capabilities for Element Call,
 // this driver provides a capability approval mechanism for untrusted widgets.
 export class GenericWidgetDriver extends WidgetDriver {
   private readonly mxClient: MatrixClient;
@@ -164,6 +164,18 @@ export class GenericWidgetDriver extends WidgetDriver {
     await this.mxClient._unstable_updateDelayedEvent(delayId, action);
   }
 
+  public async cancelScheduledDelayedEvent(delayId: string): Promise<void> {
+    await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Cancel);
+  }
+
+  public async restartScheduledDelayedEvent(delayId: string): Promise<void> {
+    await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Restart);
+  }
+
+  public async sendScheduledDelayedEvent(delayId: string): Promise<void> {
+    await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Send);
+  }
+
   public async sendToDevice(
     eventType: string,
     encrypted: boolean,

src/app/plugins/call/CallEmbed.ts

@@ -9,9 +9,11 @@
 } from 'matrix-js-sdk';
 import {
   ClientWidgetApi,
+  type IWidgetApiRequest,
   IRoomEvent,
   IWidget,
   Widget,
+  WidgetApiFromWidgetAction,
   WidgetApiToWidgetAction,
   WidgetDriver,
 } from 'matrix-widget-api';
@@ -149,9 +151,24 @@
     const controlState = initialControlState ?? new CallControlState(true, false, true);
     this.control = new CallControl(controlState, call, iframe);
 
+    this.disposables.push(
+      this.listenAction(WidgetApiFromWidgetAction.UpdateAlwaysOnScreen, (evt) => {
+        evt.preventDefault();
+        this.call.transport.reply(evt.detail as IWidgetApiRequest, { success: true });
+      })
+    );
+    this.disposables.push(
+      this.listenAction(ElementWidgetActions.Close, (evt) => {
+        evt.preventDefault();
+        this.call.transport.reply(evt.detail as IWidgetApiRequest, {});
+      })
+    );
+
     let initialMediaEvent = true;
     this.disposables.push(
       this.listenAction<ElementMediaStateDetail>(ElementWidgetActions.DeviceMute, (evt) => {
+        evt.preventDefault();
+        this.call.transport.reply(evt.detail as IWidgetApiRequest, {});
         if (initialMediaEvent) {
           initialMediaEvent = false;
           this.control.applyState();
@@ -258,7 +275,9 @@
     this.eventsToFeed = new WeakSet<MatrixEvent>();
   }
 
-  private onCallJoined(): void {
+  private onCallJoined(evt: CustomEvent): void {
+    evt.preventDefault();
+    this.call.transport.reply(evt.detail as IWidgetApiRequest, {});
     debugLog.info('call', 'Call joined', { roomId: this.roomId });
     this.joined = true;
     this.applyStyles();
@@ -291,7 +310,7 @@
     if (this.call === null) return;
     const raw = ev.getEffectiveEvent();
     this.call.feedStateUpdate(raw as IRoomEvent).catch((e) => {
       console.error('Error sending state update to widget: ', e);
     });
   }
 
@@ -397,7 +416,7 @@
       } else {
         const raw = ev.getEffectiveEvent();
         this.call.feedEvent(raw as IRoomEvent).catch((e) => {
           console.error('Error sending event to widget: ', e);
         });
       }
     }

src/app/plugins/call/CallWidgetDriver.ts

@@ -8,7 +8,7 @@ import {
   type IWidgetApiErrorResponseDataDetails,
   type ISearchUserDirectoryResult,
   type IGetMediaConfigResult,
-  type UpdateDelayedEventAction,
+  UpdateDelayedEventAction,
   OpenIDRequestState,
   SimpleObservable,
   IOpenIDUpdate,
@@ -165,6 +165,18 @@ export class CallWidgetDriver extends WidgetDriver {
     await client._unstable_updateDelayedEvent(delayId, action);
   }
 
+  public async cancelScheduledDelayedEvent(delayId: string): Promise<void> {
+    await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Cancel);
+  }
+
+  public async restartScheduledDelayedEvent(delayId: string): Promise<void> {
+    await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Restart);
+  }
+
+  public async sendScheduledDelayedEvent(delayId: string): Promise<void> {
+    await this.updateDelayedEvent(delayId, UpdateDelayedEventAction.Send);
+  }
+
   public async sendToDevice(
     eventType: string,
     encrypted: boolean,

src/app/plugins/call/utils.ts

@@ -18,6 +18,8 @@ export function getCallCapabilities(
   capabilities.add(MatrixCapabilities.MSC3846TurnServers);
   capabilities.add(MatrixCapabilities.MSC4157SendDelayedEvent);
   capabilities.add(MatrixCapabilities.MSC4157UpdateDelayedEvent);
+  capabilities.add('moe.sable.thumbnails');
+  capabilities.add('moe.sable.media_proxy');
   capabilities.add(`org.matrix.msc2762.timeline:${roomId}`);
   capabilities.add(`org.matrix.msc2762.state:${roomId}`);
 
@@ -78,20 +80,16 @@ export function getCallCapabilities(
     WidgetEventCapability.forStateEvent(EventDirection.Receive, EventType.RoomCreate).raw
   );
 
-  capabilities.add(
-    WidgetEventCapability.forRoomEvent(
-      EventDirection.Receive,
-      'org.matrix.msc4075.rtc.notification'
-    ).raw
-  );
-
   [
     'io.element.call.encryption_keys',
     'org.matrix.rageshake_request',
     EventType.Reaction,
     EventType.RoomRedaction,
     'io.element.call.reaction',
     'org.matrix.msc4310.rtc.decline',
+    'org.matrix.msc4075.call.notify',
+    'org.matrix.msc4075.rtc.notification',
+    'org.matrix.msc4143.rtc.member',
   ].forEach((type) => {
     capabilities.add(WidgetEventCapability.forRoomEvent(EventDirection.Send, type).raw);
     capabilities.add(WidgetEventCapability.forRoomEvent(EventDirection.Receive, type).raw);

src/sw.ts

@@ -596,6 +596,20 @@ self.addEventListener('fetch', (event: FetchEvent) => {
     return;
   }
 
+  // Since widgets like element call have their own client ids,
+  // we need this logic. We just go through the sessions list and get a session
+  // with the right base url. Media requests to a homeserver simply are fine with any account
+  // on the homeserver authenticating it, so this is fine. But it can be technically wrong.
+  // If you have two tabs for different users on the same homeserver, it might authenticate
+  // as the wrong one.
+  // Thus any logic in the future which cares about which user is authenticating the request
+  // might break this. Also, again, it is technically wrong.
+  const byBaseUrl = [...sessions.values()].find((s) => validMediaRequest(url, s.baseUrl));
+  if (byBaseUrl) {
+    event.respondWith(fetch(url, { ...fetchConfig(byBaseUrl.accessToken), redirect }));
+    return;
+  }
+
   event.respondWith(
     requestSessionWithTimeout(clientId).then((s) => {
       if (s && validMediaRequest(url, s.baseUrl)) {

vite.config.ts

@@ -63,7 +63,7 @@ const isReleaseTag = (() => {
 const copyFiles = {
   targets: [
     {
-      src: 'node_modules/@element-hq/element-call-embedded/dist/*',
+      src: 'node_modules/@sableclient/sable-call-embedded/dist/*',
       dest: 'public/element-call',
     },
     {