Multimodal EOU

examples/README.md

@@ -54,7 +54,7 @@ To run the examples, you'll need:
 
 - A [LiveKit Cloud](https://cloud.livekit.io) account or a local [LiveKit server](https://github.com/livekit/livekit)
 - API keys for the model providers you want to use in a `.env` file
-- Python 3.9 or higher
+- Python 3.10 or higher
 - [uv](https://docs.astral.sh/uv/)
 
 ### Environment file

examples/voice_agents/basic_agent.py

@@ -20,7 +20,6 @@
 from livekit.agents.beta import EndCallTool
 from livekit.agents.llm import function_tool
 from livekit.plugins import silero
-from livekit.plugins.turn_detector.multilingual import MultilingualModel
 
 # uncomment to enable Krisp background voice/noise cancellation
 # from livekit.plugins import noise_cancellation
@@ -98,7 +97,15 @@ async def entrypoint(ctx: JobContext) -> None:
         turn_handling=TurnHandlingOptions(
             # VAD and turn detection are used to determine when the user is speaking and when the agent should respond
             # See more at https://docs.livekit.io/agents/build/turns
-            turn_detection=MultilingualModel(),
+            # turn_detection=MultilingualModel(),
+            turn_detection=inference.MultimodalTurnDetector(
+                # TODO: @chenghao-mou remove this before merging
+                base_url="http://0.0.0.0:8080/v1",
+            ),
+            endpointing={
+                "min_delay": 0.5,
+                "max_delay": 3.0,
+            },
             interruption={
                 # sometimes background noise could interrupt the agent session, these are considered false positive interruptions
                 # when it's detected, you may resume the agent's speech

livekit-agents/livekit/agents/inference/__init__.py

@@ -7,6 +7,12 @@
 from .llm import LLM, LLMModels, LLMStream
 from .stt import STT, STTModels
 from .tts import TTS, TTSModels
+from .turn_detection import (
+    MIN_SILENCE_DURATION_MS,
+    MultimodalTurnDetector,
+    TurnDetectionEvent,
+    TurnDetectionStream,
+)
 
 __all__ = [
     "STT",
@@ -16,8 +22,12 @@
     "STTModels",
     "TTSModels",
     "LLMModels",
+    "MultimodalTurnDetector",
+    "TurnDetectionStream",
+    "TurnDetectionEvent",
     "AdaptiveInterruptionDetector",
     "InterruptionDetectionError",
     "OverlappingSpeechEvent",
     "InterruptionDataFrameType",
+    "MIN_SILENCE_DURATION_MS",
 ]

livekit-agents/livekit/agents/inference/turn_detection/__init__.py

@@ -0,0 +1,10 @@
+from .detector import MultimodalTurnDetector, TurnDetectionEvent, TurnDetectorOptions
+from .stream import MIN_SILENCE_DURATION_MS, TurnDetectionStream
+
+__all__ = [
+    "MultimodalTurnDetector",
+    "TurnDetectionStream",
+    "TurnDetectionEvent",
+    "TurnDetectorOptions",
+    "MIN_SILENCE_DURATION_MS",
+]

livekit-agents/livekit/agents/inference/turn_detection/detector.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import os
+import weakref
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Literal
+
+import aiohttp
+
+from ... import utils
+from ...language import LanguageCode
+from ...types import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    NOT_GIVEN,
+    APIConnectOptions,
+    NotGivenOr,
+)
+from .languages import LANGUAGES
+
+if TYPE_CHECKING:
+    from .stream import TurnDetectionStream
+
+
+DEFAULT_SAMPLE_RATE: int = 16000
+DEFAULT_BASE_URL = "https://agent-gateway.livekit.cloud/v1"
+
+
+@dataclass
+class TurnDetectionEvent:
+    type: Literal["eot_prediction"]
+    end_of_turn_probability: float
+    last_speaking_time: float
+    detection_delay: float | None = None
+    backend: Literal["multimodal", "text"] = "multimodal"
+
+
+@dataclass
+class TurnDetectorOptions:
+    sample_rate: int
+    base_url: str
+    api_key: str
+    api_secret: str
+    conn_options: APIConnectOptions
+
+
+class MultimodalTurnDetector:
+    def __init__(
+        self,
+        *,
+        base_url: NotGivenOr[str] = NOT_GIVEN,
+        api_key: NotGivenOr[str] = NOT_GIVEN,
+        api_secret: NotGivenOr[str] = NOT_GIVEN,
+        sample_rate: int = DEFAULT_SAMPLE_RATE,
+        http_session: aiohttp.ClientSession | None = None,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> None:
+        lk_base_url = utils.resolve_env_var(
+            base_url, "LIVEKIT_INFERENCE_URL", default=DEFAULT_BASE_URL
+        )
+        lk_api_key = utils.resolve_env_var(
+            api_key, "LIVEKIT_INFERENCE_API_KEY", "LIVEKIT_API_KEY", default=""
+        )
+        lk_api_secret = utils.resolve_env_var(
+            api_secret, "LIVEKIT_INFERENCE_API_SECRET", "LIVEKIT_API_SECRET", default=""
+        )
+        if not lk_api_secret:
+            raise ValueError(
+                "api_secret is required, either as argument or set LIVEKIT_API_SECRET env var"
+            )
+        if not lk_api_key:
+            raise ValueError(
+                "api_key is required, either as argument or set LIVEKIT_API_KEY env var"
+            )
+
+        self._worker_token = os.getenv("LIVEKIT_WORKER_TOKEN")
+        self._opts = TurnDetectorOptions(
+            sample_rate=sample_rate,
+            base_url=lk_base_url,
+            api_key=lk_api_key,
+            api_secret=lk_api_secret,
+            conn_options=conn_options,
+        )
+
+        self._session = http_session
+        self._streams: weakref.WeakSet[TurnDetectionStream] = weakref.WeakSet()
+
+    @property
+    def model(self) -> str:
+        return "eot-multimodal"
+
+    @property
+    def provider(self) -> str:
+        return "livekit"
+
+    def _ensure_session(self) -> aiohttp.ClientSession:
+        if not self._session:
+            self._session = utils.http_context.http_session()
+        return self._session
+
+    def stream(
+        self,
+        *,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> TurnDetectionStream:
+        from .stream import TurnDetectionStream
+
+        stream: TurnDetectionStream = TurnDetectionStream(
+            detector=self,
+            opts=self._opts,
+            conn_options=conn_options,
+        )
+        self._streams.add(stream)
+        return stream
+
+    async def unlikely_threshold(
+        self, language: LanguageCode | None, modality: Literal["multimodal", "text"] = "multimodal"
+    ) -> float | None:
+        thresholds = LANGUAGES.get(
+            language.language if language is not None else "en", (0.35, 0.011)
+        )
+        if modality == "multimodal":
+            return thresholds[0]
+        else:
+            return thresholds[1]
+
+    async def supports_language(
+        self, language: LanguageCode | None, modality: Literal["multimodal", "text"] = "multimodal"
+    ) -> bool:
+        # default to english if no language is provided
+        lang = language.language if language is not None else "en"
+        return lang in LANGUAGES
+
+    async def aclose(self) -> None:
+        for stream in list(self._streams):
+            await stream.aclose()
+        self._streams.clear()
+        self._session = None

livekit-agents/livekit/agents/inference/turn_detection/languages.py

@@ -0,0 +1,17 @@
+LANGUAGES = {
+    # language code: (audio threshold, text threshold)
+    "en": (0.35, 0.011),
+    "fr": (0.35, 0.0078),
+    "de": (0.35, 0.0062),
+    "hi": (0.35, 0.0398),
+    "ja": (0.35, 0.0096),
+    "ko": (0.35, 0.0156),
+    "zh": (0.35, 0.0066),
+    "es": (0.35, 0.0058),
+    "nl": (None, 0.0077),
+    "pt": (None, 0.0069),
+    "it": (None, 0.0037),
+    "ru": (None, 0.0032),
+    "tr": (None, 0.0045),
+    "id": (None, 0.0132),
+}