send multimodal message on startup of multimodal agent

2024-12-09 13:35:55 -08:00 · 2024-12-09 13:35:55 -08:00 · f6c13a13bd
parent 2f53be05db
commit f6c13a13bd
1 changed files with 3 additions and 0 deletions
--- a/software/source/server/livekit/multimodal.py
+++ b/software/source/server/livekit/multimodal.py
@ -104,6 +104,9 @@ async def entrypoint(ctx: JobContext):
        fnc_ctx=fnc_ctx,
    )

+    # Send a message to the client to switch to multimodal mode
+    await ctx.room.local_participant.publish_data(payload="{MULTIMODAL_MODE}", topic="agent_mode")
+
    # Initial message to start the interaction
    session.conversation.item.create(
      llm.ChatMessage(