deploy: 61d1907

Author: truemagic-coder

.doctrees/api/index.doctree

@@ -87,6 +87,7 @@ <h1>Source code for solana_agent.client.solana_agent</h1><div class="highlight">
 <span class="kn">from</span><span class="w"> </span><span class="nn">solana_agent.interfaces.plugins.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Tool</span>
 <span class="kn">from</span><span class="w"> </span><span class="nn">solana_agent.services.knowledge_base</span><span class="w"> </span><span class="kn">import</span> <span class="n">KnowledgeBaseService</span>
 <span class="kn">from</span><span class="w"> </span><span class="nn">solana_agent.interfaces.services.routing</span><span class="w"> </span><span class="kn">import</span> <span class="n">RoutingService</span> <span class="k">as</span> <span class="n">RoutingInterface</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">solana_agent.interfaces.providers.realtime</span><span class="w"> </span><span class="kn">import</span> <span class="n">RealtimeChunk</span>
 
 
 <div class="viewcode-block" id="SolanaAgent">
@@ -132,6 +133,7 @@ <h1>Source code for solana_agent.client.solana_agent</h1><div class="highlight">
         <span class="n">vad</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
         <span class="n">rt_encode_input</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
         <span class="n">rt_encode_output</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">rt_output_modalities</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;audio&quot;</span><span class="p">,</span> <span class="s2">&quot;text&quot;</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">rt_voice</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span>
             <span class="s2">&quot;alloy&quot;</span><span class="p">,</span>
             <span class="s2">&quot;ash&quot;</span><span class="p">,</span>
@@ -165,7 +167,9 @@ <h1>Source code for solana_agent.client.solana_agent</h1><div class="highlight">
         <span class="n">router</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">RoutingInterface</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">images</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">output_model</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Type</span><span class="p">[</span><span class="n">BaseModel</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AsyncGenerator</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">,</span> <span class="n">BaseModel</span><span class="p">],</span> <span class="kc">None</span><span class="p">]:</span>  <span class="c1"># pragma: no cover</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AsyncGenerator</span><span class="p">[</span>
+        <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">,</span> <span class="n">BaseModel</span><span class="p">,</span> <span class="n">RealtimeChunk</span><span class="p">],</span> <span class="kc">None</span>
+    <span class="p">]:</span>  <span class="c1"># pragma: no cover</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Process a user message (text or audio) and optional images, returning the response stream.</span>
 
 <span class="sd">        Args:</span>
@@ -179,6 +183,7 @@ <h1>Source code for solana_agent.client.solana_agent</h1><div class="highlight">
 <span class="sd">            vad: Whether to use voice activity detection (for audio input)</span>
 <span class="sd">            rt_encode_input: Whether to re-encode input audio for compatibility</span>
 <span class="sd">            rt_encode_output: Whether to re-encode output audio for compatibility</span>
+<span class="sd">            rt_output_modalities: Modalities to return in realtime (default both if None)</span>
 <span class="sd">            rt_voice: Voice to use for realtime audio output</span>
 <span class="sd">            audio_voice: Voice to use for audio output</span>
 <span class="sd">            audio_output_format: Audio output format</span>
@@ -199,6 +204,7 @@ <h1>Source code for solana_agent.client.solana_agent</h1><div class="highlight">
             <span class="n">vad</span><span class="o">=</span><span class="n">vad</span><span class="p">,</span>
             <span class="n">rt_encode_input</span><span class="o">=</span><span class="n">rt_encode_input</span><span class="p">,</span>
             <span class="n">rt_encode_output</span><span class="o">=</span><span class="n">rt_encode_output</span><span class="p">,</span>
+            <span class="n">rt_output_modalities</span><span class="o">=</span><span class="n">rt_output_modalities</span><span class="p">,</span>
             <span class="n">rt_voice</span><span class="o">=</span><span class="n">rt_voice</span><span class="p">,</span>
             <span class="n">audio_voice</span><span class="o">=</span><span class="n">audio_voice</span><span class="p">,</span>
             <span class="n">audio_output_format</span><span class="o">=</span><span class="n">audio_output_format</span><span class="p">,</span>

.doctrees/environment.pickle

@@ -223,9 +223,10 @@ This example will work using expo-audio on Android and iOS.
          rt_encode_input=True,
          rt_encode_output=True,
          rt_voice="marin",
+         rt_output_modalities=["audio"],
          output_format="audio",
-         audio_output_format="mp3",
          audio_input_format="m4a",
+         audio_output_format="mp3",
       ):
          yield chunk
 
@@ -240,6 +241,108 @@ This example will work using expo-audio on Android and iOS.
       },
    )
 
+Realtime Text Streaming
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Due to the overhead of the router (API call) - realtime only supports a single agent setup.
+
+Realtime uses MongoDB for memory so Zep is not needed.
+
+.. code-block:: python
+
+   from solana_agent import SolanaAgent
+
+   solana_agent = SolanaAgent(config=config)
+
+   async def generate():
+      async for chunk in solana_agent.process(
+         user_id="user123", 
+         message="What is the latest news on Solana?",
+         realtime=True,
+         rt_output_modalities=["text"],
+      ):
+         yield chunk
+
+Dual Modality Realtime Streaming
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Solana Agent now supports **dual modality realtime streaming**, allowing you to stream both audio and text simultaneously from a single realtime session. This enables rich conversational experiences where users can receive both voice responses and text transcripts in real-time.
+
+Features
+^^^^^^^^
+
+- **Simultaneous Audio & Text**: Stream both modalities from the same conversation
+- **Flexible Output**: Choose audio-only, text-only, or both modalities
+- **Real-time Demuxing**: Automatically separate audio and text streams
+- **Mobile Optimized**: Works seamlessly with compressed audio formats (MP4/MP3)
+- **Memory Efficient**: Smart buffering and streaming for optimal performance
+
+Mobile App Integration Example
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: python
+
+   from fastapi import UploadFile
+   from fastapi.responses import StreamingResponse
+   from solana_agent import SolanaAgent
+   from solana_agent.interfaces.providers.realtime import RealtimeChunk
+   import base64
+
+   solana_agent = SolanaAgent(config=config)
+
+   @app.post("/realtime/dual")
+   async def realtime_dual_endpoint(audio_file: UploadFile):
+      """
+      Dual modality (audio + text) realtime endpoint using Server-Sent Events (SSE).
+      Emits:
+         event: audio      (base64 encoded audio frames)
+         event: transcript (incremental text)
+      Notes:
+         - Do NOT set output_format when using both modalities.
+         - If only one modality is requested, plain str (text) or raw audio bytes may be yielded instead of RealtimeChunk.
+      """
+      audio_content = await audio_file.read()
+
+      async def event_stream():
+         async for chunk in solana_agent.process(
+               user_id="mobile_user",
+               message=audio_content,
+               realtime=True,
+               rt_encode_input=True,
+               rt_encode_output=True,
+               rt_output_modalities=["audio", "text"],
+               rt_voice="marin",
+               audio_input_format="mp4",
+               audio_output_format="mp3",
+               # Optionally lock transcription model (otherwise default is auto-selected):
+               # rt_transcription_model="gpt-4o-mini-transcribe",
+         ):
+               if isinstance(chunk, RealtimeChunk):
+                  if chunk.is_audio and chunk.audio_data:
+                     b64 = base64.b64encode(chunk.audio_data).decode("ascii")
+                     yield f"event: audio\ndata: {b64}\n\n"
+                  elif chunk.is_text and chunk.text_data:
+                     # Incremental transcript (not duplicated at finalize)
+                     yield f"event: transcript\ndata: {chunk.text_data}\n\n"
+                  continue
+               # (Defensive) fallback: if something else appears
+               if isinstance(chunk, bytes):
+                  b64 = base64.b64encode(chunk).decode("ascii")
+                  yield f"event: audio\ndata: {b64}\n\n"
+               elif isinstance(chunk, str):
+                  yield f"event: transcript\ndata: {chunk}\n\n"
+
+         yield "event: done\ndata: end\n\n"
+
+      return StreamingResponse(
+         event_stream(),
+         media_type="text/event-stream",
+         headers={
+               "Cache-Control": "no-store",
+               "Access-Control-Allow-Origin": "*",
+         },
+      )
+
 Image/Text Streaming
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~