initial webrtc with backend /frontend example

src/mic_rtc_streaming/backend/backend.py

@@ -0,0 +1,85 @@
+# backend/main.py
+import asyncio, logging, uuid
+
+from typing import List, Set
+
+from fastapi import FastAPI, WebSocket
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from aiortc import RTCPeerConnection, RTCSessionDescription, MediaStreamTrack
+
+# Global: desired packetization time in ms for Opus (change here to affect both backend and frontend)
+PTIME = 40
+
+app = FastAPI()
+
+# Allow CORS for frontend on localhost
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # You can restrict this to ["http://localhost:8501"] if you want
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+pcs: Set[RTCPeerConnection] = set()      # keep refs so they don’t GC early
+
+class Offer(BaseModel):
+    sdp: str
+    type: str
+
+@app.post("/offer")
+async def offer(offer: Offer):
+    logging.info("/offer endpoint called")
+
+    pc = RTCPeerConnection()  # No STUN needed for localhost
+    pcs.add(pc)
+    id_ = uuid.uuid4().hex[:8]
+    logging.info(f"{id_}: new PeerConnection")
+
+    @pc.on("track")
+    async def on_track(track: MediaStreamTrack):
+        logging.info(f"{id_}: track {track.kind} received")
+        try:
+            first = True
+            while True:
+                pkt = await track.recv()          # RTP audio frame (already decrypted)
+                pkt_bytes = bytes(pkt.planes[0])
+                if first:
+                    logging.info(
+                        f"{id_}: frame sample_rate={pkt.sample_rate}, channels={pkt.layout.channels}, format={pkt.format.name}"
+                    )
+                    logging.info(f"{id_}: received audio frame of len {len(pkt_bytes)} bytes") #received audio frame of len 23040 bytes
+                    first = False
+                # TODO: write to file, pipe to ASR, etc.
+        except Exception as e:
+            logging.error(f"{id_}: Exception in on_track: {e}")
+
+    # --- SDP negotiation ---
+    logging.info(f"{id_}: setting remote description")
+    await pc.setRemoteDescription(RTCSessionDescription(**offer.dict()))
+
+    logging.info(f"{id_}: creating answer")
+    answer = await pc.createAnswer()
+    sdp = answer.sdp
+    # Insert a=ptime using the global PTIME variable
+    ptime_line = f"a=ptime:{PTIME}"
+    if "a=sendrecv" in sdp:
+        sdp = sdp.replace("a=sendrecv", f"a=sendrecv\n{ptime_line}")
+    else:
+        sdp += f"\n{ptime_line}"
+    new_answer = RTCSessionDescription(sdp=sdp, type=answer.type)
+    await pc.setLocalDescription(new_answer)
+    logging.info(f"{id_}: sending answer with {ptime_line}")
+    return {"sdp": pc.localDescription.sdp,
+            "type": pc.localDescription.type}
+
+@app.on_event("shutdown")
+async def cleanup():
+    coros = [pc.close() for pc in pcs]
+    await asyncio.gather(*coros)
+
+if __name__ == "__main__":
+    import uvicorn
+    logging.basicConfig(level=logging.INFO)
+    uvicorn.run("backend:app", host="0.0.0.0", port=8000)

src/mic_rtc_streaming/frontend/frontend.py

@@ -0,0 +1,75 @@
+# frontend/app.py
+import streamlit as st
+import requests
+
+# Global: desired packetization time in ms for Opus (should match backend)
+PTIME = 40
+BACKEND_URL = "http://localhost:8000"
+
+def main():
+
+    st.title("🎙️ WebRTC mic → backend demo")
+    st.markdown("Click start and speak; watch your backend logs to see incoming RTP.")
+
+    component = f"""
+    <button id='go'>Start microphone</button>
+    <script>
+      const go = document.getElementById('go');
+      go.onclick = async () => {{
+        go.disabled = true;
+        const pc = new RTCPeerConnection(); // No STUN needed for localhost
+        const stream = await navigator.mediaDevices.getUserMedia({{audio:true}});
+        stream.getTracks().forEach(t => pc.addTrack(t, stream));
+
+        // --- WebRTC offer/answer exchange ---
+        const offer = await pc.createOffer()
+        // Patch SDP offer to include a=ptime using global PTIME
+        let sdp = offer.sdp;
+        const ptime_line = 'a=ptime:{PTIME}';
+        const maxptime_line = 'a=maxptime:{PTIME}';
+        if (sdp.includes('a=sendrecv')) {{
+          sdp = sdp.replace('a=sendrecv', 'a=sendrecv\\n' + ptime_line + '\\n' + maxptime_line);
+        }} else {{
+          sdp += '\\n' + ptime_line + '\\n' + maxptime_line;
+        }}
+        const patched_offer = new RTCSessionDescription({{sdp, type: offer.type}})
+        await pc.setLocalDescription(patched_offer)
+
+        // Send offer to backend
+        const response = await fetch(
+            "{BACKEND_URL}/offer",
+            {{
+                method: 'POST',
+                headers: {{'Content-Type':'application/json'}},
+                body: JSON.stringify({{sdp: pc.localDescription.sdp, type: pc.localDescription.type}})
+            }}
+        )
+        const answer = await response.json()
+        await pc.setRemoteDescription(new RTCSessionDescription({{sdp: answer.sdp, type: answer.type}}))
+      }};
+    </script>
+    """
+    st.components.v1.html(component, height=80)
+
+# if __name__ == "__main__":
+#     import sys
+#     import os
+#     from pathlib import Path
+
+#     # Prevent infinite relaunch loop if already running inside Streamlit
+#     if os.environ.get("STREAMLIT_SERVER_RUNNING") == "1":
+#         # Already running in Streamlit, do nothing special
+#         pass
+#     else:
+#         venv_bin = Path(__file__).resolve().parent.parent.parent.parent / ".venv/bin"
+#         streamlit_bin = venv_bin / "streamlit"
+#         if streamlit_bin.exists():
+#             cmd = f"{streamlit_bin} run {sys.argv[0]}"
+
+# Ensure Streamlit runs the UI code
+main()
+
+#         else:
+#             cmd = f"streamlit run {sys.argv[0]}"
+#         os.execvp("/bin/bash", ["/bin/bash", "-c", f"source {venv_bin}/activate && {cmd}"])
+

src/mic_rtc_streaming/run.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Kill any orphaned backend/frontend processes from previous runs
+pkill -f "uvicorn backend.backend:app" 2>/dev/null
+pkill -f "streamlit run frontend/frontend.py" 2>/dev/null
+
+# Start backend
+uvicorn backend.backend:app --host 0.0.0.0 --port 8000 &
+BACKEND_PID=$!
+
+# Start frontend
+streamlit run frontend/frontend.py &
+FRONTEND_PID=$!
+
+# Trap exit signals to kill both processes
+cleanup() {
+    echo "Stopping frontend (PID $FRONTEND_PID)..."
+    kill $FRONTEND_PID 2>/dev/null
+    echo "Stopping backend (PID $BACKEND_PID)..."
+    kill $BACKEND_PID 2>/dev/null
+    exit 0
+}
+trap cleanup SIGINT SIGTERM EXIT
+
+# Wait for both to finish
+wait