diff --git a/.env.example b/.env.example index 6937031..3a16af2 100644 --- a/.env.example +++ b/.env.example @@ -2,3 +2,18 @@ # https://app.supabase.com/project/_/settings/api NEXT_PUBLIC_SUPABASE_URL=your-project-url NEXT_PUBLIC_SUPABASE_ANON_KEY=your-anon-key + +# Signin and go to https://docs.livekit.io/agents/quickstart/ +LIVEKIT_API_KEY="" +LIVEKIT_API_SECRET="" +LIVEKIT_URL="" +NEXT_PUBLIC_LIVEKIT_URL="" + +OPENAI_API_KEY="" +OPENAI_ORG_ID="" + +ELEVEN_API_KEY="" +DEEPGRAM_API_KEY="" + +SUPABASE_URL="" +SUPABASE_KEY="" \ No newline at end of file diff --git a/bun.lockb b/bun.lockb index b8fee8a..66b15ea 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 59c9ed7..0e6f5e8 100644 --- a/package.json +++ b/package.json @@ -10,11 +10,18 @@ "tauri": "tauri" }, "dependencies": { + "@heroicons/react": "^2.1.3", + "@livekit/components-react": "^2.0.3", + "@radix-ui/react-dropdown-menu": "^2.0.6", "@radix-ui/react-icons": "^1.3.0", + "@radix-ui/react-menubar": "^1.0.4", + "@radix-ui/react-separator": "^1.0.3", "@radix-ui/react-slot": "^1.0.2", "@tauri-apps/api": "^1.5.3", "class-variance-authority": "^0.7.0", "clsx": "^2.1.0", + "framer-motion": "^11.0.22", + "livekit-client": "^2.0.10", "lucide-react": "^0.363.0", "react": "^18.2.0", "react-dom": "^18.2.0", diff --git a/server/livekit_agents/agent.py b/server/livekit_agents/agent.py index 7c922b6..954d01c 100644 --- a/server/livekit_agents/agent.py +++ b/server/livekit_agents/agent.py @@ -159,12 +159,12 @@ def reprompt(self, data, msg:str) -> str: for match in data: print("data", data) print("match", match) - contextText += f"Title of Demo Day Submission: {match["title"]}\ + contextText += f'Title of Demo Day Submission: {match["title"]}\ Niche: {match["niche"]}\ Summary: {match["description"]} \ Full Description: {match["youtube_transcript"]} \ Social: {match["social"]} \ - Buildspace Season: {match["season"]}" + Buildspace Season: {match["season"]}' contextText+=f"\nuser's message: {msg}" diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 644cb3b..e632af5 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -582,6 +582,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dtoa" version = "1.0.9" @@ -1708,6 +1714,7 @@ dependencies = [ name = "os1" version = "0.0.0" dependencies = [ + "dotenvy", "serde", "serde_json", "tauri", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index d1502c0..2d0a98a 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -14,6 +14,7 @@ tauri-build = { version = "1", features = [] } tauri = { version = "1", features = [ "shell-all", "system-tray"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +dotenvy = "0.15.7" [features] # This feature is used for production builds or when a dev server is not specified, DO NOT REMOVE!! diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index 82d1094..75d4dfd 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -8,8 +8,8 @@ use tauri::{ // Learn more about Tauri commands at https://tauri.app/v1/guides/features/command #[tauri::command] -fn greet(name: &str) -> String { - format!("Hello, {}! You've been greeted from Rust!", name) +fn get_env(name: &str) -> String { + std::env::var(String::from(name)).unwrap_or(String::from("")) } const links: [(&str, &str, &str); 2] = [ @@ -19,6 +19,8 @@ const links: [(&str, &str, &str); 2] = [ ]; fn main() { + dotenvy::dotenv().expect(".env file not found"); + let sub_menu_github = { let mut menu = SystemTrayMenu::new(); for (id, label, _url) in @@ -41,7 +43,7 @@ fn main() { let tray = SystemTray::new().with_menu(tray_menu); let mut app = tauri::Builder::default() - .invoke_handler(tauri::generate_handler![greet]) + .invoke_handler(tauri::generate_handler![get_env]) .system_tray(tray) .on_system_tray_event(on_system_tray_event) .build(tauri::generate_context!()) diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index ff417ce..1cef22b 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -6,7 +6,7 @@ "distDir": "../dist" }, "package": { - "productName": "os1", + "productName": "OS1", "version": "0.0.0" }, "tauri": { @@ -25,7 +25,7 @@ }, "windows": [ { - "title": "os1", + "title": "OS1", "width": 800, "height": 600 } diff --git a/src/App.tsx b/src/App.tsx index c84207e..e52ddfc 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,45 +1,135 @@ -import { useState } from "react"; -// import reactLogo from "./assets/react.svg"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { invoke } from "@tauri-apps/api/tauri"; -import { Input } from "./components/ui/input"; -import { Button } from "./components/ui/button"; -import { WelcomePage } from "./components/welcome"; +import { DynamicIsland, IslandState, IslandStates } from "./components/DynamicIsland"; +import { LiveKitRoom, RoomAudioRenderer, StartAudio, useToken } from "@livekit/components-react"; +import Playground, { PlaygroundOutputs } from './components/Playground'; +import { PlaygroundToast, ToastType } from './components/PlaygroundToast'; +import { generateRandomAlphanumeric } from './utils/livekit'; +import { AnimatePresence, motion } from "framer-motion"; +import { CallNavBar } from './components/CallNavbar'; +import { useAppConfig } from './hooks/useAppConfig'; +import { WelcomePage } from "./components/Welcome"; +import { tw } from "./utils/tw"; -function App() { - const [greetMsg, setGreetMsg] = useState(""); - const [name, setName] = useState(""); +export default function App() { + const [toastMessage, setToastMessage] = useState<{ + message: string; + type: ToastType; + } | null>(null); + const [shouldConnect, setShouldConnect] = useState(false); + const [roomName] = useState(createRoomName()); + const [liveKitUrl, setLiveKitUrl] = useState(); + + const [state, setState] = useState(IslandStates[0]) const onboarding = false - const greet=async()=> { - setGreetMsg(await invoke("greet", { name })); + const tokenOptions = useMemo(() => { + return { + userInfo: { identity: generateRandomAlphanumeric(16) }, + }; + }, []); + + // const token = useToken('/api/get-participant-token', roomName, tokenOptions); + const token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MTE1ODUwNDEsImlzcyI6IkFQSUJ2NmdzQU5lclhaWCIsIm5iZiI6MTcxMTU4NDE0MSwic3ViIjoiY2hhZCIsInZpZGVvIjp7ImNhblB1Ymxpc2giOnRydWUsImNhblB1Ymxpc2hEYXRhIjp0cnVlLCJjYW5TdWJzY3JpYmUiOnRydWUsInJvb20iOiJjaGFkIiwicm9vbUpvaW4iOnRydWV9fQ.gpPZPvtzCn9Yh8JGhz-Yub98WyWQ_1KxyQAgkFVJRcA" + + const get_env=async(name: string) => { + return await invoke("get_env", { name }); } + useEffect(()=>{ + get_env("NEXT_PUBLIC_LIVEKIT_URL").then((livekiturl) => { + setLiveKitUrl(livekiturl as string) + handleConnect(true) + }) + },[]) + + const appConfig = useAppConfig(); + const outputs = [ + appConfig?.outputs.audio && PlaygroundOutputs.Audio, + appConfig?.outputs.video && PlaygroundOutputs.Video, + appConfig?.outputs.chat && PlaygroundOutputs.Chat, + ].filter((item) => typeof item !== 'boolean') as PlaygroundOutputs[]; + + const handleConnect = useCallback((connect: boolean, opts?: { url: string; token: string }) => { + if (connect && opts) { + setLiveKitUrl(opts.url); + } + setShouldConnect(connect); + }, []); + if (onboarding){ return } + return ( -
-
-

OS1

-
{ - e.preventDefault(); - greet(); +
+ + {toastMessage && ( + + setToastMessage(null)} + /> + + )} + + + + + {liveKitUrl && ( + { + setToastMessage({ message: e.message, type: 'error' }); + console.error(e); }} > - setName(e.currentTarget.value)} - placeholder="Enter a name..." + + + + - - -

{greetMsg}

+ + )} + +
+ {IslandStates.map((curr_state)=>( + + ))}
-
+ ); } -export default App; +const createRoomName = () => { + return [generateRandomAlphanumeric(4), generateRandomAlphanumeric(4)].join('-'); +}; \ No newline at end of file diff --git a/src/components/AgentMultibandAudioVisualizer.tsx b/src/components/AgentMultibandAudioVisualizer.tsx new file mode 100644 index 0000000..d812734 --- /dev/null +++ b/src/components/AgentMultibandAudioVisualizer.tsx @@ -0,0 +1,105 @@ +import { AgentState } from '../utils/types'; +import { useEffect, useState } from 'react'; + +type AgentMultibandAudioVisualizerProps = { + state: AgentState; + barWidth: number; + minBarHeight: number; + maxBarHeight: number; + accentColor: string; + accentShade?: number; + frequencies: Float32Array[]; + borderRadius: number; + gap: number; +}; + +export const AgentMultibandAudioVisualizer = ({ + state, + barWidth, + minBarHeight, + maxBarHeight, + accentColor, + accentShade, + frequencies, + borderRadius, + gap, +}: AgentMultibandAudioVisualizerProps) => { + const summedFrequencies = frequencies.map((bandFrequencies) => { + const sum = bandFrequencies.reduce((a, b) => a + b, 0); + return Math.sqrt(sum / bandFrequencies.length); + }); + + const [thinkingIndex, setThinkingIndex] = useState(Math.floor(summedFrequencies.length / 2)); + const [thinkingDirection, setThinkingDirection] = useState<'left' | 'right'>('right'); + + useEffect(() => { + if (state !== 'thinking') { + setThinkingIndex(Math.floor(summedFrequencies.length / 2)); + return; + } + const timeout = setTimeout(() => { + if (thinkingDirection === 'right') { + if (thinkingIndex === summedFrequencies.length - 1) { + setThinkingDirection('left'); + setThinkingIndex((prev) => prev - 1); + } else { + setThinkingIndex((prev) => prev + 1); + } + } else { + if (thinkingIndex === 0) { + setThinkingDirection('right'); + setThinkingIndex((prev) => prev + 1); + } else { + setThinkingIndex((prev) => prev - 1); + } + } + }, 200); + + return () => clearTimeout(timeout); + }, [state, summedFrequencies.length, thinkingDirection, thinkingIndex]); + + return ( +
+ {summedFrequencies.map((frequency, index) => { + const isCenter = index === Math.floor(summedFrequencies.length / 2); + + let color = `${accentColor}-${accentShade}`; + let shadow = `shadow-lg-${accentColor}`; + let transform; + + if (state === 'listening' || state === 'idle') { + color = isCenter ? `${accentColor}-${accentShade}` : 'gray-950'; + shadow = !isCenter ? '' : shadow; + transform = !isCenter ? 'scale(1.0)' : 'scale(1.2)'; + } else if (state === 'speaking') { + color = `${accentColor}${accentShade ? '-' + accentShade : ''}`; + } else if (state === 'thinking') { + color = index === thinkingIndex ? `${accentColor}-${accentShade}` : 'gray-950'; + shadow = ''; + transform = thinkingIndex !== index ? 'scale(1)' : 'scale(1.1)'; + } + + return ( +
+ ); + })} +
+ ); +}; diff --git a/src/components/CallNavbar.tsx b/src/components/CallNavbar.tsx new file mode 100644 index 0000000..92e9491 --- /dev/null +++ b/src/components/CallNavbar.tsx @@ -0,0 +1,24 @@ +import { PlaygroundDeviceSelector } from './PlaygroundDeviceSelector'; +import { TrackToggle } from '@livekit/components-react'; +import { Menubar, MenubarMenu } from './ui/menubar'; +import { Track } from 'livekit-client'; + +export const CallNavBar = ({ + className, +}: { + className?: string; +}) => { + return ( + + + + + + + + + ); +}; diff --git a/src/components/ChatMessage.tsx b/src/components/ChatMessage.tsx new file mode 100644 index 0000000..b53e48a --- /dev/null +++ b/src/components/ChatMessage.tsx @@ -0,0 +1,34 @@ +type ChatMessageProps = { + message: string; + accentColor: string; + name: string; + isSelf: boolean; + }; + + export const ChatMessage = ({ + name, + message, + accentColor, + isSelf, + }: ChatMessageProps) => { + return ( +
+
+ {name} +
+
+ {message} +
+
+ ); + }; \ No newline at end of file diff --git a/src/components/ChatMessageInput.tsx b/src/components/ChatMessageInput.tsx new file mode 100644 index 0000000..df5a665 --- /dev/null +++ b/src/components/ChatMessageInput.tsx @@ -0,0 +1,123 @@ +import { useWindowResize } from "../hooks/useWindowResize"; +import { useCallback, useEffect, useRef, useState } from "react"; + +type ChatMessageInput = { + placeholder: string; + accentColor: string; + height: number; + onSend?: (message: string) => void; +}; + +export const ChatMessageInput = ({ + placeholder, + accentColor, + height, + onSend, +}: ChatMessageInput) => { + const [message, setMessage] = useState(""); + const [inputTextWidth, setInputTextWidth] = useState(0); + const [inputWidth, setInputWidth] = useState(0); + const hiddenInputRef = useRef(null); + const inputRef = useRef(null); + const windowSize = useWindowResize(); + const [isTyping, setIsTyping] = useState(false); + const [inputHasFocus, setInputHasFocus] = useState(false); + + const handleSend = useCallback(() => { + if (!onSend) { + return; + } + if (message === "") { + return; + } + + onSend(message); + setMessage(""); + }, [onSend, message]); + + useEffect(() => { + setIsTyping(true); + const timeout = setTimeout(() => { + setIsTyping(false); + }, 500); + + return () => clearTimeout(timeout); + }, [message]); + + useEffect(() => { + if (hiddenInputRef.current) { + setInputTextWidth(hiddenInputRef.current.clientWidth); + } + }, [hiddenInputRef, message]); + + useEffect(() => { + if (inputRef.current) { + setInputWidth(inputRef.current.clientWidth); + } + }, [hiddenInputRef, message, windowSize.width]); + + return ( +
+
+
0 + ? Math.min(inputTextWidth, inputWidth - 20) - 4 + : 0) + + "px)", + }} + >
+ 0 ? "12px" : "24px", + caretShape: "block", + }} + placeholder={placeholder} + value={message} + onChange={(e) => { + setMessage(e.target.value); + }} + onFocus={() => { + setInputHasFocus(true); + }} + onBlur={() => { + setInputHasFocus(false); + }} + onKeyDown={(e) => { + if (e.key === "Enter") { + handleSend(); + } + }} + > + + {/* @ts-ignore */} + {message.replaceAll(" ", "\u00a0")} + + +
+
+ ); +}; \ No newline at end of file diff --git a/src/components/ChatTile.tsx b/src/components/ChatTile.tsx new file mode 100644 index 0000000..9d4f012 --- /dev/null +++ b/src/components/ChatTile.tsx @@ -0,0 +1,58 @@ +import { ChatMessage } from "./ChatMessage"; +import { ChatMessageInput } from "./ChatMessageInput"; +import { ChatMessage as ComponentsChatMessage } from "@livekit/components-react"; +import { useEffect, useRef } from "react"; + +const inputHeight = 48; + +export type ChatMessageType = { + name: string; + message: string; + isSelf: boolean; + timestamp: number; +}; + +type ChatTileProps = { + messages: ChatMessageType[]; + accentColor: string; + onSend?: (message: string) => Promise; +}; + +export const ChatTile = ({ messages, accentColor, onSend }: ChatTileProps) => { + const containerRef = useRef(null); + useEffect(() => { + if (containerRef.current) { + containerRef.current.scrollTop = containerRef.current.scrollHeight; + } + }, [containerRef, messages]); + + return ( +
+
+
+ {messages.map((message, index) => ( + + ))} +
+
+ +
+ ); +}; \ No newline at end of file diff --git a/src/components/DynamicIsland.tsx b/src/components/DynamicIsland.tsx new file mode 100644 index 0000000..e8aba2a --- /dev/null +++ b/src/components/DynamicIsland.tsx @@ -0,0 +1,67 @@ +import {tw} from "../utils/tw" +import { motion } from "framer-motion"; + +export const IslandStates = ["default", "state_1", "state_2", "state_3"] as const +export type IslandState = typeof IslandStates[number] + +const variants : Record = { + "default": { + width: "100px", + height: "30px", + transition: { + ease: "easeInOut", + duration: 0.25, + property: "all" + } + }, + "state_1": { + width: "300px", + height: "150px", + borderRadius: "40px", + transition: { + ease: "easeInOut", + duration: 0.25, + property: "all" + } + }, + "state_2": { + width: "300px", + height: "50px", + borderRadius: "20px", + transition: { + ease: "easeInOut", + duration: 0.25, + property: "all" + } + }, + "state_3": { + width: "100px", + height: "50px", + borderRadius: "20px", + transition: { + ease: "easeInOut", + duration: 0.25, + property: "all" + } + }, +}; + +const stateStyles : Record = { + "default": "align-center", + "state_1": "align-start", + "state_2": "align-start p-[8px_16px]", + "state_3": "align-start p-[8px_16px]", +} + +export const DynamicIsland = ({state}: {state: IslandState}) => { + + return ( + + {/* put your react component here */} + + ); +}; \ No newline at end of file diff --git a/src/components/LoadingSVG.tsx b/src/components/LoadingSVG.tsx new file mode 100644 index 0000000..1e1b666 --- /dev/null +++ b/src/components/LoadingSVG.tsx @@ -0,0 +1,34 @@ +export const LoadingSVG = ({ + diameter = 20, + strokeWidth = 4, +}: { + diameter?: number; + strokeWidth?: number; +}) => ( + + + + + // + // + // +); diff --git a/src/components/NameValueRow.tsx b/src/components/NameValueRow.tsx new file mode 100644 index 0000000..e241966 --- /dev/null +++ b/src/components/NameValueRow.tsx @@ -0,0 +1,17 @@ +type NameValueRowProps = { + name: string; + value?: React.ReactNode; + valueColor?: string; +}; + +export const NameValueRow: React.FC = ({ + value, + valueColor = 'gray-300', +}) => { + return ( +
+ {/*
{name}
*/} +
{value}
+
+ ); +}; diff --git a/src/components/Playground.tsx b/src/components/Playground.tsx new file mode 100644 index 0000000..35cccc7 --- /dev/null +++ b/src/components/Playground.tsx @@ -0,0 +1,274 @@ +import { LoadingSVG } from './LoadingSVG'; +import { NameValueRow } from './NameValueRow'; +import { PlaygroundTile } from './PlaygroundTile'; +import { AgentMultibandAudioVisualizer } from './AgentMultibandAudioVisualizer'; +import { useMultibandTrackVolume } from '../hooks/useTrackVolume'; +import { AgentState } from '../utils/types'; +import { + TrackReference, + VideoTrack, + useChat, + useConnectionState, + useDataChannel, + useLocalParticipant, + useRemoteParticipants, + useTracks, +} from '@livekit/components-react'; +import { ConnectionState, LocalParticipant, RoomEvent, Track } from 'livekit-client'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { Separator } from './ui/separator'; +import { ChatMessageType, ChatTile } from './ChatTile'; + +export enum PlaygroundOutputs { + Video, + Audio, + Chat, +} + +export interface PlaygroundProps { + themeColor: string; + outputs?: PlaygroundOutputs[]; + videoFit?: 'contain' | 'cover'; +} + +const headerHeight = 56; + +export default function Playground({ outputs, themeColor, videoFit }: PlaygroundProps) { + const [agentState, setAgentState] = useState('offline'); + const [transcripts, setTranscripts] = useState([]); + const { send: sendChat, chatMessages } = useChat(); + const [messages, setMessages] = useState([]); + const { localParticipant } = useLocalParticipant(); + + const participants = useRemoteParticipants({ + updateOnlyOn: [RoomEvent.ParticipantMetadataChanged], + }); + const agentParticipant = participants.find((p) => p.isAgent); + + const visualizerState = useMemo(() => { + if (agentState === 'thinking') { + return 'thinking'; + } else if (agentState === 'speaking') { + return 'talking'; + } + return 'idle'; + }, [agentState]); + + const roomState = useConnectionState(); + const tracks = useTracks(); + + const agentAudioTrack = tracks.find( + (trackRef) => trackRef.publication.kind === Track.Kind.Audio && trackRef.participant.isAgent, + ); + + const agentVideoTrack = tracks.find( + (trackRef) => trackRef.publication.kind === Track.Kind.Video && trackRef.participant.isAgent, + ); + + const subscribedVolumes = useMultibandTrackVolume(agentAudioTrack?.publication.track, 5); + + const localTracks = tracks.filter(({ participant }) => participant instanceof LocalParticipant); + const localVideoTrack = localTracks.find(({ source }) => source === Track.Source.Camera); + const localMicTrack = localTracks.find(({ source }) => source === Track.Source.Microphone); + + const localMultibandVolume = useMultibandTrackVolume(localMicTrack?.publication.track, 20); + + useEffect(() => { + const allMessages = [...transcripts]; + for (const msg of chatMessages) { + const isAgent = msg.from?.identity === agentParticipant?.identity; + const isSelf = msg.from?.identity === localParticipant?.identity; + let name = msg.from?.name; + if (!name) { + if (isAgent) { + name = "Agent"; + } else if (isSelf) { + name = "You"; + } else { + name = "Unknown"; + } + } + allMessages.push({ + name, + message: msg.message, + timestamp: msg?.timestamp, + isSelf: isSelf, + }); + } + allMessages.sort((a, b) => a.timestamp - b.timestamp); + setMessages(allMessages); + }, [transcripts, chatMessages, localParticipant, agentParticipant]); + + const isAgentConnected = agentState !== 'offline'; + + const onDataReceived = useCallback( + (msg: any) => { + if (msg.topic === 'transcription') { + const decoded = JSON.parse(new TextDecoder('utf-8').decode(msg.payload)); + let timestamp = new Date().getTime(); + if ('timestamp' in decoded && decoded.timestamp > 0) { + timestamp = decoded.timestamp; + } + setTranscripts([ + ...transcripts, + { + name: 'You', + message: decoded.text, + timestamp: timestamp, + isSelf: true, + }, + ]); + } + }, + [transcripts], + ); + + useDataChannel(onDataReceived); + + const chatTileContent = useMemo(() => { + return ( + + ); + }, [messages, themeColor, sendChat]); + + const mixedMediaContent = useMemo(() => { + return ( + + ); + }, [agentAudioTrack, subscribedVolumes, themeColor, agentState, agentVideoTrack, videoFit]); + + return ( + <> +
+ + +
+ + ) : ( + roomState + ) + } + valueColor={ + roomState === ConnectionState.Connected ? `${themeColor}-500` : 'gray-500' + } + /> + + + ) : ( + 'false' + ) + } + valueColor={isAgentConnected ? `${themeColor}-500` : 'gray-500'} + /> + + + + {agentState} +
+ ) : ( + agentState + ) + } + valueColor={agentState === 'speaking' ? `${themeColor}-500` : 'gray-500'} + /> +
+
+ } + > + {mixedMediaContent} + + + {chatTileContent} + + + + ); +} + +const MixedMedia = ({ + agentVideoTrack, + agentAudioTrack, + agentState, + videoFit, + subscribedVolumes, + themeColor, +}: { + themeColor: string; + agentVideoTrack?: TrackReference; + subscribedVolumes: Float32Array[]; + agentAudioTrack?: TrackReference; + agentState: AgentState; + videoFit: PlaygroundProps['videoFit']; +}) => { + if (agentVideoTrack) { + const videoFitClassName = `object-${videoFit}`; + return ( +
+ +
+ ); + } else if (agentAudioTrack) { + return ( +
+ +
+ ); + } + return ( +
+
+ + waiting for audio / video from buildspace AI +
+
+ ); +}; diff --git a/src/components/PlaygroundDeviceSelector.tsx b/src/components/PlaygroundDeviceSelector.tsx new file mode 100644 index 0000000..e86a9e3 --- /dev/null +++ b/src/components/PlaygroundDeviceSelector.tsx @@ -0,0 +1,91 @@ +import { useMediaDeviceSelect } from '@livekit/components-react'; +import { useEffect, useState } from 'react'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from './ui/dropdown-menu'; + +type PlaygroundDeviceSelectorProps = { + kind: MediaDeviceKind; +}; + +export const PlaygroundDeviceSelector = ({ kind }: PlaygroundDeviceSelectorProps) => { + const [showMenu, setShowMenu] = useState(false); + const deviceSelect = useMediaDeviceSelect({ kind: kind }); + const [selectedDeviceName, setSelectedDeviceName] = useState(''); + + useEffect(() => { + deviceSelect?.devices?.forEach((device) => { + if (device.deviceId === deviceSelect.activeDeviceId) { + setSelectedDeviceName(device.label); + } + }); + }, [deviceSelect.activeDeviceId, deviceSelect.devices, selectedDeviceName]); + + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if (showMenu) { + setShowMenu(false); + } + }; + document.addEventListener('click', handleClickOutside); + return () => { + document.removeEventListener('click', handleClickOutside); + }; + }, [showMenu]); + + return ( + + + + + + Audio Device + + {deviceSelect?.devices?.map((device, index) => ( + { + deviceSelect.setActiveMediaDevice(device.deviceId); + setShowMenu(false); + }} + className={`${ + device.deviceId === deviceSelect.activeDeviceId + ? 'text-white font-semibold' + : 'text-gray-500' + }`} + key={index} + > + {device.label} + + ))} + + + ); +}; + +const ChevronSVG = () => ( + + + +); diff --git a/src/components/PlaygroundTile.tsx b/src/components/PlaygroundTile.tsx new file mode 100644 index 0000000..62c714e --- /dev/null +++ b/src/components/PlaygroundTile.tsx @@ -0,0 +1,99 @@ +import { ReactNode, useState } from 'react'; + +const titleHeight = 32; + +type PlaygroundTileProps = { + children?: ReactNode; + className?: string; + childrenClassName?: string; + padding?: boolean; + status?: ReactNode; + backgroundColor?: string; +}; + +export type PlaygroundTab = { + title: string; + content: ReactNode; +}; + +export type PlaygroundTabbedTileProps = { + tabs: PlaygroundTab[]; + initialTab?: number; +} & PlaygroundTileProps; + +export const PlaygroundTile: React.FC = ({ + children, + className, + childrenClassName, + padding = true, + status, + backgroundColor = 'transparent', +}) => { + const contentPadding = padding ? 4 : 0; + return ( +
+ {status && ( +
+
{status}
+
+ )} +
+ {children} +
+
+ ); +}; + +export const PlaygroundTabbedTile: React.FC = ({ + tabs, + initialTab = 0, + className, + childrenClassName, + backgroundColor = 'transparent', +}) => { + const contentPadding = 4; + const [activeTab, setActiveTab] = useState(initialTab); + return ( +
+
+ {tabs.map((tab, index) => ( + + ))} +
+
+ {tabs[activeTab].content} +
+
+ ); +}; diff --git a/src/components/PlaygroundToast.tsx b/src/components/PlaygroundToast.tsx new file mode 100644 index 0000000..e278a3a --- /dev/null +++ b/src/components/PlaygroundToast.tsx @@ -0,0 +1,36 @@ +export type ToastType = 'error' | 'success' | 'info'; +export type ToastProps = { + message: string; + type: ToastType; + onDismiss: () => void; +}; + +export const PlaygroundToast = ({ message, type, onDismiss }: ToastProps) => { + const color = type === 'error' ? 'red' : type === 'success' ? 'green' : 'amber'; + return ( +
+ + {message} +
+ ); +}; diff --git a/src/components/ui/dropdown-menu.tsx b/src/components/ui/dropdown-menu.tsx new file mode 100644 index 0000000..4dc1191 --- /dev/null +++ b/src/components/ui/dropdown-menu.tsx @@ -0,0 +1,184 @@ +import * as React from 'react'; +import * as DropdownMenuPrimitive from '@radix-ui/react-dropdown-menu'; +import { Check, ChevronRight, Circle } from 'lucide-react'; +import { tw } from '../../utils/tw'; + +const DropdownMenu = DropdownMenuPrimitive.Root; + +const DropdownMenuTrigger = DropdownMenuPrimitive.Trigger; + +const DropdownMenuGroup = DropdownMenuPrimitive.Group; + +const DropdownMenuPortal = DropdownMenuPrimitive.Portal; + +const DropdownMenuSub = DropdownMenuPrimitive.Sub; + +const DropdownMenuRadioGroup = DropdownMenuPrimitive.RadioGroup; + +const DropdownMenuSubTrigger = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & { + inset?: boolean; + } +>(({ className, inset, children, ...props }, ref) => ( + + {children} + + +)); +DropdownMenuSubTrigger.displayName = DropdownMenuPrimitive.SubTrigger.displayName; + +const DropdownMenuSubContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +DropdownMenuSubContent.displayName = DropdownMenuPrimitive.SubContent.displayName; + +const DropdownMenuContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, sideOffset = 4, ...props }, ref) => ( + + + +)); +DropdownMenuContent.displayName = DropdownMenuPrimitive.Content.displayName; + +const DropdownMenuItem = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & { + inset?: boolean; + } +>(({ className, inset, ...props }, ref) => ( + +)); +DropdownMenuItem.displayName = DropdownMenuPrimitive.Item.displayName; + +const DropdownMenuCheckboxItem = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, checked, ...props }, ref) => ( + + + + + + + {children} + +)); +DropdownMenuCheckboxItem.displayName = DropdownMenuPrimitive.CheckboxItem.displayName; + +const DropdownMenuRadioItem = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + + + + + + + {children} + +)); +DropdownMenuRadioItem.displayName = DropdownMenuPrimitive.RadioItem.displayName; + +const DropdownMenuLabel = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & { + inset?: boolean; + } +>(({ className, inset, ...props }, ref) => ( + +)); +DropdownMenuLabel.displayName = DropdownMenuPrimitive.Label.displayName; + +const DropdownMenuSeparator = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +DropdownMenuSeparator.displayName = DropdownMenuPrimitive.Separator.displayName; + +const DropdownMenuShortcut = ({ className, ...props }: React.HTMLAttributes) => { + return ( + + ); +}; +DropdownMenuShortcut.displayName = 'DropdownMenuShortcut'; + +export { + DropdownMenu, + DropdownMenuTrigger, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuCheckboxItem, + DropdownMenuRadioItem, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuShortcut, + DropdownMenuGroup, + DropdownMenuPortal, + DropdownMenuSub, + DropdownMenuSubContent, + DropdownMenuSubTrigger, + DropdownMenuRadioGroup, +}; diff --git a/src/components/ui/menubar.tsx b/src/components/ui/menubar.tsx new file mode 100644 index 0000000..c61feed --- /dev/null +++ b/src/components/ui/menubar.tsx @@ -0,0 +1,218 @@ +import * as React from 'react'; +import * as MenubarPrimitive from '@radix-ui/react-menubar'; +import { Check, ChevronRight, Circle } from 'lucide-react'; +import { tw } from '../../utils/tw'; + +const MenubarMenu = MenubarPrimitive.Menu; + +const MenubarGroup = MenubarPrimitive.Group; + +const MenubarPortal = MenubarPrimitive.Portal; + +const MenubarSub = MenubarPrimitive.Sub; + +const MenubarRadioGroup = MenubarPrimitive.RadioGroup; + +const Menubar = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +Menubar.displayName = MenubarPrimitive.Root.displayName; + +const MenubarTrigger = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +MenubarTrigger.displayName = MenubarPrimitive.Trigger.displayName; + +const MenubarSubTrigger = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & { + inset?: boolean; + } +>(({ className, inset, children, ...props }, ref) => ( + + {children} + + +)); +MenubarSubTrigger.displayName = MenubarPrimitive.SubTrigger.displayName; + +const MenubarSubContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +MenubarSubContent.displayName = MenubarPrimitive.SubContent.displayName; + +const MenubarContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, align = 'start', alignOffset = -4, sideOffset = 8, ...props }, ref) => ( + + + +)); +MenubarContent.displayName = MenubarPrimitive.Content.displayName; + +const MenubarItem = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & { + inset?: boolean; + } +>(({ className, inset, ...props }, ref) => ( + +)); +MenubarItem.displayName = MenubarPrimitive.Item.displayName; + +const MenubarCheckboxItem = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, checked, ...props }, ref) => ( + + + + + + + {children} + +)); +MenubarCheckboxItem.displayName = MenubarPrimitive.CheckboxItem.displayName; + +const MenubarRadioItem = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + + + + + + + {children} + +)); +MenubarRadioItem.displayName = MenubarPrimitive.RadioItem.displayName; + +const MenubarLabel = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & { + inset?: boolean; + } +>(({ className, inset, ...props }, ref) => ( + +)); +MenubarLabel.displayName = MenubarPrimitive.Label.displayName; + +const MenubarSeparator = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)); +MenubarSeparator.displayName = MenubarPrimitive.Separator.displayName; + +const MenubarShortcut = ({ className, ...props }: React.HTMLAttributes) => { + return ( + + ); +}; +MenubarShortcut.displayname = 'MenubarShortcut'; + +export { + Menubar, + MenubarMenu, + MenubarTrigger, + MenubarContent, + MenubarItem, + MenubarSeparator, + MenubarLabel, + MenubarCheckboxItem, + MenubarRadioGroup, + MenubarRadioItem, + MenubarPortal, + MenubarSubContent, + MenubarSubTrigger, + MenubarGroup, + MenubarSub, + MenubarShortcut, +}; diff --git a/src/components/ui/separator.tsx b/src/components/ui/separator.tsx new file mode 100644 index 0000000..e32693e --- /dev/null +++ b/src/components/ui/separator.tsx @@ -0,0 +1,23 @@ +import * as React from 'react'; +import * as SeparatorPrimitive from '@radix-ui/react-separator'; +import { tw } from '../../utils/tw'; + +const Separator = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, orientation = 'horizontal', decorative = true, ...props }, ref) => ( + +)); +Separator.displayName = SeparatorPrimitive.Root.displayName; + +export { Separator }; diff --git a/src/hooks/useAppConfig.tsx b/src/hooks/useAppConfig.tsx new file mode 100644 index 0000000..9669bda --- /dev/null +++ b/src/hooks/useAppConfig.tsx @@ -0,0 +1,29 @@ +export type AppConfig = { + theme_color: string; + video_fit: 'cover' | 'contain'; + outputs: { + audio: boolean; + video: boolean; + chat: boolean; + }; + inputs: { + mic: boolean; + camera: boolean; + }; +}; + +export const useAppConfig = (): AppConfig => { + return { + theme_color: 'blue', + video_fit: 'cover', + outputs: { + audio: true, + video: true, + chat: false, + }, + inputs: { + mic: true, + camera: false, + }, + }; +}; diff --git a/src/hooks/useTrackVolume.tsx b/src/hooks/useTrackVolume.tsx new file mode 100644 index 0000000..61416cf --- /dev/null +++ b/src/hooks/useTrackVolume.tsx @@ -0,0 +1,110 @@ +import { Track } from 'livekit-client'; +import { useEffect, useState } from 'react'; + +export const useTrackVolume = (track?: Track) => { + const [volume, setVolume] = useState(0); + useEffect(() => { + if (!track || !track.mediaStream) { + return; + } + + const ctx = new AudioContext(); + const source = ctx.createMediaStreamSource(track.mediaStream); + const analyser = ctx.createAnalyser(); + analyser.fftSize = 32; + analyser.smoothingTimeConstant = 0; + source.connect(analyser); + + const bufferLength = analyser.frequencyBinCount; + const dataArray = new Uint8Array(bufferLength); + + const updateVolume = () => { + analyser.getByteFrequencyData(dataArray); + let sum = 0; + for (let i = 0; i < dataArray.length; i++) { + const a = dataArray[i]; + sum += a * a; + } + setVolume(Math.sqrt(sum / dataArray.length) / 255); + }; + + const interval = setInterval(updateVolume, 1000 / 30); + + return () => { + source.disconnect(); + clearInterval(interval); + }; + }, [track, track?.mediaStream]); + + return volume; +}; + +const normalizeFrequencies = (frequencies: Float32Array) => { + const normalizeDb = (value: number) => { + const minDb = -100; + const maxDb = -10; + let db = 1 - (Math.max(minDb, Math.min(maxDb, value)) * -1) / 100; + db = Math.sqrt(db); + + return db; + }; + + // Normalize all frequency values + return frequencies.map((value) => { + if (value === -Infinity) { + return 0; + } + return normalizeDb(value); + }); +}; + +export const useMultibandTrackVolume = ( + track?: Track, + bands: number = 5, + loPass: number = 100, + hiPass: number = 600, +) => { + const [frequencyBands, setFrequencyBands] = useState([]); + + useEffect(() => { + if (!track || !track.mediaStream) { + return; + } + + const ctx = new AudioContext(); + const source = ctx.createMediaStreamSource(track.mediaStream); + const analyser = ctx.createAnalyser(); + analyser.fftSize = 2048; + source.connect(analyser); + + const bufferLength = analyser.frequencyBinCount; + const dataArray = new Float32Array(bufferLength); + + const updateVolume = () => { + analyser.getFloatFrequencyData(dataArray); + let frequencies: Float32Array = new Float32Array(dataArray.length); + for (let i = 0; i < dataArray.length; i++) { + frequencies[i] = dataArray[i]; + } + frequencies = frequencies.slice(loPass, hiPass); + + const normalizedFrequencies = normalizeFrequencies(frequencies); + const chunkSize = Math.ceil(normalizedFrequencies.length / bands); + const chunks: Float32Array[] = []; + for (let i = 0; i < bands; i++) { + chunks.push(normalizedFrequencies.slice(i * chunkSize, (i + 1) * chunkSize)); + } + + setFrequencyBands(chunks); + }; + + const interval = setInterval(updateVolume, 10); + + return () => { + source.disconnect(); + clearInterval(interval); + }; + }, [track, track?.mediaStream, loPass, hiPass, bands]); + + return frequencyBands; +}; diff --git a/src/hooks/useWindowResize.ts b/src/hooks/useWindowResize.ts new file mode 100644 index 0000000..d9e959a --- /dev/null +++ b/src/hooks/useWindowResize.ts @@ -0,0 +1,27 @@ +import { useEffect, useState } from 'react'; + +export const useWindowResize = () => { + const [size, setSize] = useState({ + width: 0, + height: 0, + }); + + useEffect(() => { + const handleResize = () => { + setSize({ + width: window.innerWidth, + height: window.innerHeight, + }); + }; + + handleResize(); + + window.addEventListener('resize', handleResize); + + return () => { + window.removeEventListener('resize', handleResize); + }; + }, []); + + return size; +}; diff --git a/src/utils/livekit.ts b/src/utils/livekit.ts new file mode 100644 index 0000000..47659eb --- /dev/null +++ b/src/utils/livekit.ts @@ -0,0 +1,12 @@ +export function generateRandomAlphanumeric(length: number): string { + const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + let result = ''; + const charactersLength = characters.length; + + for (let i = 0; i < length; i++) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)); + } + + return result; + } + \ No newline at end of file diff --git a/src/utils/types.ts b/src/utils/types.ts new file mode 100644 index 0000000..a828b2e --- /dev/null +++ b/src/utils/types.ts @@ -0,0 +1,18 @@ +import { LocalAudioTrack, LocalVideoTrack } from 'livekit-client'; + +export interface SessionProps { + roomName: string; + identity: string; + audioTrack?: LocalAudioTrack; + videoTrack?: LocalVideoTrack; + region?: string; + turnServer?: RTCIceServer; + forceRelay?: boolean; +} + +export interface TokenResult { + identity: string; + accessToken: string; +} + +export type AgentState = 'idle' | 'listening' | 'speaking' | 'thinking' | 'offline' | 'starting';