not perfect, 'just ship' (#10)

Dockerfile

@@ -33,4 +33,4 @@ RUN python -m pip install -r requirements.txt
 USER appuser
 
 # Run the application.
-CMD python kitt.py start
+CMD python agent.py start

requirements.txt

@@ -1,6 +1,45 @@
-livekit>=0.9.0
-livekit-api
-livekit-agents>=0.4.0
-livekit-plugins-deepgram>=0.2.0
-livekit-plugins-elevenlabs>=0.2.0
-openai
+aiohttp==3.9.3
+aiosignal==1.3.1
+annotated-types==0.6.0
+anyio==4.3.0
+attrs==23.2.0
+certifi==2024.2.2
+click==8.1.7
+deprecation==2.1.0
+distro==1.9.0
+frozenlist==1.4.1
+gotrue==2.4.1
+h11==0.14.0
+httpcore==1.0.4
+httpx==0.25.2
+idna==3.6
+livekit==0.9.1
+livekit-agents==0.4.0
+livekit-api==0.4.2
+livekit-plugins-deepgram==0.2.0
+livekit-plugins-elevenlabs==0.2.0
+livekit-protocol==0.3.1
+multidict==6.0.5
+openai==1.12.0
+packaging==23.2
+postgrest==0.16.1
+protobuf==4.25.3
+pydantic==2.6.2
+pydantic_core==2.16.3
+PyJWT==2.8.0
+python-dateutil==2.9.0
+python-dotenv==1.0.1
+realtime==1.0.2
+setuptools==68.2.2
+six==1.16.0
+sniffio==1.3.0
+storage3==0.7.0
+StrEnum==0.4.15
+supabase==2.4.0
+supafunc==0.3.3
+tqdm==4.66.2
+types-protobuf==4.24.0.20240129
+typing_extensions==4.9.0
+websockets==11.0.3
+wheel==0.41.2
+yarl==1.9.4

save_embeddings_to_db.py

@@ -1,3 +1,4 @@
+import inspect
 import os
 import json
 from supabase import create_client, Client
@@ -17,5 +18,5 @@
 for submission in s3_data:
     submission["season"]="3"
 
-
 data, count = supabase.table("demoday_submission").upsert(s3_data).execute()
+

schema.sql

@@ -1,3 +1,5 @@
+-- https://supabase.com/blog/openai-embeddings-postgres-vector ❤️
+
 -- one to one with DemoDaySubmission struct
 create table demoday_submission (
   id serial primary key,
@@ -13,4 +15,46 @@ create table demoday_submission (
 
 -- requests are only going to made on the server at the moment
 -- might change later
-alter table demoday_submission enable row level security;
+alter table demoday_submission enable row level security;
+
+
+-- 
+create index on demoday_submission using ivfflat (embedding vector_cosine_ops)
+with
+  (lists = 500);
+
+
+-- function
+create or replace function match_person (
+  query_embedding vector(1536),
+  match_threshold float,
+  match_count int
+)
+returns table (
+  id bigint,
+  title text,
+  niche text,
+  description text,
+  youtube_url text,
+  youtube_transcript text,
+  social text,
+  season text,
+  similarity float
+)
+language sql stable
+as $$
+  select
+    demoday_submission.id,
+    demoday_submission.title,
+    demoday_submission.niche,
+    demoday_submission.description,
+    demoday_submission.youtube_url,
+    demoday_submission.youtube_transcript,
+    demoday_submission.social,
+    demoday_submission.season,
+    1 - (demoday_submission.embedding <=> query_embedding) as similarity
+  from demoday_submission
+  where demoday_submission.embedding <=> query_embedding < 1 - match_threshold
+  order by demoday_submission.embedding <=> query_embedding
+  limit match_count;
+$$;

server/agent.py

@@ -16,12 +16,15 @@
 import asyncio
 from datetime import datetime
 from enum import Enum
+import inspect
 import json
 import logging
+import os
 from typing import AsyncIterable
 
 from livekit import rtc, agents
 from livekit.agents.tts import SynthesisEvent, SynthesisEventType
+from supabase import Client, create_client
 from llm import (
     ChatGPTMessage,
     ChatGPTMessageRole,
@@ -30,15 +33,14 @@
 from livekit.plugins.deepgram import STT
 from livekit.plugins.elevenlabs import TTS
 
-PROMPT = "You are KITT, a friendly voice assistant powered by LiveKit.  \
+PROMPT = "You are Buildspace AI, a friendly voice assistant that connects buildspace members together. \
           Conversation should be personable, and be sure to ask follow up questions. \
           If your response is a question, please append a question mark symbol to the end of it.\
           Don't respond with more than a few sentences."
-INTRO = "Hello, I am KITT, a friendly voice assistant powered by LiveKit Agents. \
-        You can find my source code in the top right of this screen if you're curious how I work. \
-        Feel free to ask me anything — I'm here to help! Just start talking or type in the chat."
-SIP_INTRO = "Hello, I am KITT, a friendly voice assistant powered by LiveKit Agents. \
-             Feel free to ask me anything — I'm here to help! Just start talking."
+
+INTRO = "Hey, I'm Buildspace AI, what's your name?."
+
+SIP_INTRO = "Hey, I'm Buildspace AI, what's your name?.."
 
 
 
@@ -58,11 +60,11 @@ async def intro_text_stream(sip: bool):
 ELEVEN_TTS_CHANNELS = 1
 
 
-class KITT:
+class BuildspaceAI:
     @classmethod
     async def create(cls, ctx: agents.JobContext):
-        kitt = KITT(ctx)
-        await kitt.start()
+        buildspace = BuildspaceAI(ctx)
+        await buildspace.start()
 
     def __init__(self, ctx: agents.JobContext):
         # plugins
@@ -76,6 +78,9 @@ def __init__(self, ctx: agents.JobContext):
             model_id="eleven_turbo_v2", sample_rate=ELEVEN_TTS_SAMPLE_RATE
         )
 
+        url: str = os.environ.get("SUPABASE_URL")
+        key: str = os.environ.get("SUPABASE_SERVICE_KEY")
+        self.supabase: Client = create_client(url, key)
         self.ctx: agents.JobContext = ctx
         self.chat = rtc.ChatManager(ctx.room)
         self.audio_out = rtc.AudioSource(ELEVEN_TTS_SAMPLE_RATE, ELEVEN_TTS_CHANNELS)
@@ -130,6 +135,33 @@ async def process_track(self, track: rtc.Track):
             stream.push_frame(audio_frame_event.frame)
         await stream.flush()
 
+    def reprompt(self, data, msg:str) -> str:
+        # handle later
+        # tokenCount = 0
+        contextText = "You are a very enthusiastic Supabase representative who loves \
+        to help people! Your main and only goal is to assist people with building and help connect them with \
+        other buildspace members. You currently only have information on participants from Buildspace \
+        season 3. Given the following sections from previous \
+        Buildspace seasons, answer the question using only that information, \
+        when asked to find or connect with people. If you are unsure and the answer \
+        is not explicitly written in the section below, say \
+        'Sorry, I can't find anyone from Buildspace to connect you with.' closest matches from previous buildspace seasons: "
+
+        data = data[1]    
+
+        for match in data:
+            print("data", data)
+            print("match", match)
+            contextText += f"Title of Demo Day Submission: {match["title"]}\
+                Niche: {match["niche"]}\
+                Summary: {match["description"]} \
+                Full Description: {match["youtube_transcript"]} \
+                Buildspace Season: {match["season"]}"
+
+        contextText+=f"\nuser's message or question: {msg}"
+
+        return inspect.cleandoc(contextText);
+
     async def process_stt_stream(self, stream):
         buffered_text = ""
         async for event in stream:
@@ -149,6 +181,16 @@ async def process_stt_stream(self, stream):
                 ),
                 topic="transcription",
             )
+
+            msg_embedding = await self.chatgpt_plugin.embed(buffered_text)
+            if msg_embedding:
+                data, count = self.supabase.rpc('match_person', {
+                    "query_embedding": msg_embedding,
+                    "match_threshold": 0.20,
+                    "match_count": 4,
+                }).execute()
+                buffered_text = self.reprompt(data,buffered_text)
+
 
             msg = ChatGPTMessage(role=ChatGPTMessageRole.user, content=buffered_text)
             chatgpt_stream = self.chatgpt_plugin.add_message(msg)
@@ -207,12 +249,12 @@ def update_state(self, sending_audio: bool = None, processing: bool = None):
     logging.basicConfig(level=logging.INFO)
 
     async def job_request_cb(job_request: agents.JobRequest):
-        logging.info("Accepting job for KITT")
+        logging.info("Accepting job for Buildspace AI")
 
         await job_request.accept(
-            KITT.create,
-            identity="kitt_agent",
-            name="KITT",
+            BuildspaceAI.create,
+            identity="Buildspace AI",
+            name="Buildspace AI",
             auto_subscribe=agents.AutoSubscribe.AUDIO_ONLY,
             auto_disconnect=agents.AutoDisconnect.DEFAULT,
         )

server/llm.py

@@ -19,10 +19,11 @@
 from dataclasses import dataclass
 from typing import AsyncIterable, List, Optional
 from enum import Enum
+from dotenv import load_dotenv
+load_dotenv()
 
 ChatGPTMessageRole = Enum("MessageRole", ["system", "user", "assistant", "function"])
 
-
 @dataclass
 class ChatGPTMessage:
     role: ChatGPTMessageRole
@@ -87,6 +88,20 @@ async def add_message(
         async for text in self._generate_text_streamed(self._model):
             yield text
 
+    async def embed(
+        self, 
+        msg: str
+    ):
+        try:
+            resp = await self._client.embeddings.create(
+                    input=msg,
+                    encoding_format="float",
+                    model="text-embedding-3-small"
+            );
+            return resp.data[0].embedding
+        except: 
+            return None
+
     async def _generate_text_streamed(self, model: str) -> AsyncIterable[str]:
         prompt_message = ChatGPTMessage(
             role=ChatGPTMessageRole.system, content=self._prompt