Merge pull request #385 from twinnydotdev/fix/embeddings-database

Fix embeddings database update lancedb

package.json

@@ -2,7 +2,7 @@
   "name": "twinny",
   "displayName": "twinny - AI Code Completion and Chat",
   "description": "Locally hosted AI code completion plugin for vscode",
-  "version": "3.18.12",
+  "version": "3.19.0",
   "icon": "assets/icon.png",
   "keywords": [
     "code-inference",
@@ -464,7 +464,7 @@
     "web-tree-sitter": "^0.22.1"
   },
   "dependencies": {
-    "@lancedb/lancedb": "^0.9.0",
+    "@lancedb/lancedb": "^0.12.0",
     "@tiptap/extension-mention": "^2.5.9",
     "@tiptap/extension-placeholder": "^2.5.9",
     "@tiptap/pm": "^2.5.9",

src/common/constants.ts

@@ -179,7 +179,6 @@ export const EXTENSION_CONTEXT_NAME = {
   twinnyOverlapSize: "twinnyOverlapSize",
   twinnyRelevantFilePaths: "twinnyRelevantFilePaths",
   twinnyRelevantCodeSnippets: "twinnyRelevantCodeSnippets",
-  twinnyVectorSearchMetric: "twinnyVectorSearchMetric",
   twinnySymmetryTab: "twinnySymmetryTab",
   twinnyEnableRag: "twinnyEnableRag",
 }
@@ -338,9 +337,6 @@ export const WASM_LANGUAGES: { [key: string]: string } = {
 
 export const DEFAULT_RELEVANT_FILE_COUNT = 10
 export const DEFAULT_RELEVANT_CODE_COUNT = 5
-export const DEFAULT_VECTOR_SEARCH_METRIC = "l2"
-
-export const EMBEDDING_METRICS = ["cosine", "l2", "dot"]
 
 export const MULTILINE_OUTSIDE = [
   "class_body",

src/common/types.ts

@@ -290,7 +290,7 @@ export interface ChunkOptions {
 }
 
 export type Embedding = {
-  embeddings: number[]
+  embeddings: [number[]]
 }
 
 export type EmbeddedDocument = {

src/extension/chat-service.ts

@@ -16,7 +16,6 @@ import {
   DEFAULT_RELEVANT_CODE_COUNT,
   DEFAULT_RELEVANT_FILE_COUNT,
   DEFAULT_RERANK_THRESHOLD,
-  DEFAULT_VECTOR_SEARCH_METRIC,
   EVENT_NAME,
   EXTENSION_CONTEXT_NAME,
   EXTENSION_SESSION_NAME,
@@ -118,18 +117,11 @@ export class ChatService extends Base {
       ) as number
       const relevantFileCount = Number(stored) || DEFAULT_RELEVANT_FILE_COUNT
 
-      const storedMetric = this._context?.globalState.get(
-        `${EVENT_NAME.twinnyGlobalContext}-${EXTENSION_CONTEXT_NAME.twinnyVectorSearchMetric}`
-      ) as number
-
-      const metric = storedMetric || DEFAULT_VECTOR_SEARCH_METRIC
-
       const filePaths =
         (await this._db.getDocuments(
           embedding,
           relevantFileCount,
           table,
-          metric as "cosine" | "l2" | "dot"
         )) || []
 
       if (!filePaths.length) return []
@@ -226,11 +218,6 @@ export class ChatService extends Base {
 
       if (!embedding) return ""
 
-      const storedMetric = this._context?.globalState.get(
-        `${EVENT_NAME.twinnyGlobalContext}-${EXTENSION_CONTEXT_NAME.twinnyVectorSearchMetric}`
-      ) as number
-      const metric = storedMetric || DEFAULT_VECTOR_SEARCH_METRIC
-
       const query = relevantFiles?.length
         ? `file IN ("${relevantFiles.map((file) => file[0]).join("\",\"")}")`
         : ""
@@ -240,7 +227,6 @@ export class ChatService extends Base {
           embedding,
           Math.round(relevantCodeCount / 2),
           table,
-          metric as "cosine" | "l2" | "dot",
           query
         )) || []
 
@@ -249,7 +235,6 @@ export class ChatService extends Base {
           embedding,
           Math.round(relevantCodeCount / 2),
           table,
-          metric as "cosine" | "l2" | "dot"
         )) || []
 
       const documents = [...embeddedDocuments, ...queryEmbeddedDocuments]