Merge branch 'main' into brace/api-refs-docs

docs/docs/integrations/text_embedding/voyageai.mdx

@@ -0,0 +1,11 @@
+# Voyage AI
+
+The `VoyageEmbeddings` class uses the Voyage AI REST API to generate embeddings for a given text.
+
+```typescript
+import { VoyageEmbeddings } from "langchain/embeddings/voyage";
+
+const embeddings = new VoyageEmbeddings({
+  apiKey: "YOUR-API-KEY", // In Node.js defaults to process.env.VOYAGEAI_API_KEY
+});
+```

environment_tests/test-exports-bun/src/entrypoints.js

@@ -21,6 +21,7 @@ export * from "langchain/embeddings/fake";
 export * from "langchain/embeddings/ollama";
 export * from "langchain/embeddings/openai";
 export * from "langchain/embeddings/minimax";
+export * from "langchain/embeddings/voyage";
 export * from "langchain/llms/base";
 export * from "langchain/llms/openai";
 export * from "langchain/llms/ai21";

environment_tests/test-exports-cf/src/entrypoints.js

@@ -21,6 +21,7 @@ export * from "langchain/embeddings/fake";
 export * from "langchain/embeddings/ollama";
 export * from "langchain/embeddings/openai";
 export * from "langchain/embeddings/minimax";
+export * from "langchain/embeddings/voyage";
 export * from "langchain/llms/base";
 export * from "langchain/llms/openai";
 export * from "langchain/llms/ai21";

environment_tests/test-exports-cjs/src/entrypoints.js

@@ -21,6 +21,7 @@ const embeddings_fake = require("langchain/embeddings/fake");
 const embeddings_ollama = require("langchain/embeddings/ollama");
 const embeddings_openai = require("langchain/embeddings/openai");
 const embeddings_minimax = require("langchain/embeddings/minimax");
+const embeddings_voyage = require("langchain/embeddings/voyage");
 const llms_base = require("langchain/llms/base");
 const llms_openai = require("langchain/llms/openai");
 const llms_ai21 = require("langchain/llms/ai21");

environment_tests/test-exports-esbuild/src/entrypoints.js

@@ -21,6 +21,7 @@ import * as embeddings_fake from "langchain/embeddings/fake";
 import * as embeddings_ollama from "langchain/embeddings/ollama";
 import * as embeddings_openai from "langchain/embeddings/openai";
 import * as embeddings_minimax from "langchain/embeddings/minimax";
+import * as embeddings_voyage from "langchain/embeddings/voyage";
 import * as llms_base from "langchain/llms/base";
 import * as llms_openai from "langchain/llms/openai";
 import * as llms_ai21 from "langchain/llms/ai21";

environment_tests/test-exports-esm/src/entrypoints.js

@@ -21,6 +21,7 @@ import * as embeddings_fake from "langchain/embeddings/fake";
 import * as embeddings_ollama from "langchain/embeddings/ollama";
 import * as embeddings_openai from "langchain/embeddings/openai";
 import * as embeddings_minimax from "langchain/embeddings/minimax";
+import * as embeddings_voyage from "langchain/embeddings/voyage";
 import * as llms_base from "langchain/llms/base";
 import * as llms_openai from "langchain/llms/openai";
 import * as llms_ai21 from "langchain/llms/ai21";

environment_tests/test-exports-vercel/src/entrypoints.js

@@ -21,6 +21,7 @@ export * from "langchain/embeddings/fake";
 export * from "langchain/embeddings/ollama";
 export * from "langchain/embeddings/openai";
 export * from "langchain/embeddings/minimax";
+export * from "langchain/embeddings/voyage";
 export * from "langchain/llms/base";
 export * from "langchain/llms/openai";
 export * from "langchain/llms/ai21";

environment_tests/test-exports-vite/src/entrypoints.js

@@ -21,6 +21,7 @@ export * from "langchain/embeddings/fake";
 export * from "langchain/embeddings/ollama";
 export * from "langchain/embeddings/openai";
 export * from "langchain/embeddings/minimax";
+export * from "langchain/embeddings/voyage";
 export * from "langchain/llms/base";
 export * from "langchain/llms/openai";
 export * from "langchain/llms/ai21";

langchain/.eslintrc.cjs

@@ -15,6 +15,7 @@ module.exports = {
   ignorePatterns: [
     "src/util/@cfworker",
     "src/util/fast-json-patch",
+    "src/util/js-sha1",
     ".eslintrc.cjs",
     "scripts",
     "node_modules",

langchain/.gitignore

@@ -133,6 +133,9 @@ embeddings/googlepalm.d.ts
 embeddings/minimax.cjs
 embeddings/minimax.js
 embeddings/minimax.d.ts
+embeddings/voyage.cjs
+embeddings/voyage.js
+embeddings/voyage.d.ts
 embeddings/llama_cpp.cjs
 embeddings/llama_cpp.js
 embeddings/llama_cpp.d.ts

langchain/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.0.180",
+  "version": "0.0.181",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -145,6 +145,9 @@
     "embeddings/minimax.cjs",
     "embeddings/minimax.js",
     "embeddings/minimax.d.ts",
+    "embeddings/voyage.cjs",
+    "embeddings/voyage.js",
+    "embeddings/voyage.d.ts",
     "embeddings/llama_cpp.cjs",
     "embeddings/llama_cpp.js",
     "embeddings/llama_cpp.d.ts",
@@ -867,7 +870,6 @@
     "@types/jsdom": "^21.1.1",
     "@types/lodash": "^4",
     "@types/mozilla-readability": "^0.2.1",
-    "@types/object-hash": "^3.0.2",
     "@types/pdf-parse": "^1.1.1",
     "@types/pg": "^8",
     "@types/pg-copy-streams": "^1.2.2",
@@ -1360,7 +1362,6 @@
     "langchainhub": "~0.0.6",
     "langsmith": "~0.0.48",
     "ml-distance": "^4.0.0",
-    "object-hash": "^3.0.0",
     "openai": "~4.4.0",
     "openapi-types": "^12.1.3",
     "p-queue": "^6.6.2",
@@ -1613,6 +1614,11 @@
       "import": "./embeddings/minimax.js",
       "require": "./embeddings/minimax.cjs"
     },
+    "./embeddings/voyage": {
+      "types": "./embeddings/voyage.d.ts",
+      "import": "./embeddings/voyage.js",
+      "require": "./embeddings/voyage.cjs"
+    },
     "./embeddings/llama_cpp": {
       "types": "./embeddings/llama_cpp.d.ts",
       "import": "./embeddings/llama_cpp.js",

langchain/scripts/create-entrypoints.js

@@ -59,6 +59,7 @@ const entrypoints = {
   "embeddings/googlevertexai": "embeddings/googlevertexai",
   "embeddings/googlepalm": "embeddings/googlepalm",
   "embeddings/minimax": "embeddings/minimax",
+  "embeddings/voyage": "embeddings/voyage",
   "embeddings/llama_cpp": "embeddings/llama_cpp",
   // llms
   "llms/load": "llms/load",

langchain/src/cache/base.ts

@@ -1,4 +1,4 @@
-import hash from "object-hash";
+import { insecureHash } from "../util/js-sha1/hash.js";
 
 import {
   ChatGeneration,
@@ -18,7 +18,7 @@ import {
  * TODO: Make cache key consistent across versions of langchain.
  */
 export const getCacheKey = (...strings: string[]): string =>
-  hash(strings.join("_"));
+  insecureHash(strings.join("_"));
 
 export function deserializeStoredGeneration(
   storedGeneration: StoredGeneration

langchain/src/cache/tests/ioredis.test.ts

@@ -1,14 +1,14 @@
 import { test, expect, jest } from "@jest/globals";
-import hash from "object-hash";
+import { insecureHash } from "../../util/js-sha1/hash.js";
 
 import { RedisCache } from "../ioredis.js";
 
-const sha256 = (str: string) => hash(str);
+const sha1 = (str: string) => insecureHash(str);
 
 test("RedisCache", async () => {
   const redis = {
     get: jest.fn(async (key: string) => {
-      if (key === sha256("foo_bar_0")) {
+      if (key === sha1("foo_bar_0")) {
         return JSON.stringify({ text: "baz" });
       }
       return null;

langchain/src/cache/tests/redis.test.ts

@@ -1,14 +1,14 @@
 import { test, expect, jest } from "@jest/globals";
-import hash from "object-hash";
+import { insecureHash } from "../../util/js-sha1/hash.js";
 
 import { RedisCache } from "../redis.js";
 
-const sha256 = (str: string) => hash(str);
+const sha1 = (str: string) => insecureHash(str);
 
 test("RedisCache", async () => {
   const redis = {
     get: jest.fn(async (key: string) => {
-      if (key === sha256("foo_bar_0")) {
+      if (key === sha1("foo_bar_0")) {
         return JSON.stringify({ text: "baz" });
       }
       return null;

langchain/src/cache/tests/upstash_redis.test.ts

@@ -1,15 +1,15 @@
 import { test, expect, jest } from "@jest/globals";
-import hash from "object-hash";
+import { insecureHash } from "../../util/js-sha1/hash.js";
 
 import { UpstashRedisCache } from "../upstash_redis.js";
 import { StoredGeneration } from "../../schema/index.js";
 
-const sha256 = (str: string) => hash(str);
+const sha1 = (str: string) => insecureHash(str);
 
 test("UpstashRedisCache", async () => {
   const redis = {
     get: jest.fn(async (key: string): Promise<StoredGeneration | null> => {
-      if (key === sha256("foo_bar_0")) {
+      if (key === sha1("foo_bar_0")) {
         return { text: "baz" };
       }
       return null;

langchain/src/embeddings/cache_backed.ts

@@ -1,4 +1,4 @@
-import hash from "object-hash";
+import { insecureHash } from "../util/js-sha1/hash.js";
 
 import { BaseStore } from "../schema/storage.js";
 import { EncoderBackedStore } from "../storage/encoder_backed.js";
@@ -112,7 +112,7 @@ export class CacheBackedEmbeddings extends Embeddings {
       Uint8Array
     >({
       store: documentEmbeddingStore,
-      keyEncoder: (key) => (options?.namespace ?? "") + hash(key),
+      keyEncoder: (key) => (options?.namespace ?? "") + insecureHash(key),
       valueSerializer: (value) => encoder.encode(JSON.stringify(value)),
       valueDeserializer: (serializedValue) =>
         JSON.parse(decoder.decode(serializedValue)),

langchain/src/embeddings/tests/voyage.int.test.ts

@@ -0,0 +1,36 @@
+import { test, expect } from "@jest/globals";
+import { VoyageEmbeddings } from "../voyage.js";
+
+test.skip("Test VoyageEmbeddings.embedQuery", async () => {
+  const embeddings = new VoyageEmbeddings();
+  const res = await embeddings.embedQuery("Hello world");
+
+  expect(typeof res[0]).toBe("number");
+});
+
+test.skip("Test VoyageEmbeddings.embedDocuments", async () => {
+  const embeddings = new VoyageEmbeddings();
+  const res = await embeddings.embedDocuments(["Hello world", "Bye bye"]);
+  expect(res).toHaveLength(2);
+  expect(typeof res[0][0]).toBe("number");
+  expect(typeof res[1][0]).toBe("number");
+});
+
+test.skip("Test VoyageEmbeddings concurrency", async () => {
+  const embeddings = new VoyageEmbeddings({
+    batchSize: 1,
+    maxConcurrency: 2,
+  });
+  const res = await embeddings.embedDocuments([
+    "Hello world",
+    "Bye bye",
+    "Hello world",
+    "Bye bye",
+    "Hello world",
+    "Bye bye",
+  ]);
+  expect(res).toHaveLength(6);
+  expect(res.find((embedding) => typeof embedding[0] !== "number")).toBe(
+    undefined
+  );
+});