fix: fix module not found errors in Vercel edge (#300)

ecosystem-tests/cli.ts

@@ -21,8 +21,14 @@ const projects = {
   'vercel-edge': async () => {
     await installPackage();
 
+    if (state.live) {
+      await run('npm', ['run', 'test:ci:dev']);
+    }
     await run('npm', ['run', 'build']);
 
+    if (state.live) {
+      await run('npm', ['run', 'test:ci']);
+    }
     if (state.deploy) {
       await run('npm', ['run', 'vercel', 'deploy', '--prod', '--force']);
     }

ecosystem-tests/vercel-edge/jest.config.cjs

@@ -0,0 +1,9 @@
+/** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testMatch: ['<rootDir>/tests/*.ts'],
+  watchPathIgnorePatterns: ['<rootDir>/node_modules/'],
+  verbose: false,
+  testTimeout: 60000,
+};

ecosystem-tests/vercel-edge/package.json

@@ -8,7 +8,10 @@
     "start": "next start",
     "lint": "next lint",
     "edge-runtime": "edge-runtime",
-    "vercel": "vercel"
+    "vercel": "vercel",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+    "test:ci:dev": "start-server-and-test dev http://localhost:3000 test",
+    "test:ci": "start-server-and-test start http://localhost:3000 test"
   },
   "dependencies": {
     "ai": "2.1.34",
@@ -21,6 +24,10 @@
     "@types/react": "18.2.13",
     "@types/react-dom": "18.2.6",
     "edge-runtime": "^2.4.3",
+    "fastest-levenshtein": "^1.0.16",
+    "jest": "^29.5.0",
+    "start-server-and-test": "^2.0.0",
+    "ts-jest": "^29.1.0",
     "typescript": "4.7.4",
     "vercel": "^31.0.0"
   }

ecosystem-tests/vercel-edge/src/pages/api/edge-test.ts

@@ -0,0 +1,79 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { distance } from 'fastest-levenshtein';
+import OpenAI from 'openai';
+import { uploadWebApiTestCases } from '../../uploadWebApiTestCases';
+
+export const config = {
+  runtime: 'edge',
+  unstable_allowDynamic: [
+    // This is currently required because `qs` uses `side-channel` which depends on this.
+    //
+    // Warning: Some features may be broken at runtime because of this.
+    '/node_modules/function-bind/**',
+  ],
+};
+
+type Test = { description: string; handler: () => Promise<void> };
+
+const tests: Test[] = [];
+function it(description: string, handler: () => Promise<void>) {
+  tests.push({ description, handler });
+}
+function expectEqual(a: any, b: any) {
+  if (!Object.is(a, b)) {
+    throw new Error(`expected values to be equal: ${JSON.stringify({ a, b })}`);
+  }
+}
+function expectSimilar(received: string, expected: string, maxDistance: number) {
+  const receivedDistance = distance(received, expected);
+  if (receivedDistance < maxDistance) {
+    return;
+  }
+
+  const message = [
+    `Received: ${JSON.stringify(received)}`,
+    `Expected: ${JSON.stringify(expected)}`,
+    `Max distance: ${maxDistance}`,
+    `Received distance: ${receivedDistance}`,
+  ].join('\n');
+
+  throw new Error(message);
+}
+
+export default async (request: NextRequest) => {
+  try {
+    console.error('creating client');
+    const client = new OpenAI();
+    console.error('created client');
+
+    uploadWebApiTestCases({
+      client: client as any,
+      it,
+      expectEqual,
+      expectSimilar,
+      runtime: 'edge',
+    });
+
+    let allPassed = true;
+    const results = [];
+
+    for (const { description, handler } of tests) {
+      console.error('running', description);
+      let result;
+      try {
+        result = await handler();
+        console.error('passed ', description);
+      } catch (error) {
+        console.error('failed ', description, error);
+        allPassed = false;
+        result = error instanceof Error ? error.stack : String(error);
+      }
+      results.push(`${description}\n\n${String(result)}`);
+    }
+
+    return new NextResponse(allPassed ? 'Passed!' : results.join('\n\n'));
+  } catch (error) {
+    console.error(error instanceof Error ? error.stack : String(error));
+    return new NextResponse(error instanceof Error ? error.stack : String(error), { status: 500 });
+  }
+};

ecosystem-tests/vercel-edge/src/pages/api/node-test.ts

@@ -0,0 +1,68 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { distance } from 'fastest-levenshtein';
+import OpenAI from 'openai';
+import { uploadWebApiTestCases } from '../../uploadWebApiTestCases';
+
+type Test = { description: string; handler: () => Promise<void> };
+
+const tests: Test[] = [];
+function it(description: string, handler: () => Promise<void>) {
+  tests.push({ description, handler });
+}
+function expectEqual(a: any, b: any) {
+  if (!Object.is(a, b)) {
+    throw new Error(`expected values to be equal: ${JSON.stringify({ a, b })}`);
+  }
+}
+function expectSimilar(received: string, expected: string, maxDistance: number) {
+  const receivedDistance = distance(received, expected);
+  if (receivedDistance < maxDistance) {
+    return;
+  }
+
+  const message = [
+    `Received: ${JSON.stringify(received)}`,
+    `Expected: ${JSON.stringify(expected)}`,
+    `Max distance: ${maxDistance}`,
+    `Received distance: ${receivedDistance}`,
+  ].join('\n');
+
+  throw new Error(message);
+}
+
+export default async (request: NextApiRequest, response: NextApiResponse) => {
+  try {
+    console.error('creating client');
+    const client = new OpenAI();
+    console.error('created client');
+
+    uploadWebApiTestCases({
+      client: client as any,
+      it,
+      expectEqual,
+      expectSimilar,
+    });
+
+    let allPassed = true;
+    const results = [];
+
+    for (const { description, handler } of tests) {
+      console.error('running', description);
+      let result;
+      try {
+        result = await handler();
+        console.error('passed ', description);
+      } catch (error) {
+        console.error('failed ', description, error);
+        allPassed = false;
+        result = error instanceof Error ? error.stack : String(error);
+      }
+      results.push(`${description}\n\n${String(result)}`);
+    }
+
+    response.status(200).end(allPassed ? 'Passed!' : results.join('\n\n'));
+  } catch (error) {
+    console.error(error instanceof Error ? error.stack : String(error));
+    response.status(500).end(error instanceof Error ? error.stack : String(error));
+  }
+};

ecosystem-tests/vercel-edge/src/uploadWebApiTestCases.ts

@@ -0,0 +1,122 @@
+import OpenAI, { toFile } from 'openai';
+import { TranscriptionCreateParams } from 'openai/resources/audio/transcriptions';
+
+/**
+ * Tests uploads using various Web API data objects.
+ * This is structured to support running these tests on builtins in the environment in
+ * Node or Cloudflare workers etc. or on polyfills like from node-fetch/formdata-node
+ */
+export function uploadWebApiTestCases({
+  client,
+  it,
+  expectEqual,
+  expectSimilar,
+  runtime = 'node',
+}: {
+  /**
+   * OpenAI client instance
+   */
+  client: OpenAI;
+  /**
+   * Jest it() function, or an imitation in envs like Cloudflare workers
+   */
+  it: (desc: string, handler: () => Promise<void>) => void;
+  /**
+   * Jest expect(a).toEqual(b) function, or an imitation in envs like Cloudflare workers
+   */
+  expectEqual(a: unknown, b: unknown): void;
+  /**
+   * Assert that the levenshtein distance between the two given strings is less than the given max distance.
+   */
+  expectSimilar(received: string, expected: string, maxDistance: number): void;
+  runtime?: 'node' | 'edge';
+}) {
+  const url = 'https://audio-samples.github.io/samples/mp3/blizzard_biased/sample-1.mp3';
+  const filename = 'sample-1.mp3';
+
+  const correctAnswer =
+    'It was anxious to find him no one that expectation of a man who were giving his father enjoyment. But he was avoided in sight in the minister to which indeed,';
+  const model = 'whisper-1';
+
+  async function typeTests() {
+    // @ts-expect-error this should error if the `Uploadable` type was resolved correctly
+    await client.audio.transcriptions.create({ file: { foo: true }, model: 'whisper-1' });
+    // @ts-expect-error this should error if the `Uploadable` type was resolved correctly
+    await client.audio.transcriptions.create({ file: null, model: 'whisper-1' });
+    // @ts-expect-error this should error if the `Uploadable` type was resolved correctly
+    await client.audio.transcriptions.create({ file: 'test', model: 'whisper-1' });
+  }
+
+  it(`streaming works`, async function () {
+    const stream = await client.chat.completions.create({
+      model: 'gpt-4',
+      messages: [{ role: 'user', content: 'Say this is a test' }],
+      stream: true,
+    });
+    const chunks = [];
+    for await (const part of stream) {
+      chunks.push(part);
+    }
+    expectSimilar(chunks.map((c) => c.choices[0]?.delta.content || '').join(''), 'This is a test', 10);
+  });
+
+  if (runtime !== 'node') {
+    it('handles File', async () => {
+      const file = await fetch(url)
+        .then((x) => x.arrayBuffer())
+        .then((x) => new File([x], filename));
+
+      const params: TranscriptionCreateParams = { file, model };
+
+      const result = await client.audio.transcriptions.create(params);
+      expectSimilar(result.text, correctAnswer, 12);
+    });
+
+    it('handles Response', async () => {
+      const file = await fetch(url);
+
+      const result = await client.audio.transcriptions.create({ file, model });
+      expectSimilar(result.text, correctAnswer, 12);
+    });
+  }
+
+  const fineTune = `{"prompt": "<prompt text>", "completion": "<ideal generated text>"}`;
+
+  it('toFile handles string', async () => {
+    // @ts-expect-error we don't type support for `string` to avoid a footgun with passing the file path
+    const file = await toFile(fineTune, 'finetune.jsonl');
+    const result = await client.files.create({ file, purpose: 'fine-tune' });
+    expectEqual(result.status, 'uploaded');
+  });
+  it('toFile handles Blob', async () => {
+    const result = await client.files.create({
+      file: await toFile(new Blob([fineTune]), 'finetune.jsonl'),
+      purpose: 'fine-tune',
+    });
+    expectEqual(result.status, 'uploaded');
+  });
+  it('toFile handles Uint8Array', async () => {
+    const result = await client.files.create({
+      file: await toFile(new TextEncoder().encode(fineTune), 'finetune.jsonl'),
+      purpose: 'fine-tune',
+    });
+    expectEqual(result.status, 'uploaded');
+  });
+  it('toFile handles ArrayBuffer', async () => {
+    const result = await client.files.create({
+      file: await toFile(new TextEncoder().encode(fineTune).buffer, 'finetune.jsonl'),
+      purpose: 'fine-tune',
+    });
+    expectEqual(result.status, 'uploaded');
+  });
+  if (runtime !== 'edge') {
+    // this fails in edge for some reason
+    it('toFile handles DataView', async () => {
+      const result = await client.files.create({
+        file: await toFile(new DataView(new TextEncoder().encode(fineTune).buffer), 'finetune.jsonl'),
+        purpose: 'fine-tune',
+      });
+      expectEqual(result.status, 'uploaded');
+    });
+  }
+}

ecosystem-tests/vercel-edge/tests/test.ts

@@ -0,0 +1,20 @@
+import fetch from 'node-fetch';
+
+const baseUrl = process.env.TEST_BASE_URL || 'http://localhost:3000';
+console.log(baseUrl);
+
+it(
+  'node runtime',
+  async () => {
+    expect(await (await fetch(`${baseUrl}/api/node-test`)).text()).toEqual('Passed!');
+  },
+  3 * 60000,
+);
+
+it(
+  'edge runtime',
+  async () => {
+    expect(await (await fetch(`${baseUrl}/api/edge-test`)).text()).toEqual('Passed!');
+  },
+  3 * 60000,
+);