Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add multi-file upload capability to SDKs #503

Closed
wants to merge 8 commits into from
Closed
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 28 additions & 17 deletions packages/js-sdk/src/sandbox/filesystem/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ import { FileType as FsFileType, Filesystem as FilesystemService } from '../../e

import { WatchHandle, FilesystemEvent } from './watchHandle'

export type WriteData = string | ArrayBuffer | Blob | ReadableStream

export type WriteEntry = {
path: string
data: WriteData
}

/**
* Sandbox filesystem object information.
*/
Expand Down Expand Up @@ -205,7 +212,7 @@ export class Filesystem {
}

/**
* Write content to a file.
* Write content to a file(s).
*
*
* Writing to a file that doesn't exist creates the file.
Expand All @@ -214,32 +221,36 @@ export class Filesystem {
*
* Writing to a file at path that doesn't exist creates the necessary directories.
*
* @param path path to file.
* @param data data to write to the file. Data can be a string, `ArrayBuffer`, `Blob`, or `ReadableStream`.
* @param files array of WriteEntry objects to write
* @param opts connection options.
*
* @returns information about the written file
* @returns array of information about the written files
*/
async write(
path: string,
data: string | ArrayBuffer | Blob | ReadableStream,
files: WriteEntry[],
opts?: FilesystemRequestOpts
): Promise<EntryInfo> {
const blob = await new Response(data).blob()
): Promise<EntryInfo[]> {

if (files.length === 0) return [] as EntryInfo[]

const blobs = await Promise.all(files.map((f) => new Response(f.data).blob()))

const res = await this.envdApi.api.POST('/files', {
params: {
query: {
path,
username: opts?.user || defaultUsername,
},
},
bodySerializer() {
const fd = new FormData()

fd.append('file', blob)

return fd
return blobs.reduce((fd, blob, i) => {
// Important: RFC 7578, Section 4.2 requires that if a filename is provided,
// the directory path information must not be used.
// BUT in our case we need to use the directory path information with a custom
// muktipart part name getter in envd.
0div marked this conversation as resolved.
Show resolved Hide resolved
fd.append('file', blob, files[i].path)

return fd
}, new FormData())
},
body: {},
headers: {
Expand All @@ -253,12 +264,12 @@ export class Filesystem {
throw err
}

const files = res.data
if (!files || files.length === 0) {
const createdFiles = res.data as EntryInfo[]
if (!createdFiles || createdFiles.length === 0) {
throw new Error('Expected to receive information about written file')
}

return files[0] as EntryInfo
return createdFiles
}

/**
Expand Down
2 changes: 1 addition & 1 deletion packages/js-sdk/tests/sandbox/files/exists.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { sandboxTest } from '../../setup.js'
sandboxTest('file exists', async ({ sandbox }) => {
const filename = 'test_exists.txt'

await sandbox.files.write(filename, 'test')
await sandbox.files.write([{ path: filename, data: 'test' }])
const exists = await sandbox.files.exists(filename)
assert.isTrue(exists)
})
Expand Down
2 changes: 1 addition & 1 deletion packages/js-sdk/tests/sandbox/files/list.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ sandboxTest('list directory', async ({ sandbox }) => {
const files = await sandbox.files.list(dirName)
assert.equal(files.length, 0)

await sandbox.files.write('test_directory4/test_file', 'test')
await sandbox.files.write([{ path: 'test_directory4/test_file', data: 'test' }])

const files1 = await sandbox.files.list(dirName)
assert.equal(files1.length, 1)
Expand Down
2 changes: 1 addition & 1 deletion packages/js-sdk/tests/sandbox/files/read.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ sandboxTest('read file', async ({ sandbox }) => {
const filename = 'test_read.txt'
const content = 'Hello, world!'

await sandbox.files.write(filename, content)
await sandbox.files.write([{ path: filename, data: content }])
const readContent = await sandbox.files.read(filename)
assert.equal(readContent, content)
})
Expand Down
2 changes: 1 addition & 1 deletion packages/js-sdk/tests/sandbox/files/remove.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ sandboxTest('remove file', async ({ sandbox }) => {
const filename = 'test_remove.txt'
const content = 'This file will be removed.'

await sandbox.files.write(filename, content)
await sandbox.files.write([{ path: filename, data: content }])
await sandbox.files.remove(filename)

const exists = await sandbox.files.exists(filename)
Expand Down
2 changes: 1 addition & 1 deletion packages/js-sdk/tests/sandbox/files/rename.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ sandboxTest('rename file', async ({ sandbox }) => {
const newFilename = 'test_rename_new.txt'
const content = 'This file will be renamed.'

await sandbox.files.write(oldFilename, content)
await sandbox.files.write([{ path: oldFilename, data: content }])
const info = await sandbox.files.rename(oldFilename, newFilename)
assert.equal(info.name, newFilename)
assert.equal(info.type, 'file')
Expand Down
6 changes: 3 additions & 3 deletions packages/js-sdk/tests/sandbox/files/watch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ sandboxTest('watch directory changes', async ({ sandbox }) => {
const newContent = 'This file has been modified.'

await sandbox.files.makeDir(dirname)
await sandbox.files.write(`${dirname}/${filename}`, content)
await sandbox.files.write([{ path: `${dirname}/${filename}`, data: content }])

let trigger: () => void

Expand All @@ -24,7 +24,7 @@ sandboxTest('watch directory changes', async ({ sandbox }) => {
}
})

await sandbox.files.write(`${dirname}/${filename}`, newContent)
await sandbox.files.write([{ path: `${dirname}/${filename}`, data: newContent }])

await eventPromise

Expand All @@ -42,7 +42,7 @@ sandboxTest('watch non-existing directory', async ({ sandbox }) => {
sandboxTest('watch file', async ({ sandbox }) => {
const filename = 'test_watch.txt'
const content = 'This file will be watched.'
await sandbox.files.write(filename, content)
await sandbox.files.write([{ path: filename, data: content }])

await expect(sandbox.files.watchDir(filename, () => {})).rejects.toThrowError(
SandboxError
Expand Down
66 changes: 51 additions & 15 deletions packages/js-sdk/tests/sandbox/files/write.test.ts
Original file line number Diff line number Diff line change
@@ -1,29 +1,63 @@
import { assert } from 'vitest'
import path from 'path'
import { assert, onTestFinished } from 'vitest'

import { sandboxTest } from '../../setup.js'
import { WriteEntry } from '../../../src/sandbox/filesystem/index.js'

sandboxTest('write file', async ({ sandbox }) => {
const filename = 'test_write.txt'
const content = 'This is a test file.'
sandboxTest('write files', async ({ sandbox }) => {
// Attempt to write with one file in array
const info = await sandbox.files.write([{ path: 'one_test_file.txt', data: 'This is a test file.' }])
assert.isTrue(Array.isArray(info))
assert.equal(info[0].name, 'one_test_file.txt')
assert.equal(info[0].type, 'file')
assert.equal(info[0].path, `/home/user/one_test_file.txt`)

const info = await sandbox.files.write(filename, content)
assert.equal(info.name, filename)
assert.equal(info.type, 'file')
assert.equal(info.path, `/home/user/${filename}`)
// Attempt to write with multiple files in array
let files: WriteEntry[] = []

const exists = await sandbox.files.exists(filename)
assert.isTrue(exists)
const readContent = await sandbox.files.read(filename)
assert.equal(readContent, content)
for (let i = 0; i < 10; i++) {
let path = ''
if (i % 2 == 0) {
path = `/${i}/multi_test_file_${i}.txt`
} else {
path = `/home/user/multi_test_file_${i}.txt`
}

onTestFinished(async () => await sandbox.files.remove(path))

files.push({
path: path,
data: `This is a test file ${i}.`,
})
}

const infos = await sandbox.files.write(files)

assert.isTrue(Array.isArray(infos))
assert.equal(infos.length, files.length)

for (let i = 0; i < files.length; i++) {
const file = files[i]
const info = infos[i]

assert.equal(info.name, path.basename(file.path))
assert.equal(info.path, file.path)
assert.equal(info.type, 'file')

const exists = await sandbox.files.exists(file.path)
assert.isTrue(exists)
const readContent = await sandbox.files.read(file.path)
assert.equal(readContent, file.data)
}
})

sandboxTest('overwrite file', async ({ sandbox }) => {
const filename = 'test_overwrite.txt'
const initialContent = 'Initial content.'
const newContent = 'New content.'

await sandbox.files.write(filename, initialContent)
await sandbox.files.write(filename, newContent)
await sandbox.files.write([{ path: filename, data: initialContent }])
await sandbox.files.write([{ path: filename, data: newContent }])
const readContent = await sandbox.files.read(filename)
assert.equal(readContent, newContent)
})
Expand All @@ -32,7 +66,9 @@ sandboxTest('write to non-existing directory', async ({ sandbox }) => {
const filename = 'non_existing_dir/test_write.txt'
const content = 'This should succeed too.'

await sandbox.files.write(filename, content)
const info = await sandbox.files.write([{ path: filename, data: content }])
console.log('info')
console.log(info)
const exists = await sandbox.files.exists(filename)
assert.isTrue(exists)
const readContent = await sandbox.files.read(filename)
Expand Down
7 changes: 1 addition & 6 deletions packages/js-sdk/vitest.workspace.mts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { defineWorkspace } from 'vitest/config'
import { config } from 'dotenv'

const env = config()

export default defineWorkspace([
{
test: {
Expand All @@ -11,12 +12,6 @@ export default defineWorkspace([
exclude: [
'tests/runtimes/**',
],
poolOptions: {
threads: {
minThreads: 1,
maxThreads: 4,
},
},
globals: false,
testTimeout: 30000,
environment: 'node',
Expand Down
Loading