From 22725d7ef485a30cb90d0230b25a33c9e8a18cd5 Mon Sep 17 00:00:00 2001 From: Daniel Tschinder <231804+danez@users.noreply.github.com> Date: Fri, 4 Nov 2022 11:46:06 +0100 Subject: [PATCH] feat: improve performance by reworking caching and supply cache to NFT (#1244) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: improve performance by reworking caching and supply cache to NFT * chore: move cache into class * chore: comment about fileIOConcurrency * Update src/runtimes/node/bundlers/nft/es_modules.ts Co-authored-by: Eduardo Bouças * chore: fix comment * chore: fix comment * chore: add ff Co-authored-by: Eduardo Bouças --- package-lock.json | 38 ++---------- package.json | 1 - src/feature_flags.ts | 6 ++ src/main.ts | 10 +++- src/runtimes/detect_runtime.ts | 20 ++----- src/runtimes/go/index.ts | 35 ++++++----- src/runtimes/index.ts | 32 +++++----- .../esbuild/plugin_dynamic_imports.ts | 11 ++-- .../bundlers/esbuild/plugin_native_modules.ts | 3 +- .../node/bundlers/esbuild/src_files.ts | 6 +- src/runtimes/node/bundlers/nft/es_modules.ts | 28 +++++---- src/runtimes/node/bundlers/nft/index.ts | 15 +++-- src/runtimes/node/bundlers/types.ts | 2 + src/runtimes/node/finder.ts | 11 ++-- src/runtimes/node/index.ts | 3 + src/runtimes/node/parser/index.ts | 4 ++ src/runtimes/node/utils/package_json.ts | 10 ++-- src/runtimes/node/utils/zip.ts | 11 ++-- src/runtimes/runtime.ts | 15 ++--- src/runtimes/rust/builder.ts | 24 ++++++-- src/runtimes/rust/index.ts | 42 ++++++------- src/utils/cache.ts | 32 ++++++++++ src/utils/fs.ts | 59 +++++++++++-------- src/zip.ts | 15 ++++- .../runtimes/node/utils/package_json.test.ts | 8 +-- 25 files changed, 249 insertions(+), 192 deletions(-) create mode 100644 src/utils/cache.ts diff --git a/package-lock.json b/package-lock.json index 40e6ae98f..4ca87f592 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,7 +33,6 @@ "p-map": "^4.0.0", "path-exists": "^5.0.0", "precinct": "^9.0.1", - "read-package-json-fast": "^2.0.2", "require-package-name": "^2.0.1", "resolve": "^2.0.0-next.1", "semver": "^7.0.0", @@ -6984,7 +6983,8 @@ "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true }, "node_modules/json-schema-traverse": { "version": "1.0.0", @@ -7671,11 +7671,6 @@ "node": ">=0.10.0" } }, - "node_modules/npm-normalize-package-bin": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", - "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==" - }, "node_modules/npm-run-all": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/npm-run-all/-/npm-run-all-4.1.5.tgz", @@ -8544,18 +8539,6 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "dev": true }, - "node_modules/read-package-json-fast": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/read-package-json-fast/-/read-package-json-fast-2.0.3.tgz", - "integrity": "sha512-W/BKtbL+dUjTuRL2vziuYhp76s5HZ9qQhd/dKfWIZveD0O40453QNyZhC0e63lqZrAQ4jiOapVoeJ7JrszenQQ==", - "dependencies": { - "json-parse-even-better-errors": "^2.3.0", - "npm-normalize-package-bin": "^1.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/read-pkg": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", @@ -15061,7 +15044,8 @@ "json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true }, "json-schema-traverse": { "version": "1.0.0", @@ -15573,11 +15557,6 @@ "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" }, - "npm-normalize-package-bin": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", - "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==" - }, "npm-run-all": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/npm-run-all/-/npm-run-all-4.1.5.tgz", @@ -16159,15 +16138,6 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "dev": true }, - "read-package-json-fast": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/read-package-json-fast/-/read-package-json-fast-2.0.3.tgz", - "integrity": "sha512-W/BKtbL+dUjTuRL2vziuYhp76s5HZ9qQhd/dKfWIZveD0O40453QNyZhC0e63lqZrAQ4jiOapVoeJ7JrszenQQ==", - "requires": { - "json-parse-even-better-errors": "^2.3.0", - "npm-normalize-package-bin": "^1.0.1" - } - }, "read-pkg": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", diff --git a/package.json b/package.json index ad652edd6..39fed8390 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,6 @@ "p-map": "^4.0.0", "path-exists": "^5.0.0", "precinct": "^9.0.1", - "read-package-json-fast": "^2.0.2", "require-package-name": "^2.0.1", "resolve": "^2.0.0-next.1", "semver": "^7.0.0", diff --git a/src/feature_flags.ts b/src/feature_flags.ts index 22ada2af4..6bb3d0410 100644 --- a/src/feature_flags.ts +++ b/src/feature_flags.ts @@ -21,6 +21,12 @@ export const defaultFlags: Record = { // Load configuration from per-function JSON files. project_deploy_configuration_api_use_per_function_configuration_files: false, + // Enable runtime cache for NFT + zisi_nft_use_cache: false, + + // Raise file IO limit for NFT + zisi_nft_higher_fileio_limit: false, + // Provide banner to esbuild which allows requires in ESM output zisi_esbuild_require_banner: false, } diff --git a/src/main.ts b/src/main.ts index d1af6a73e..59ca99f34 100644 --- a/src/main.ts +++ b/src/main.ts @@ -6,6 +6,7 @@ import { FunctionSource } from './function.js' import { getFunctionFromPath, getFunctionsFromPaths } from './runtimes/index.js' import { findISCDeclarationsInPath, ISCValues } from './runtimes/node/in_source_config/index.js' import { GetSrcFilesFunction, RuntimeType } from './runtimes/runtime.js' +import { RuntimeCache } from './utils/cache.js' import { listFunctionsDirectories, resolveFunctionsDirectories } from './utils/fs.js' export { zipFunction, zipFunctions } from './zip.js' @@ -60,7 +61,8 @@ export const listFunctions = async function ( const featureFlags = getFlags(inputFeatureFlags) const srcFolders = resolveFunctionsDirectories(relativeSrcFolders) const paths = await listFunctionsDirectories(srcFolders) - const functionsMap = await getFunctionsFromPaths(paths, { featureFlags, config }) + const cache = new RuntimeCache() + const functionsMap = await getFunctionsFromPaths(paths, { cache, config, featureFlags }) const functions = [...functionsMap.values()] const augmentedFunctions = parseISC ? await Promise.all(functions.map(augmentWithISC)) : functions @@ -77,7 +79,8 @@ export const listFunction = async function ( }: { featureFlags?: FeatureFlags; config?: Config; parseISC?: boolean } = {}, ) { const featureFlags = getFlags(inputFeatureFlags) - const func = await getFunctionFromPath(path, { featureFlags, config }) + const cache = new RuntimeCache() + const func = await getFunctionFromPath(path, { cache, config, featureFlags }) if (!func) { return @@ -96,7 +99,8 @@ export const listFunctionsFiles = async function ( const featureFlags = getFlags(inputFeatureFlags) const srcFolders = resolveFunctionsDirectories(relativeSrcFolders) const paths = await listFunctionsDirectories(srcFolders) - const functionsMap = await getFunctionsFromPaths(paths, { config, featureFlags }) + const cache = new RuntimeCache() + const functionsMap = await getFunctionsFromPaths(paths, { cache, config, featureFlags }) const functions = [...functionsMap.values()] const augmentedFunctions = parseISC ? await Promise.all(functions.map(augmentWithISC)) : functions const listedFunctionsFiles = await Promise.all( diff --git a/src/runtimes/detect_runtime.ts b/src/runtimes/detect_runtime.ts index 1c1fa63d1..ff2fae806 100644 --- a/src/runtimes/detect_runtime.ts +++ b/src/runtimes/detect_runtime.ts @@ -1,9 +1,7 @@ -import type { Buffer } from 'buffer' +import { readFile } from 'fs/promises' import { detect, Runtime as BinaryRuntime, Arch, Platform, BinaryInfo } from '@netlify/binary-info' -import { cachedReadFile, FsCache } from '../utils/fs.js' - import { RuntimeType } from './runtime.js' const isValidFunctionBinary = (info: BinaryInfo) => info.arch === Arch.Amd64 && info.platform === Platform.Linux @@ -21,20 +19,10 @@ The binary needs to be built for Linux/Amd64, but it was built for ${Platform[bi } // Try to guess the runtime by inspecting the binary file. -export const detectBinaryRuntime = async function ({ - fsCache, - path, -}: { - fsCache: FsCache - path: string -}): Promise { +export const detectBinaryRuntime = async function ({ path }: { path: string }): Promise { try { - const buffer = await cachedReadFile(fsCache, path) - - // We're using the Type Assertion because the `cachedReadFile` abstraction - // loses part of the return type information. We can safely say it's a - // Buffer in this case because we're not specifying an encoding. - const binaryInfo = detect(buffer as Buffer) + const fileContents = await readFile(path) + const binaryInfo = detect(fileContents) if (!isValidFunctionBinary(binaryInfo)) { return warnIncompatibleBinary(path, binaryInfo) diff --git a/src/runtimes/go/index.ts b/src/runtimes/go/index.ts index 15adfd958..bcf60d9d8 100644 --- a/src/runtimes/go/index.ts +++ b/src/runtimes/go/index.ts @@ -4,7 +4,8 @@ import { basename, dirname, extname, join } from 'path' import { copyFile } from 'cp-file' import { SourceFile } from '../../function.js' -import { cachedLstat, cachedReaddir, FsCache } from '../../utils/fs.js' +import type { RuntimeCache } from '../../utils/cache.js' +import { cachedLstat, cachedReaddir } from '../../utils/fs.js' import { nonNullable } from '../../utils/non_nullable.js' import { zipBinary } from '../../zip_binary.js' import { detectBinaryRuntime } from '../detect_runtime.js' @@ -23,8 +24,8 @@ interface GoBinary { stat: Stats } -const detectGoFunction = async ({ fsCache, path }: { fsCache: FsCache; path: string }) => { - const stat = await cachedLstat(fsCache, path) +const detectGoFunction = async ({ cache, path }: { cache: RuntimeCache; path: string }) => { + const stat = await cachedLstat(cache.lstatCache, path) if (!stat.isDirectory()) { return @@ -32,9 +33,7 @@ const detectGoFunction = async ({ fsCache, path }: { fsCache: FsCache; path: str const directoryName = basename(path) - // @ts-expect-error TODO: The `makeCachedFunction` abstraction is causing the - // return value of `readdir` to be incorrectly typed. - const files = (await cachedReaddir(fsCache, path)) as string[] + const files = await cachedReaddir(cache.readdirCache, path) const mainFileName = [`${directoryName}.go`, 'main.go'].find((name) => files.includes(name)) if (mainFileName === undefined) { @@ -44,28 +43,28 @@ const detectGoFunction = async ({ fsCache, path }: { fsCache: FsCache; path: str return mainFileName } -const findFunctionsInPaths: FindFunctionsInPathsFunction = async function ({ featureFlags, fsCache, paths }) { - const functions = await Promise.all(paths.map((path) => findFunctionInPath({ featureFlags, fsCache, path }))) +const findFunctionsInPaths: FindFunctionsInPathsFunction = async function ({ cache, featureFlags, paths }) { + const functions = await Promise.all(paths.map((path) => findFunctionInPath({ cache, featureFlags, path }))) return functions.filter(nonNullable) } -const findFunctionInPath: FindFunctionInPathFunction = async function ({ fsCache, path }) { - const runtime = await detectBinaryRuntime({ fsCache, path }) +const findFunctionInPath: FindFunctionInPathFunction = async function ({ cache, path }) { + const runtime = await detectBinaryRuntime({ path }) if (runtime === RuntimeType.GO) { - return processBinary({ fsCache, path }) + return processBinary({ cache, path }) } - const goSourceFile = await detectGoFunction({ fsCache, path }) + const goSourceFile = await detectGoFunction({ cache, path }) if (goSourceFile) { - return processSource({ fsCache, mainFile: goSourceFile, path }) + return processSource({ cache, mainFile: goSourceFile, path }) } } -const processBinary = async ({ fsCache, path }: { fsCache: FsCache; path: string }): Promise => { - const stat = (await cachedLstat(fsCache, path)) as Stats +const processBinary = async ({ cache, path }: { cache: RuntimeCache; path: string }): Promise => { + const stat = await cachedLstat(cache.lstatCache, path) const extension = extname(path) const filename = basename(path) const name = basename(path, extname(path)) @@ -82,11 +81,11 @@ const processBinary = async ({ fsCache, path }: { fsCache: FsCache; path: string } const processSource = async ({ - fsCache, + cache, mainFile, path, }: { - fsCache: FsCache + cache: RuntimeCache mainFile: string path: string }): Promise => { @@ -94,7 +93,7 @@ const processSource = async ({ // the `FunctionSource` interface. We should revisit whether `stat` should be // part of that interface in the first place, or whether we could compute it // downstream when needed (maybe using the FS cache as an optimisation). - const stat = (await cachedLstat(fsCache, path)) as Stats + const stat = (await cachedLstat(cache.lstatCache, path)) as Stats const filename = basename(path) const extension = extname(mainFile) const name = basename(path, extname(path)) diff --git a/src/runtimes/index.ts b/src/runtimes/index.ts index f46e1bd14..4d9274c56 100644 --- a/src/runtimes/index.ts +++ b/src/runtimes/index.ts @@ -3,7 +3,7 @@ import { extname, basename } from 'path' import { Config, getConfigForFunction, FunctionWithoutConfig } from '../config.js' import { defaultFlags, FeatureFlags } from '../feature_flags.js' import { FunctionSource } from '../function.js' -import { FsCache } from '../utils/fs.js' +import type { RuntimeCache } from '../utils/cache.js' import goRuntime from './go/index.js' import jsRuntime from './node/index.js' @@ -27,19 +27,19 @@ type FunctionTupleWithoutConfig = [string, FunctionWithoutConfig] * (`remainingPaths`). */ const findFunctionsInRuntime = async function ({ + cache, dedupe = false, featureFlags, - fsCache, paths, runtime, }: { + cache: RuntimeCache dedupe: boolean featureFlags: FeatureFlags - fsCache: FsCache paths: string[] runtime: Runtime }) { - const functions = await runtime.findFunctionsInPaths({ featureFlags, fsCache, paths }) + const functions = await runtime.findFunctionsInPaths({ cache, featureFlags, paths }) // If `dedupe` is true, we use the function name (`filename`) as the map key, // so that `function-1.js` will overwrite `function-1.go`. Otherwise, we use @@ -62,11 +62,6 @@ const findFunctionsInRuntime = async function ({ return { functions: augmentedFunctions, remainingPaths } } -// An object to cache filesystem operations. This allows different functions -// to perform IO operations on the same file (i.e. getting its stats or its -// contents) without duplicating work. -const makeFsCache = (): FsCache => ({}) - // The order of this array determines the priority of the runtimes. If a path // is used by the first time, it won't be made available to the subsequent // runtimes. @@ -78,14 +73,19 @@ const RUNTIMES = [jsRuntime, goRuntime, rustRuntime] export const getFunctionsFromPaths = async ( paths: string[], { + cache, config, configFileDirectories = [], dedupe = false, featureFlags = defaultFlags, - }: { config?: Config; configFileDirectories?: string[]; dedupe?: boolean; featureFlags?: FeatureFlags } = {}, + }: { + cache: RuntimeCache + config?: Config + configFileDirectories?: string[] + dedupe?: boolean + featureFlags?: FeatureFlags + }, ): Promise => { - const fsCache = makeFsCache() - // We cycle through the ordered array of runtimes, passing each one of them // through `findFunctionsInRuntime`. For each iteration, we collect all the // functions found plus the list of paths that still need to be evaluated, @@ -93,9 +93,9 @@ export const getFunctionsFromPaths = async ( const { functions } = await RUNTIMES.reduce(async (aggregate, runtime) => { const { functions: aggregateFunctions, remainingPaths: aggregatePaths } = await aggregate const { functions: runtimeFunctions, remainingPaths: runtimePaths } = await findFunctionsInRuntime({ + cache, dedupe, featureFlags, - fsCache, paths: aggregatePaths, runtime, }) @@ -121,12 +121,10 @@ export const getFunctionsFromPaths = async ( */ export const getFunctionFromPath = async ( path: string, - { config, featureFlags = defaultFlags }: { config?: Config; featureFlags?: FeatureFlags } = {}, + { cache, config, featureFlags = defaultFlags }: { cache: RuntimeCache; config?: Config; featureFlags?: FeatureFlags }, ): Promise => { - const fsCache = makeFsCache() - for (const runtime of RUNTIMES) { - const func = await runtime.findFunctionInPath({ path, fsCache, featureFlags }) + const func = await runtime.findFunctionInPath({ path, cache, featureFlags }) if (func) { const functionConfig = await getConfigForFunction({ config, func: { ...func, runtime }, featureFlags }) diff --git a/src/runtimes/node/bundlers/esbuild/plugin_dynamic_imports.ts b/src/runtimes/node/bundlers/esbuild/plugin_dynamic_imports.ts index e37a6d533..86ed9f8c9 100644 --- a/src/runtimes/node/bundlers/esbuild/plugin_dynamic_imports.ts +++ b/src/runtimes/node/bundlers/esbuild/plugin_dynamic_imports.ts @@ -3,9 +3,9 @@ import { basename, join, relative } from 'path' import type { Plugin } from '@netlify/esbuild' import { findUp, findUpStop, pathExists } from 'find-up' import normalizePath from 'normalize-path' -import readPackageJson from 'read-package-json-fast' import { parseExpression } from '../../parser/index.js' +import { readPackageJson } from '../../utils/package_json.js' type PackageCache = Map> @@ -102,11 +102,14 @@ const getPackageNameCached = ({ resolveDir: string srcDir: string }) => { - if (!cache.has(resolveDir)) { - cache.set(resolveDir, getPackageName({ resolveDir, srcDir })) + let result = cache.get(resolveDir) + + if (result === undefined) { + result = getPackageName({ resolveDir, srcDir }) + cache.set(resolveDir, result) } - return cache.get(resolveDir) + return result } const getShimContents = ({ diff --git a/src/runtimes/node/bundlers/esbuild/plugin_native_modules.ts b/src/runtimes/node/bundlers/esbuild/plugin_native_modules.ts index ebb1f4d2b..e2fce8817 100644 --- a/src/runtimes/node/bundlers/esbuild/plugin_native_modules.ts +++ b/src/runtimes/node/bundlers/esbuild/plugin_native_modules.ts @@ -1,10 +1,9 @@ import path from 'path' import type { Plugin } from '@netlify/esbuild' -import readPackageJson from 'read-package-json-fast' import { isNativeModule } from '../../utils/detect_native_module.js' -import { PackageJson } from '../../utils/package_json.js' +import { PackageJson, readPackageJson } from '../../utils/package_json.js' import type { NativeNodeModules } from '../types.js' type NativeModuleCacheEntry = [boolean | undefined, PackageJson] diff --git a/src/runtimes/node/bundlers/esbuild/src_files.ts b/src/runtimes/node/bundlers/esbuild/src_files.ts index ee9f68c63..0d05dac68 100644 --- a/src/runtimes/node/bundlers/esbuild/src_files.ts +++ b/src/runtimes/node/bundlers/esbuild/src_files.ts @@ -11,8 +11,8 @@ export const getSrcFiles: GetSrcFilesFunction = async ({ config, mainFile, plugi includedFilesBasePath, ) const dependencyPaths = await getSrcFilesForDependencies({ - dependencies: externalNodeModules, basedir: srcDir, + dependencies: externalNodeModules, pluginsModulesPath, }) const srcFiles = filterExcludedPaths(dependencyPaths, excludePatterns) @@ -25,13 +25,13 @@ export const getSrcFiles: GetSrcFilesFunction = async ({ config, mainFile, plugi } const getSrcFilesForDependencies = async function ({ - dependencies: dependencyNames, basedir, + dependencies: dependencyNames, state = getNewCache(), pluginsModulesPath, }: { - dependencies: string[] basedir: string + dependencies: string[] state?: TraversalCache pluginsModulesPath?: string }) { diff --git a/src/runtimes/node/bundlers/nft/es_modules.ts b/src/runtimes/node/bundlers/nft/es_modules.ts index e656a6b06..a00454f3e 100644 --- a/src/runtimes/node/bundlers/nft/es_modules.ts +++ b/src/runtimes/node/bundlers/nft/es_modules.ts @@ -4,15 +4,16 @@ import { NodeFileTraceReasons } from '@vercel/nft' import type { FunctionConfig } from '../../../../config.js' import { FeatureFlags } from '../../../../feature_flags.js' -import { cachedReadFile, FsCache } from '../../../../utils/fs.js' +import type { RuntimeCache } from '../../../../utils/cache.js' +import { cachedReadFile } from '../../../../utils/fs.js' import { ModuleFileExtension, ModuleFormat } from '../../utils/module_format.js' import { getNodeSupportMatrix } from '../../utils/node_version.js' import { getPackageJsonIfAvailable, PackageJson } from '../../utils/package_json.js' import { transpile } from './transpile.js' -const getPatchedESMPackages = async (packages: string[], fsCache: FsCache) => { - const patchedPackages = await Promise.all(packages.map((path) => patchESMPackage(path, fsCache))) +const getPatchedESMPackages = async (packages: string[], cache: RuntimeCache) => { + const patchedPackages = await Promise.all(packages.map((path) => patchESMPackage(path, cache))) const patchedPackagesMap = new Map() packages.forEach((packagePath, index) => { @@ -37,8 +38,8 @@ const isEntrypointESM = ({ return entrypointIsESM } -const patchESMPackage = async (path: string, fsCache: FsCache) => { - const file = (await cachedReadFile(fsCache, path, 'utf8')) as string +const patchESMPackage = async (path: string, cache: RuntimeCache) => { + const file = await cachedReadFile(cache.fileCache, path) const packageJson: PackageJson = JSON.parse(file) const patchedPackageJson = { ...packageJson, @@ -50,19 +51,19 @@ const patchESMPackage = async (path: string, fsCache: FsCache) => { export const processESM = async ({ basePath, + cache, config, esmPaths, featureFlags, - fsCache, mainFile, reasons, name, }: { basePath: string | undefined + cache: RuntimeCache config: FunctionConfig esmPaths: Set featureFlags: FeatureFlags - fsCache: FsCache mainFile: string reasons: NodeFileTraceReasons name: string @@ -94,7 +95,7 @@ export const processESM = async ({ } } - const rewrites = await transpileESM({ basePath, config, esmPaths, fsCache, reasons, name }) + const rewrites = await transpileESM({ basePath, cache, config, esmPaths, reasons, name }) return { moduleFormat: ModuleFormat.COMMONJS, @@ -148,21 +149,22 @@ const shouldTranspile = ( const transpileESM = async ({ basePath, + cache, config, esmPaths, - fsCache, reasons, name, }: { basePath: string | undefined + cache: RuntimeCache config: FunctionConfig esmPaths: Set - fsCache: FsCache reasons: NodeFileTraceReasons name: string }) => { - const cache: Map = new Map() - const pathsToTranspile = [...esmPaths].filter((path) => shouldTranspile(path, cache, esmPaths, reasons)) + // Used for memoizing the check for whether a path should be transpiled. + const shouldCompileCache: Map = new Map() + const pathsToTranspile = [...esmPaths].filter((path) => shouldTranspile(path, shouldCompileCache, esmPaths, reasons)) const pathsToTranspileSet = new Set(pathsToTranspile) const packageJsonPaths: string[] = [...reasons.entries()] .filter(([path, reason]) => { @@ -175,7 +177,7 @@ const transpileESM = async ({ return needsPatch }) .map(([path]) => (basePath ? resolve(basePath, path) : resolve(path))) - const rewrites = await getPatchedESMPackages(packageJsonPaths, fsCache) + const rewrites = await getPatchedESMPackages(packageJsonPaths, cache) await Promise.all( pathsToTranspile.map(async (path) => { diff --git a/src/runtimes/node/bundlers/nft/index.ts b/src/runtimes/node/bundlers/nft/index.ts index 6ea41a57c..6ca107614 100644 --- a/src/runtimes/node/bundlers/nft/index.ts +++ b/src/runtimes/node/bundlers/nft/index.ts @@ -5,7 +5,8 @@ import nftResolveDependency from '@vercel/nft/out/resolve-dependency.js' import type { FunctionConfig } from '../../../../config.js' import { FeatureFlags } from '../../../../feature_flags.js' -import { cachedReadFile, FsCache } from '../../../../utils/fs.js' +import type { RuntimeCache } from '../../../../utils/cache.js' +import { cachedReadFile } from '../../../../utils/fs.js' import { minimatch } from '../../../../utils/matching.js' import { getBasePath } from '../../utils/base_path.js' import { filterExcludedPaths, getPathsOfIncludedFiles } from '../../utils/included_files.js' @@ -25,6 +26,7 @@ const appearsToBeModuleName = (name: string) => !name.startsWith('.') const bundle: BundleFunction = async ({ basePath, + cache, config, featureFlags, mainFile, @@ -43,6 +45,7 @@ const bundle: BundleFunction = async ({ rewrites, } = await traceFilesAndTranspile({ basePath: repositoryRoot, + cache, config, featureFlags, mainFile, @@ -75,6 +78,7 @@ const ignoreFunction = (path: string) => { const traceFilesAndTranspile = async function ({ basePath, + cache, config, featureFlags, mainFile, @@ -82,23 +86,26 @@ const traceFilesAndTranspile = async function ({ name, }: { basePath?: string + cache: RuntimeCache config: FunctionConfig featureFlags: FeatureFlags mainFile: string pluginsModulesPath?: string name: string }) { - const fsCache: FsCache = {} const { fileList: dependencyPaths, esmFileList, reasons, } = await nodeFileTrace([mainFile], { + // Default is 1024. Allowing double the fileIO in parallel makes nft faster, but uses a little more memory. + fileIOConcurrency: featureFlags.zisi_nft_higher_fileio_limit ? 2048 : 1024, base: basePath, + cache: featureFlags.zisi_nft_use_cache ? cache.nftCache : undefined, ignore: ignoreFunction, readFile: async (path: string) => { try { - const source = (await cachedReadFile(fsCache, path, 'utf8')) as string + const source = await cachedReadFile(cache.fileCache, path) return source } catch (error) { @@ -131,10 +138,10 @@ const traceFilesAndTranspile = async function ({ ) const { moduleFormat, rewrites } = await processESM({ basePath, + cache, config, esmPaths: esmFileList, featureFlags, - fsCache, mainFile, reasons, name, diff --git a/src/runtimes/node/bundlers/types.ts b/src/runtimes/node/bundlers/types.ts index 84e3e219c..84b539460 100644 --- a/src/runtimes/node/bundlers/types.ts +++ b/src/runtimes/node/bundlers/types.ts @@ -3,6 +3,7 @@ import type { Message } from '@netlify/esbuild' import type { FunctionConfig } from '../../../config.js' import type { FeatureFlag, FeatureFlags } from '../../../feature_flags.js' import type { FunctionSource } from '../../../function.js' +import type { RuntimeCache } from '../../../utils/cache.js' import type { ModuleFormat } from '../utils/module_format.js' export const enum NodeBundlerType { @@ -23,6 +24,7 @@ export type NativeNodeModules = Record pluginsModulesPath?: string diff --git a/src/runtimes/node/finder.ts b/src/runtimes/node/finder.ts index fe94a9ab4..4d1ff2953 100644 --- a/src/runtimes/node/finder.ts +++ b/src/runtimes/node/finder.ts @@ -1,9 +1,10 @@ -import { Stats, promises as fs } from 'fs' +import type { Stats } from 'fs' import { join, dirname, basename, extname } from 'path' import { locatePath } from 'locate-path' import { SourceFile } from '../../function.js' +import { cachedLstat } from '../../utils/fs.js' import { nonNullable } from '../../utils/non_nullable.js' import { FindFunctionsInPathsFunction, FindFunctionInPathFunction } from '../runtime.js' @@ -21,8 +22,8 @@ const sortByExtension = (fA: SourceFile, fB: SourceFile) => { return indexB - indexA } -export const findFunctionsInPaths: FindFunctionsInPathsFunction = async function ({ paths, fsCache, featureFlags }) { - const functions = await Promise.all(paths.map((path) => findFunctionInPath({ path, fsCache, featureFlags }))) +export const findFunctionsInPaths: FindFunctionsInPathsFunction = async function ({ cache, featureFlags, paths }) { + const functions = await Promise.all(paths.map((path) => findFunctionInPath({ cache, featureFlags, path }))) // It's fine to mutate the array since its scope is local to this function. const sortedFunctions = functions.filter(nonNullable).sort((fA, fB) => { @@ -46,14 +47,14 @@ export const findFunctionsInPaths: FindFunctionsInPathsFunction = async function return sortedFunctions } -export const findFunctionInPath: FindFunctionInPathFunction = async function ({ path: srcPath }) { +export const findFunctionInPath: FindFunctionInPathFunction = async function ({ cache, path: srcPath }) { const filename = basename(srcPath) if (filename === 'node_modules') { return } - const stat = await fs.lstat(srcPath) + const stat = await cachedLstat(cache.lstatCache, srcPath) const mainFile = await getMainFile(srcPath, filename, stat) if (mainFile === undefined) { diff --git a/src/runtimes/node/index.ts b/src/runtimes/node/index.ts index 038ee787b..c3feb3a34 100644 --- a/src/runtimes/node/index.ts +++ b/src/runtimes/node/index.ts @@ -32,6 +32,7 @@ const getSrcFilesWithBundler: GetSrcFilesFunction = async (parameters) => { const zipFunction: ZipFunction = async function ({ archiveFormat, basePath, + cache, config = {}, destFolder, extension, @@ -79,6 +80,7 @@ const zipFunction: ZipFunction = async function ({ srcFiles, } = await bundler.bundle({ basePath, + cache, config, extension, featureFlags, @@ -101,6 +103,7 @@ const zipFunction: ZipFunction = async function ({ aliases, archiveFormat, basePath: finalBasePath, + cache, destFolder, extension, featureFlags, diff --git a/src/runtimes/node/parser/index.ts b/src/runtimes/node/parser/index.ts index 780feea0c..bde8e407a 100644 --- a/src/runtimes/node/parser/index.ts +++ b/src/runtimes/node/parser/index.ts @@ -110,6 +110,10 @@ const parseFile = async (path: string) => { const ast = parse(code, { plugins: ['typescript'], sourceType: 'module', + // disable tokens, ranges and comments for performance and we do not use them + tokens: false, + ranges: false, + attachComment: false, }) return ast.program diff --git a/src/runtimes/node/utils/package_json.ts b/src/runtimes/node/utils/package_json.ts index f661cfdfb..576937f9b 100644 --- a/src/runtimes/node/utils/package_json.ts +++ b/src/runtimes/node/utils/package_json.ts @@ -68,18 +68,18 @@ export const getPackageJsonIfAvailable = async (srcDir: string): Promise { +export const readPackageJson = async (path: string) => { try { // The path depends on the user's build, i.e. must be dynamic const packageJson = JSON.parse(await fs.readFile(path, 'utf8')) - return sanitisePackageJson(packageJson) + return sanitizePackageJson(packageJson) } catch (error) { throw new Error(`${path} is invalid JSON: ${error.message}`) } } -const sanitiseFiles = (files: unknown): string[] | undefined => { +const sanitizeFiles = (files: unknown): string[] | undefined => { if (!Array.isArray(files)) { return undefined } @@ -87,7 +87,7 @@ const sanitiseFiles = (files: unknown): string[] | undefined => { return files.filter((file) => typeof file === 'string') } -export const sanitisePackageJson = (packageJson: Record): PackageJson => ({ +export const sanitizePackageJson = (packageJson: Record): PackageJson => ({ ...packageJson, - files: sanitiseFiles(packageJson.files), + files: sanitizeFiles(packageJson.files), }) diff --git a/src/runtimes/node/utils/zip.ts b/src/runtimes/node/utils/zip.ts index 382fd7015..335393b02 100644 --- a/src/runtimes/node/utils/zip.ts +++ b/src/runtimes/node/utils/zip.ts @@ -9,7 +9,8 @@ import pMap from 'p-map' import { startZip, addZipFile, addZipContent, endZip, ZipArchive } from '../../../archive.js' import type { FeatureFlags } from '../../../feature_flags.js' -import { mkdirAndWriteFile } from '../../../utils/fs.js' +import type { RuntimeCache } from '../../../utils/cache.js' +import { cachedLstat, mkdirAndWriteFile } from '../../../utils/fs.js' import { getEntryFile } from './entry_file.js' import { getFileExtensionForFormat, ModuleFormat } from './module_format.js' @@ -27,6 +28,7 @@ export type ArchiveFormat = 'none' | 'zip' interface ZipNodeParameters { aliases?: Map basePath: string + cache: RuntimeCache destFolder: string extension: string featureFlags: FeatureFlags @@ -94,6 +96,7 @@ const createDirectory = async function ({ const createZipArchive = async function ({ aliases, basePath, + cache, destFolder, extension, featureFlags, @@ -128,7 +131,7 @@ const createZipArchive = async function ({ addEntryFileToZip(archive, entryFile, basename(entryFilePath)) } - const srcFilesInfos = await Promise.all(srcFiles.map(addStat)) + const srcFilesInfos = await Promise.all(srcFiles.map((file) => addStat(cache, file))) // We ensure this is not async, so that the archive's checksum is // deterministic. Otherwise it depends on the order the files were added. @@ -166,8 +169,8 @@ const addEntryFileToZip = function (archive: ZipArchive, contents: string, filen addZipContent(archive, contentBuffer, filename) } -const addStat = async function (srcFile: string) { - const stat = await fs.lstat(srcFile) +const addStat = async function (cache: RuntimeCache, srcFile: string) { + const stat = await cachedLstat(cache.lstatCache, srcFile) return { srcFile, stat } } diff --git a/src/runtimes/runtime.ts b/src/runtimes/runtime.ts index 26660a188..63ff6cdc7 100644 --- a/src/runtimes/runtime.ts +++ b/src/runtimes/runtime.ts @@ -1,8 +1,8 @@ -import { ArchiveFormat } from '../archive.js' -import { FunctionConfig } from '../config.js' -import { FeatureFlags } from '../feature_flags.js' -import { FunctionSource, SourceFile } from '../function.js' -import { FsCache } from '../utils/fs.js' +import type { ArchiveFormat } from '../archive.js' +import type { FunctionConfig } from '../config.js' +import type { FeatureFlags } from '../feature_flags.js' +import type { FunctionSource, SourceFile } from '../function.js' +import type { RuntimeCache } from '../utils/cache.js' import type { NodeBundlerType } from './node/bundlers/types.js' import type { ISCValues } from './node/in_source_config/index.js' @@ -14,14 +14,14 @@ export const enum RuntimeType { } export type FindFunctionsInPathsFunction = (args: { + cache: RuntimeCache featureFlags: FeatureFlags - fsCache: FsCache paths: string[] }) => Promise export type FindFunctionInPathFunction = (args: { + cache: RuntimeCache featureFlags: FeatureFlags - fsCache: FsCache path: string }) => Promise @@ -51,6 +51,7 @@ export type ZipFunction = ( args: { archiveFormat: ArchiveFormat basePath?: string + cache: RuntimeCache config: FunctionConfig destFolder: string featureFlags: FeatureFlags diff --git a/src/runtimes/rust/builder.ts b/src/runtimes/rust/builder.ts index c36a4687c..6f3a8cafb 100644 --- a/src/runtimes/rust/builder.ts +++ b/src/runtimes/rust/builder.ts @@ -1,18 +1,30 @@ -import { promises as fs } from 'fs' +import { mkdir } from 'fs/promises' import { basename, join } from 'path' import tmp from 'tmp-promise' import toml from 'toml' -import { FunctionConfig } from '../../config.js' +import type { FunctionConfig } from '../../config.js' +import type { RuntimeCache } from '../../utils/cache.js' import { FunctionBundlingUserError } from '../../utils/error.js' +import { cachedLstat, cachedReadFile } from '../../utils/fs.js' import { shellUtils } from '../../utils/shell.js' import { RuntimeType } from '../runtime.js' import { CargoManifest } from './cargo_manifest.js' import { BUILD_TARGET, MANIFEST_NAME } from './constants.js' -export const build = async ({ config, name, srcDir }: { config: FunctionConfig; name: string; srcDir: string }) => { +export const build = async ({ + cache, + config, + name, + srcDir, +}: { + cache: RuntimeCache + config: FunctionConfig + name: string + srcDir: string +}) => { const functionName = basename(srcDir) try { @@ -29,12 +41,12 @@ export const build = async ({ config, name, srcDir }: { config: FunctionConfig; // way to override it (https://github.com/rust-lang/cargo/issues/1706). We // must extract the crate name from the manifest and use it to form the path // to the binary. - const manifest = await fs.readFile(join(srcDir, MANIFEST_NAME), 'utf8') + const manifest = await cachedReadFile(cache.fileCache, join(srcDir, MANIFEST_NAME)) const { package: { name: packageName }, }: CargoManifest = toml.parse(manifest) const binaryPath = join(targetDirectory, BUILD_TARGET, 'release', packageName) - const stat = await fs.lstat(binaryPath) + const stat = await cachedLstat(cache.lstatCache, binaryPath) return { path: binaryPath, @@ -91,7 +103,7 @@ const getTargetDirectory = async ({ config, name }: { config: FunctionConfig; na // We replace the [name] placeholder with the name of the function. const path = rustTargetDirectory.replace(/\[name]/g, name) - await fs.mkdir(path, { recursive: true }) + await mkdir(path, { recursive: true }) return path } diff --git a/src/runtimes/rust/index.ts b/src/runtimes/rust/index.ts index 9b15ee6c8..51bfb5cf7 100644 --- a/src/runtimes/rust/index.ts +++ b/src/runtimes/rust/index.ts @@ -3,7 +3,8 @@ import { join, extname, dirname, basename } from 'path' import { FeatureFlags } from '../../feature_flags.js' import { SourceFile } from '../../function.js' -import { cachedLstat, cachedReaddir, FsCache } from '../../utils/fs.js' +import type { RuntimeCache } from '../../utils/cache.js' +import { cachedLstat, cachedReaddir } from '../../utils/fs.js' import { nonNullable } from '../../utils/non_nullable.js' import { zipBinary } from '../../zip_binary.js' import { detectBinaryRuntime } from '../detect_runtime.js' @@ -18,16 +19,14 @@ import { import { build } from './builder.js' import { MANIFEST_NAME } from './constants.js' -const detectRustFunction = async ({ fsCache, path }: { fsCache: FsCache; path: string }) => { - const stat = await cachedLstat(fsCache, path) +const detectRustFunction = async ({ cache, path }: { cache: RuntimeCache; path: string }) => { + const stat = await cachedLstat(cache.lstatCache, path) if (!stat.isDirectory()) { return } - // @ts-expect-error TODO: The `makeCachedFunction` abstraction is causing the - // return value of `readdir` to be incorrectly typed. - const files = (await cachedReaddir(fsCache, path)) as string[] + const files = await cachedReaddir(cache.readdirCache, path) const hasCargoManifest = files.includes(MANIFEST_NAME) if (!hasCargoManifest) { @@ -37,7 +36,7 @@ const detectRustFunction = async ({ fsCache, path }: { fsCache: FsCache; path: s const mainFilePath = join(path, 'src', 'main.rs') try { - const mainFile = await cachedLstat(fsCache, mainFilePath) + const mainFile = await cachedLstat(cache.lstatCache, mainFilePath) if (mainFile.isFile()) { return mainFilePath @@ -48,39 +47,39 @@ const detectRustFunction = async ({ fsCache, path }: { fsCache: FsCache; path: s } const findFunctionsInPaths: FindFunctionsInPathsFunction = async function ({ + cache, featureFlags, - fsCache, paths, }: { + cache: RuntimeCache featureFlags: FeatureFlags - fsCache: FsCache paths: string[] }) { - const functions = await Promise.all(paths.map((path) => findFunctionInPath({ path, featureFlags, fsCache }))) + const functions = await Promise.all(paths.map((path) => findFunctionInPath({ cache, featureFlags, path }))) return functions.filter(nonNullable) } -const findFunctionInPath: FindFunctionInPathFunction = async function ({ path, featureFlags, fsCache }) { - const runtime = await detectBinaryRuntime({ fsCache, path }) +const findFunctionInPath: FindFunctionInPathFunction = async function ({ cache, featureFlags, path }) { + const runtime = await detectBinaryRuntime({ path }) if (runtime === RuntimeType.RUST) { - return processBinary({ fsCache, path }) + return processBinary({ cache, path }) } if (featureFlags.buildRustSource !== true) { return } - const rustSourceFile = await detectRustFunction({ fsCache, path }) + const rustSourceFile = await detectRustFunction({ cache, path }) if (rustSourceFile) { - return processSource({ fsCache, mainFile: rustSourceFile, path }) + return processSource({ cache, mainFile: rustSourceFile, path }) } } -const processBinary = async ({ fsCache, path }: { fsCache: FsCache; path: string }): Promise => { - const stat = (await cachedLstat(fsCache, path)) as Stats +const processBinary = async ({ cache, path }: { cache: RuntimeCache; path: string }): Promise => { + const stat = (await cachedLstat(cache.lstatCache, path)) as Stats const filename = basename(path) const extension = extname(path) const name = basename(path, extension) @@ -97,11 +96,11 @@ const processBinary = async ({ fsCache, path }: { fsCache: FsCache; path: string } const processSource = async ({ - fsCache, + cache, mainFile, path, }: { - fsCache: FsCache + cache: RuntimeCache mainFile: string path: string }): Promise => { @@ -109,7 +108,7 @@ const processSource = async ({ // the `FunctionSource` interface. We should revisit whether `stat` should be // part of that interface in the first place, or whether we could compute it // downstream when needed (maybe using the FS cache as an optimisation). - const stat = (await cachedLstat(fsCache, path)) as Stats + const stat = (await cachedLstat(cache.lstatCache, path)) as Stats const filename = basename(path) const extension = extname(path) const name = basename(path, extension) @@ -129,6 +128,7 @@ const processSource = async ({ // because they include the Lambda runtime, and that's the name that AWS // expects for those kind of functions. const zipFunction: ZipFunction = async function ({ + cache, config, destFolder, filename, @@ -150,7 +150,7 @@ const zipFunction: ZipFunction = async function ({ // the resulting binary. Otherwise, we're dealing with a binary so we zip it // directly. if (isSource) { - const { path: binaryPath, stat: binaryStat } = await build({ config, name: filename, srcDir }) + const { path: binaryPath, stat: binaryStat } = await build({ cache, config, name: filename, srcDir }) await zipBinary({ ...zipOptions, srcPath: binaryPath, stat: binaryStat }) } else { diff --git a/src/utils/cache.ts b/src/utils/cache.ts new file mode 100644 index 000000000..8260fbcce --- /dev/null +++ b/src/utils/cache.ts @@ -0,0 +1,32 @@ +import type { Stats } from 'fs' + +export type FileCache = Map> +export type LstatCache = Map> +export type ReaddirCache = Map> + +interface NFTCache { + fileCache: FileCache + // nft actually sets even more properties on this object, but + // they do not have any relevance for us here +} + +export class RuntimeCache { + // Cache for fs.readFile() calls + fileCache: FileCache + // Cache for fs.lstat() calls + lstatCache: LstatCache + // Cache fs.readdir() calls + readdirCache: ReaddirCache + // NFT cache, which should not be used in zisi and only supplied to NFT + // this cache shares the file cache with zisi + nftCache: NFTCache + + constructor() { + this.fileCache = new Map() + this.lstatCache = new Map() + this.readdirCache = new Map() + + this.nftCache = Object.create(null) + this.nftCache.fileCache = this.fileCache + } +} diff --git a/src/utils/fs.ts b/src/utils/fs.ts index c06111810..20ca5acde 100644 --- a/src/utils/fs.ts +++ b/src/utils/fs.ts @@ -1,32 +1,45 @@ -import { promises as fs } from 'fs' +import { promises as fs, Stats } from 'fs' import { dirname, format, join, parse, resolve } from 'path' +import { FileCache, LstatCache, ReaddirCache } from './cache.js' import { nonNullable } from './non_nullable.js' -export type FsCache = Record - -// This caches multiple FS calls to the same path. It creates a cache key with -// the name of the function and the path (e.g. "readdir:/some/directory"). -// -// TODO: This abstraction is stripping out some type data. For example, when -// calling `readFile` without an encoding, the return type should be narrowed -// down from `string | Buffer` to `Buffer`, but that's not happening. -const makeCachedFunction = - (func: (path: string, ...args: Args) => ReturnType) => - (cache: FsCache, path: string, ...args: Args): ReturnType => { - const key = `${func.name}:${path}` - - if (cache[key] === undefined) { - // eslint-disable-next-line no-param-reassign - cache[key] = func(path, ...args) - } - - return cache[key] as ReturnType +export const cachedLstat = (cache: LstatCache, path: string): Promise => { + let result = cache.get(path) + + if (result === undefined) { + // no await as we want to populate the cache instantly with the promise + result = fs.lstat(path) + cache.set(path, result) + } + + return result +} + +export const cachedReaddir = (cache: ReaddirCache, path: string): Promise => { + let result = cache.get(path) + + if (result === undefined) { + // no await as we want to populate the cache instantly with the promise + result = fs.readdir(path, 'utf-8') + cache.set(path, result) } -export const cachedLstat = makeCachedFunction(fs.lstat) -export const cachedReaddir = makeCachedFunction(fs.readdir) -export const cachedReadFile = makeCachedFunction(fs.readFile) + return result +} + +export const cachedReadFile = (cache: FileCache, path: string): Promise => { + let result = cache.get(path) + + // Check for null here, as we use the same cache in NFT which sets null on a not found file + if (result === undefined || result === null) { + // no await as we want to populate the cache instantly with the promise + result = fs.readFile(path, 'utf-8') + cache.set(path, result) + } + + return result +} export const getPathWithExtension = (path: string, extension: string) => format({ ...parse(path), base: undefined, ext: extension }) diff --git a/src/zip.ts b/src/zip.ts index a33213dc1..9b36e2390 100644 --- a/src/zip.ts +++ b/src/zip.ts @@ -11,6 +11,7 @@ import { createManifest } from './manifest.js' import { getFunctionsFromPaths } from './runtimes/index.js' import { ModuleFormat } from './runtimes/node/utils/module_format.js' import { addArchiveSize } from './utils/archive_size.js' +import { RuntimeCache } from './utils/cache.js' import { formatZipResult } from './utils/format_result.js' import { listFunctionsDirectories, resolveFunctionsDirectories } from './utils/fs.js' import { nonNullable } from './utils/non_nullable.js' @@ -57,10 +58,17 @@ export const zipFunctions = async function ( ) { validateArchiveFormat(archiveFormat) + const cache = new RuntimeCache() const featureFlags = getFlags(inputFeatureFlags) const srcFolders = resolveFunctionsDirectories(relativeSrcFolders) const [paths] = await Promise.all([listFunctionsDirectories(srcFolders), fs.mkdir(destFolder, { recursive: true })]) - const functions = await getFunctionsFromPaths(paths, { config, configFileDirectories, dedupe: true, featureFlags }) + const functions = await getFunctionsFromPaths(paths, { + cache, + config, + configFileDirectories, + dedupe: true, + featureFlags, + }) const results = await pMap( functions.values(), async (func) => { @@ -75,6 +83,7 @@ export const zipFunctions = async function ( const zipResult = await func.runtime.zipFunction({ archiveFormat, basePath, + cache, config: func.config, destFolder, extension: func.extension, @@ -125,7 +134,8 @@ export const zipFunction = async function ( const featureFlags = getFlags(inputFeatureFlags) const srcPath = resolve(relativeSrcPath) - const functions = await getFunctionsFromPaths([srcPath], { config: inputConfig, dedupe: true, featureFlags }) + const cache = new RuntimeCache() + const functions = await getFunctionsFromPaths([srcPath], { cache, config: inputConfig, dedupe: true, featureFlags }) if (functions.size === 0) { return @@ -154,6 +164,7 @@ export const zipFunction = async function ( const zipResult = await runtime.zipFunction({ archiveFormat, basePath, + cache, config, destFolder, extension, diff --git a/tests/unit/runtimes/node/utils/package_json.test.ts b/tests/unit/runtimes/node/utils/package_json.test.ts index b44492b42..da46a5908 100644 --- a/tests/unit/runtimes/node/utils/package_json.test.ts +++ b/tests/unit/runtimes/node/utils/package_json.test.ts @@ -1,10 +1,10 @@ import { describe, expect, test } from 'vitest' -import { sanitisePackageJson } from '../../../../../src/runtimes/node/utils/package_json.js' +import { sanitizePackageJson } from '../../../../../src/runtimes/node/utils/package_json.js' -describe('sanitisePackageJson', () => { +describe('sanitizePackageJson', () => { test('removes nulls from files', () => { - const result = sanitisePackageJson({ + const result = sanitizePackageJson({ files: ['a.js', null, 'b.js'], }) @@ -14,7 +14,7 @@ describe('sanitisePackageJson', () => { }) test('does not crash on invalid files entries', () => { - const result = sanitisePackageJson({ + const result = sanitizePackageJson({ files: { 'a.js': true, 'b.js': false }, })