1480 speedup npm install step for npm based formatters

CHANGES.md

@@ -13,10 +13,12 @@ We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format (
 ### Added
 * `gradlew equoIde` opens a repeatable clean Spotless dev environment. ([#1523](https://github.com/diffplug/spotless/pull/1523))
 * `cleanthat` added `includeDraft` option, to include draft mutators from composite mutators. ([#1574](https://github.com/diffplug/spotless/pull/1574))
+* `npm`-based formatters now support caching of `node_modules` directory ([#1590](https://github.com/diffplug/spotless/pull/1590))
 ### Fixed
 * `JacksonJsonFormatterFunc` handles json files with an Array as root. ([#1585](https://github.com/diffplug/spotless/pull/1585))
 ### Changes
 * Bump default `cleanthat` version to latest `2.1` -> `2.6` ([#1569](https://github.com/diffplug/spotless/pull/1569) and [#1574](https://github.com/diffplug/spotless/pull/1574))
+* Reduce logging-noise created by `npm`-based formatters ([#1590](https://github.com/diffplug/spotless/pull/1590) fixes [#1582](https://github.com/diffplug/spotless/issues/1582))
 
 ## [2.35.0] - 2023-02-10
 ### Added

lib/build.gradle

@@ -56,6 +56,8 @@ tasks.named("check").configure {
 
 dependencies {
 	compileOnly 'org.slf4j:slf4j-api:2.0.0'
+	testCommonImplementation 'org.slf4j:slf4j-api:2.0.0'
+
 	// zero runtime reqs is a hard requirements for spotless-lib
 	// if you need a dep, put it in lib-extra
 	testCommonImplementation "org.junit.jupiter:junit-jupiter:$VER_JUNIT"

lib/src/main/java/com/diffplug/spotless/npm/EslintFormatterStep.java

@@ -15,7 +15,6 @@
  */
 package com.diffplug.spotless.npm;
 
-import static com.diffplug.spotless.LazyArgLogger.lazy;
 import static java.util.Objects.requireNonNull;
 
 import java.io.File;
@@ -71,13 +70,13 @@ public static Map<String, String> defaultDevDependenciesWithEslint(String versio
 		return Collections.singletonMap("eslint", version);
 	}
 
-	public static FormatterStep create(Map<String, String> devDependencies, Provisioner provisioner, File projectDir, File buildDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) {
+	public static FormatterStep create(Map<String, String> devDependencies, Provisioner provisioner, File projectDir, File buildDir, File cacheDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) {
 		requireNonNull(devDependencies);
 		requireNonNull(provisioner);
 		requireNonNull(projectDir);
 		requireNonNull(buildDir);
 		return FormatterStep.createLazy(NAME,
-				() -> new State(NAME, devDependencies, projectDir, buildDir, npmPathResolver, eslintConfig),
+				() -> new State(NAME, devDependencies, projectDir, buildDir, cacheDir, npmPathResolver, eslintConfig),
 				State::createFormatterFunc);
 	}
 
@@ -89,20 +88,20 @@ private static class State extends NpmFormatterStepStateBase implements Serializ
 		@SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED")
 		private transient EslintConfig eslintConfigInUse;
 
-		State(String stepName, Map<String, String> devDependencies, File projectDir, File buildDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) throws IOException {
+		State(String stepName, Map<String, String> devDependencies, File projectDir, File buildDir, File cacheDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) throws IOException {
 			super(stepName,
 					new NpmConfig(
 							replaceDevDependencies(
 									NpmResourceHelper.readUtf8StringFromClasspath(EslintFormatterStep.class, "/com/diffplug/spotless/npm/eslint-package.json"),
 									new TreeMap<>(devDependencies)),
-							"eslint",
 							NpmResourceHelper.readUtf8StringFromClasspath(EslintFormatterStep.class,
 									"/com/diffplug/spotless/npm/common-serve.js",
 									"/com/diffplug/spotless/npm/eslint-serve.js"),
 							npmPathResolver.resolveNpmrcContent()),
 					new NpmFormatterStepLocations(
 							projectDir,
 							buildDir,
+							cacheDir,
 							npmPathResolver::resolveNpmExecutable,
 							npmPathResolver::resolveNodeExecutable));
 			this.origEslintConfig = requireNonNull(eslintConfig.verify());
@@ -116,7 +115,7 @@ protected void prepareNodeServerLayout() throws IOException {
 				// If any config files are provided, we need to make sure they are at the same location as the node modules
 				// as eslint will try to resolve plugin/config names relatively to the config file location and some
 				// eslint configs contain relative paths to additional config files (such as tsconfig.json e.g.)
-				logger.info("Copying config file <{}> to <{}> and using the copy", origEslintConfig.getEslintConfigPath(), nodeServerLayout.nodeModulesDir());
+				logger.debug("Copying config file <{}> to <{}> and using the copy", origEslintConfig.getEslintConfigPath(), nodeServerLayout.nodeModulesDir());
 				File configFileCopy = NpmResourceHelper.copyFileToDir(origEslintConfig.getEslintConfigPath(), nodeServerLayout.nodeModulesDir());
 				this.eslintConfigInUse = this.origEslintConfig.withEslintConfigPath(configFileCopy).verify();
 			}
@@ -162,8 +161,6 @@ public EslintFilePathPassingFormatterFunc(File projectDir, File nodeModulesDir,
 
 		@Override
 		public String applyWithFile(String unix, File file) throws Exception {
-			logger.info("formatting String '{}[...]' in file '{}'", lazy(() -> unix.substring(0, Math.min(50, unix.length()))), file);
-
 			Map<FormatOption, Object> eslintCallOptions = new HashMap<>();
 			setConfigToCallOptions(eslintCallOptions);
 			setFilePathToCallOptions(eslintCallOptions, file);

lib/src/main/java/com/diffplug/spotless/npm/NodeApp.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2023 DiffPlug
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.diffplug.spotless.npm;
+
+import java.util.Objects;
+
+import javax.annotation.Nonnull;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NodeApp {
+
+	private static final Logger logger = LoggerFactory.getLogger(NodeApp.class);
+
+	private static final TimedLogger timedLogger = TimedLogger.forLogger(logger);
+
+	@Nonnull
+	protected final NodeServerLayout nodeServerLayout;
+
+	@Nonnull
+	protected final NpmConfig npmConfig;
+
+	@Nonnull
+	protected final NpmProcessFactory npmProcessFactory;
+
+	@Nonnull
+	protected final NpmFormatterStepLocations formatterStepLocations;
+
+	public NodeApp(@Nonnull NodeServerLayout nodeServerLayout, @Nonnull NpmConfig npmConfig, @Nonnull NpmFormatterStepLocations formatterStepLocations) {
+		this.nodeServerLayout = Objects.requireNonNull(nodeServerLayout);
+		this.npmConfig = Objects.requireNonNull(npmConfig);
+		this.npmProcessFactory = processFactory(formatterStepLocations);
+		this.formatterStepLocations = Objects.requireNonNull(formatterStepLocations);
+	}
+
+	private static NpmProcessFactory processFactory(NpmFormatterStepLocations formatterStepLocations) {
+		if (formatterStepLocations.cacheDir() != null) {
+			logger.info("Caching npm install results in {}.", formatterStepLocations.cacheDir());
+			return NodeModulesCachingNpmProcessFactory.create(formatterStepLocations.cacheDir());
+		}
+		logger.debug("Not caching npm install results.");
+		return StandardNpmProcessFactory.INSTANCE;
+	}
+
+	boolean needsNpmInstall() {
+		return !this.nodeServerLayout.isNodeModulesPrepared();
+	}
+
+	boolean needsPrepareNodeAppLayout() {
+		return !this.nodeServerLayout.isLayoutPrepared();
+	}
+
+	void prepareNodeAppLayout() {
+		timedLogger.withInfo("Preparing {} for npm step {}.", this.nodeServerLayout, getClass().getName()).run(() -> {
+			NpmResourceHelper.assertDirectoryExists(nodeServerLayout.nodeModulesDir());
+			NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.packageJsonFile(), this.npmConfig.getPackageJsonContent());
+			if (this.npmConfig.getServeScriptContent() != null) {
+				NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.serveJsFile(), this.npmConfig.getServeScriptContent());
+			} else {
+				NpmResourceHelper.deleteFileIfExists(nodeServerLayout.serveJsFile());
+			}
+			if (this.npmConfig.getNpmrcContent() != null) {
+				NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.npmrcFile(), this.npmConfig.getNpmrcContent());
+			} else {
+				NpmResourceHelper.deleteFileIfExists(nodeServerLayout.npmrcFile());
+			}
+		});
+	}
+
+	void npmInstall() {
+		timedLogger.withInfo("Installing npm dependencies for {} with {}.", this.nodeServerLayout, this.npmProcessFactory.describe())
+				.run(() -> npmProcessFactory.createNpmInstallProcess(nodeServerLayout, formatterStepLocations).waitFor());
+	}
+}

lib/src/main/java/com/diffplug/spotless/npm/NodeModulesCachingNpmProcessFactory.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2023 DiffPlug
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.diffplug.spotless.npm;
+
+import java.io.File;
+import java.util.List;
+import java.util.Objects;
+
+import javax.annotation.Nonnull;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.diffplug.spotless.ProcessRunner.Result;
+
+public class NodeModulesCachingNpmProcessFactory implements NpmProcessFactory {
+
+	private static final Logger logger = LoggerFactory.getLogger(NodeModulesCachingNpmProcessFactory.class);
+
+	private static final TimedLogger timedLogger = TimedLogger.forLogger(logger);
+
+	private final File cacheDir;
+
+	private final ShadowCopy shadowCopy;
+
+	private NodeModulesCachingNpmProcessFactory(@Nonnull File cacheDir) {
+		this.cacheDir = Objects.requireNonNull(cacheDir);
+		assertDir(cacheDir);
+		this.shadowCopy = new ShadowCopy(cacheDir);
+	}
+
+	private void assertDir(File cacheDir) {
+		if (cacheDir.exists() && !cacheDir.isDirectory()) {
+			throw new IllegalArgumentException("Cache dir must be a directory");
+		}
+		if (!cacheDir.exists()) {
+			if (!cacheDir.mkdirs()) {
+				throw new IllegalArgumentException("Cache dir could not be created.");
+			}
+		}
+	}
+
+	public static NodeModulesCachingNpmProcessFactory create(@Nonnull File cacheDir) {
+		return new NodeModulesCachingNpmProcessFactory(cacheDir);
+	}
+
+	@Override
+	public NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
+		NpmProcess actualNpmInstallProcess = StandardNpmProcessFactory.INSTANCE.createNpmInstallProcess(nodeServerLayout, formatterStepLocations);
+		return new CachingNmpInstall(actualNpmInstallProcess, nodeServerLayout);
+	}
+
+	@Override
+	public NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
+		return StandardNpmProcessFactory.INSTANCE.createNpmServeProcess(nodeServerLayout, formatterStepLocations);
+	}
+
+	private class CachingNmpInstall implements NpmProcess {
+
+		private final NpmProcess actualNpmInstallProcess;
+		private final NodeServerLayout nodeServerLayout;
+
+		public CachingNmpInstall(NpmProcess actualNpmInstallProcess, NodeServerLayout nodeServerLayout) {
+			this.actualNpmInstallProcess = actualNpmInstallProcess;
+			this.nodeServerLayout = nodeServerLayout;
+		}
+
+		@Override
+		public Result waitFor() {
+			String entryName = entryName();
+			if (shadowCopy.entryExists(entryName, NodeServerLayout.NODE_MODULES)) {
+				timedLogger.withInfo("Using cached node_modules for {} from {}", entryName, cacheDir)
+						.run(() -> shadowCopy.copyEntryInto(entryName(), NodeServerLayout.NODE_MODULES, nodeServerLayout.nodeModulesDir()));
+				return new CachedResult();
+			} else {
+				Result result = timedLogger.withInfo("calling actual npm install {}", actualNpmInstallProcess.describe())
+						.call(actualNpmInstallProcess::waitFor);
+				assert result.exitCode() == 0;
+				storeShadowCopy(entryName);
+				return result;
+			}
+		}
+
+		private void storeShadowCopy(String entryName) {
+			timedLogger.withInfo("Caching node_modules for {} in {}", entryName, cacheDir)
+					.run(() -> shadowCopy.addEntry(entryName(), new File(nodeServerLayout.nodeModulesDir(), NodeServerLayout.NODE_MODULES)));
+		}
+
+		private String entryName() {
+			return nodeServerLayout.nodeModulesDir().getName();
+		}
+
+		@Override
+		public String describe() {
+			return String.format("Wrapper around [%s] to cache node_modules in [%s]", actualNpmInstallProcess.describe(), cacheDir.getAbsolutePath());
+		}
+	}
+
+	private class CachedResult extends Result {
+
+		public CachedResult() {
+			super(List.of("(from cache dir " + cacheDir + ")"), 0, new byte[0], new byte[0]);
+		}
+	}
+}

lib/src/main/java/com/diffplug/spotless/npm/NodeServeApp.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2023 DiffPlug
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.diffplug.spotless.npm;
+
+import javax.annotation.Nonnull;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.diffplug.spotless.ProcessRunner;
+
+public class NodeServeApp extends NodeApp {
+
+	private static final Logger logger = LoggerFactory.getLogger(NodeApp.class);
+
+	private static final TimedLogger timedLogger = TimedLogger.forLogger(logger);
+
+	public NodeServeApp(@Nonnull NodeServerLayout nodeServerLayout, @Nonnull NpmConfig npmConfig, @Nonnull NpmFormatterStepLocations formatterStepLocations) {
+		super(nodeServerLayout, npmConfig, formatterStepLocations);
+	}
+
+	ProcessRunner.LongRunningProcess startNpmServeProcess() {
+		return timedLogger.withInfo("Starting npm based server in {} with {}.", this.nodeServerLayout.nodeModulesDir(), this.npmProcessFactory.describe())
+				.call(() -> npmProcessFactory.createNpmServeProcess(nodeServerLayout, formatterStepLocations).start());
+	}
+
+}

lib/src/main/java/com/diffplug/spotless/npm/NodeServerLayout.java

@@ -27,6 +27,7 @@
 class NodeServerLayout {
 
 	private static final Pattern PACKAGE_JSON_NAME_PATTERN = Pattern.compile("\"name\"\\s*:\\s*\"([^\"]+)\"");
+	static final String NODE_MODULES = "node_modules";
 
 	private final File nodeModulesDir;
 	private final File packageJsonFile;
@@ -55,7 +56,6 @@ private static String nodeModulesDirName(String packageJsonContent) {
 	}
 
 	File nodeModulesDir() {
-
 		return nodeModulesDir;
 	}
 
@@ -89,7 +89,7 @@ public boolean isLayoutPrepared() {
 	}
 
 	public boolean isNodeModulesPrepared() {
-		Path nodeModulesInstallDirPath = new File(nodeModulesDir(), "node_modules").toPath();
+		Path nodeModulesInstallDirPath = new File(nodeModulesDir(), NODE_MODULES).toPath();
 		if (!Files.isDirectory(nodeModulesInstallDirPath)) {
 			return false;
 		}