Skip to content

Commit

Permalink
speedup npm install step for npm based formatters (#1590 fixes #1480
Browse files Browse the repository at this point in the history
…and #1582)
  • Loading branch information
nedtwigg authored Feb 27, 2023
2 parents 6716228 + 8ebb896 commit 39c84f2
Show file tree
Hide file tree
Showing 40 changed files with 1,966 additions and 172 deletions.
2 changes: 2 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,12 @@ We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format (
### Added
* `gradlew equoIde` opens a repeatable clean Spotless dev environment. ([#1523](https://github.com/diffplug/spotless/pull/1523))
* `cleanthat` added `includeDraft` option, to include draft mutators from composite mutators. ([#1574](https://github.com/diffplug/spotless/pull/1574))
* `npm`-based formatters now support caching of `node_modules` directory ([#1590](https://github.com/diffplug/spotless/pull/1590))
### Fixed
* `JacksonJsonFormatterFunc` handles json files with an Array as root. ([#1585](https://github.com/diffplug/spotless/pull/1585))
### Changes
* Bump default `cleanthat` version to latest `2.1` -> `2.6` ([#1569](https://github.com/diffplug/spotless/pull/1569) and [#1574](https://github.com/diffplug/spotless/pull/1574))
* Reduce logging-noise created by `npm`-based formatters ([#1590](https://github.com/diffplug/spotless/pull/1590) fixes [#1582](https://github.com/diffplug/spotless/issues/1582))

## [2.35.0] - 2023-02-10
### Added
Expand Down
2 changes: 2 additions & 0 deletions lib/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ tasks.named("check").configure {

dependencies {
compileOnly 'org.slf4j:slf4j-api:2.0.0'
testCommonImplementation 'org.slf4j:slf4j-api:2.0.0'

// zero runtime reqs is a hard requirements for spotless-lib
// if you need a dep, put it in lib-extra
testCommonImplementation "org.junit.jupiter:junit-jupiter:$VER_JUNIT"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
*/
package com.diffplug.spotless.npm;

import static com.diffplug.spotless.LazyArgLogger.lazy;
import static java.util.Objects.requireNonNull;

import java.io.File;
Expand Down Expand Up @@ -71,13 +70,13 @@ public static Map<String, String> defaultDevDependenciesWithEslint(String versio
return Collections.singletonMap("eslint", version);
}

public static FormatterStep create(Map<String, String> devDependencies, Provisioner provisioner, File projectDir, File buildDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) {
public static FormatterStep create(Map<String, String> devDependencies, Provisioner provisioner, File projectDir, File buildDir, File cacheDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) {
requireNonNull(devDependencies);
requireNonNull(provisioner);
requireNonNull(projectDir);
requireNonNull(buildDir);
return FormatterStep.createLazy(NAME,
() -> new State(NAME, devDependencies, projectDir, buildDir, npmPathResolver, eslintConfig),
() -> new State(NAME, devDependencies, projectDir, buildDir, cacheDir, npmPathResolver, eslintConfig),
State::createFormatterFunc);
}

Expand All @@ -89,20 +88,20 @@ private static class State extends NpmFormatterStepStateBase implements Serializ
@SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED")
private transient EslintConfig eslintConfigInUse;

State(String stepName, Map<String, String> devDependencies, File projectDir, File buildDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) throws IOException {
State(String stepName, Map<String, String> devDependencies, File projectDir, File buildDir, File cacheDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) throws IOException {
super(stepName,
new NpmConfig(
replaceDevDependencies(
NpmResourceHelper.readUtf8StringFromClasspath(EslintFormatterStep.class, "/com/diffplug/spotless/npm/eslint-package.json"),
new TreeMap<>(devDependencies)),
"eslint",
NpmResourceHelper.readUtf8StringFromClasspath(EslintFormatterStep.class,
"/com/diffplug/spotless/npm/common-serve.js",
"/com/diffplug/spotless/npm/eslint-serve.js"),
npmPathResolver.resolveNpmrcContent()),
new NpmFormatterStepLocations(
projectDir,
buildDir,
cacheDir,
npmPathResolver::resolveNpmExecutable,
npmPathResolver::resolveNodeExecutable));
this.origEslintConfig = requireNonNull(eslintConfig.verify());
Expand All @@ -116,7 +115,7 @@ protected void prepareNodeServerLayout() throws IOException {
// If any config files are provided, we need to make sure they are at the same location as the node modules
// as eslint will try to resolve plugin/config names relatively to the config file location and some
// eslint configs contain relative paths to additional config files (such as tsconfig.json e.g.)
logger.info("Copying config file <{}> to <{}> and using the copy", origEslintConfig.getEslintConfigPath(), nodeServerLayout.nodeModulesDir());
logger.debug("Copying config file <{}> to <{}> and using the copy", origEslintConfig.getEslintConfigPath(), nodeServerLayout.nodeModulesDir());
File configFileCopy = NpmResourceHelper.copyFileToDir(origEslintConfig.getEslintConfigPath(), nodeServerLayout.nodeModulesDir());
this.eslintConfigInUse = this.origEslintConfig.withEslintConfigPath(configFileCopy).verify();
}
Expand Down Expand Up @@ -162,8 +161,6 @@ public EslintFilePathPassingFormatterFunc(File projectDir, File nodeModulesDir,

@Override
public String applyWithFile(String unix, File file) throws Exception {
logger.info("formatting String '{}[...]' in file '{}'", lazy(() -> unix.substring(0, Math.min(50, unix.length()))), file);

Map<FormatOption, Object> eslintCallOptions = new HashMap<>();
setConfigToCallOptions(eslintCallOptions);
setFilePathToCallOptions(eslintCallOptions, file);
Expand Down
88 changes: 88 additions & 0 deletions lib/src/main/java/com/diffplug/spotless/npm/NodeApp.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
* Copyright 2023 DiffPlug
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.diffplug.spotless.npm;

import java.util.Objects;

import javax.annotation.Nonnull;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class NodeApp {

private static final Logger logger = LoggerFactory.getLogger(NodeApp.class);

private static final TimedLogger timedLogger = TimedLogger.forLogger(logger);

@Nonnull
protected final NodeServerLayout nodeServerLayout;

@Nonnull
protected final NpmConfig npmConfig;

@Nonnull
protected final NpmProcessFactory npmProcessFactory;

@Nonnull
protected final NpmFormatterStepLocations formatterStepLocations;

public NodeApp(@Nonnull NodeServerLayout nodeServerLayout, @Nonnull NpmConfig npmConfig, @Nonnull NpmFormatterStepLocations formatterStepLocations) {
this.nodeServerLayout = Objects.requireNonNull(nodeServerLayout);
this.npmConfig = Objects.requireNonNull(npmConfig);
this.npmProcessFactory = processFactory(formatterStepLocations);
this.formatterStepLocations = Objects.requireNonNull(formatterStepLocations);
}

private static NpmProcessFactory processFactory(NpmFormatterStepLocations formatterStepLocations) {
if (formatterStepLocations.cacheDir() != null) {
logger.info("Caching npm install results in {}.", formatterStepLocations.cacheDir());
return NodeModulesCachingNpmProcessFactory.create(formatterStepLocations.cacheDir());
}
logger.debug("Not caching npm install results.");
return StandardNpmProcessFactory.INSTANCE;
}

boolean needsNpmInstall() {
return !this.nodeServerLayout.isNodeModulesPrepared();
}

boolean needsPrepareNodeAppLayout() {
return !this.nodeServerLayout.isLayoutPrepared();
}

void prepareNodeAppLayout() {
timedLogger.withInfo("Preparing {} for npm step {}.", this.nodeServerLayout, getClass().getName()).run(() -> {
NpmResourceHelper.assertDirectoryExists(nodeServerLayout.nodeModulesDir());
NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.packageJsonFile(), this.npmConfig.getPackageJsonContent());
if (this.npmConfig.getServeScriptContent() != null) {
NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.serveJsFile(), this.npmConfig.getServeScriptContent());
} else {
NpmResourceHelper.deleteFileIfExists(nodeServerLayout.serveJsFile());
}
if (this.npmConfig.getNpmrcContent() != null) {
NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.npmrcFile(), this.npmConfig.getNpmrcContent());
} else {
NpmResourceHelper.deleteFileIfExists(nodeServerLayout.npmrcFile());
}
});
}

void npmInstall() {
timedLogger.withInfo("Installing npm dependencies for {} with {}.", this.nodeServerLayout, this.npmProcessFactory.describe())
.run(() -> npmProcessFactory.createNpmInstallProcess(nodeServerLayout, formatterStepLocations).waitFor());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/*
* Copyright 2023 DiffPlug
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.diffplug.spotless.npm;

import java.io.File;
import java.util.List;
import java.util.Objects;

import javax.annotation.Nonnull;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.diffplug.spotless.ProcessRunner.Result;

public class NodeModulesCachingNpmProcessFactory implements NpmProcessFactory {

private static final Logger logger = LoggerFactory.getLogger(NodeModulesCachingNpmProcessFactory.class);

private static final TimedLogger timedLogger = TimedLogger.forLogger(logger);

private final File cacheDir;

private final ShadowCopy shadowCopy;

private NodeModulesCachingNpmProcessFactory(@Nonnull File cacheDir) {
this.cacheDir = Objects.requireNonNull(cacheDir);
assertDir(cacheDir);
this.shadowCopy = new ShadowCopy(cacheDir);
}

private void assertDir(File cacheDir) {
if (cacheDir.exists() && !cacheDir.isDirectory()) {
throw new IllegalArgumentException("Cache dir must be a directory");
}
if (!cacheDir.exists()) {
if (!cacheDir.mkdirs()) {
throw new IllegalArgumentException("Cache dir could not be created.");
}
}
}

public static NodeModulesCachingNpmProcessFactory create(@Nonnull File cacheDir) {
return new NodeModulesCachingNpmProcessFactory(cacheDir);
}

@Override
public NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
NpmProcess actualNpmInstallProcess = StandardNpmProcessFactory.INSTANCE.createNpmInstallProcess(nodeServerLayout, formatterStepLocations);
return new CachingNmpInstall(actualNpmInstallProcess, nodeServerLayout);
}

@Override
public NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
return StandardNpmProcessFactory.INSTANCE.createNpmServeProcess(nodeServerLayout, formatterStepLocations);
}

private class CachingNmpInstall implements NpmProcess {

private final NpmProcess actualNpmInstallProcess;
private final NodeServerLayout nodeServerLayout;

public CachingNmpInstall(NpmProcess actualNpmInstallProcess, NodeServerLayout nodeServerLayout) {
this.actualNpmInstallProcess = actualNpmInstallProcess;
this.nodeServerLayout = nodeServerLayout;
}

@Override
public Result waitFor() {
String entryName = entryName();
if (shadowCopy.entryExists(entryName, NodeServerLayout.NODE_MODULES)) {
timedLogger.withInfo("Using cached node_modules for {} from {}", entryName, cacheDir)
.run(() -> shadowCopy.copyEntryInto(entryName(), NodeServerLayout.NODE_MODULES, nodeServerLayout.nodeModulesDir()));
return new CachedResult();
} else {
Result result = timedLogger.withInfo("calling actual npm install {}", actualNpmInstallProcess.describe())
.call(actualNpmInstallProcess::waitFor);
assert result.exitCode() == 0;
storeShadowCopy(entryName);
return result;
}
}

private void storeShadowCopy(String entryName) {
timedLogger.withInfo("Caching node_modules for {} in {}", entryName, cacheDir)
.run(() -> shadowCopy.addEntry(entryName(), new File(nodeServerLayout.nodeModulesDir(), NodeServerLayout.NODE_MODULES)));
}

private String entryName() {
return nodeServerLayout.nodeModulesDir().getName();
}

@Override
public String describe() {
return String.format("Wrapper around [%s] to cache node_modules in [%s]", actualNpmInstallProcess.describe(), cacheDir.getAbsolutePath());
}
}

private class CachedResult extends Result {

public CachedResult() {
super(List.of("(from cache dir " + cacheDir + ")"), 0, new byte[0], new byte[0]);
}
}
}
40 changes: 40 additions & 0 deletions lib/src/main/java/com/diffplug/spotless/npm/NodeServeApp.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright 2023 DiffPlug
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.diffplug.spotless.npm;

import javax.annotation.Nonnull;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.diffplug.spotless.ProcessRunner;

public class NodeServeApp extends NodeApp {

private static final Logger logger = LoggerFactory.getLogger(NodeApp.class);

private static final TimedLogger timedLogger = TimedLogger.forLogger(logger);

public NodeServeApp(@Nonnull NodeServerLayout nodeServerLayout, @Nonnull NpmConfig npmConfig, @Nonnull NpmFormatterStepLocations formatterStepLocations) {
super(nodeServerLayout, npmConfig, formatterStepLocations);
}

ProcessRunner.LongRunningProcess startNpmServeProcess() {
return timedLogger.withInfo("Starting npm based server in {} with {}.", this.nodeServerLayout.nodeModulesDir(), this.npmProcessFactory.describe())
.call(() -> npmProcessFactory.createNpmServeProcess(nodeServerLayout, formatterStepLocations).start());
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
class NodeServerLayout {

private static final Pattern PACKAGE_JSON_NAME_PATTERN = Pattern.compile("\"name\"\\s*:\\s*\"([^\"]+)\"");
static final String NODE_MODULES = "node_modules";

private final File nodeModulesDir;
private final File packageJsonFile;
Expand Down Expand Up @@ -55,7 +56,6 @@ private static String nodeModulesDirName(String packageJsonContent) {
}

File nodeModulesDir() {

return nodeModulesDir;
}

Expand Down Expand Up @@ -89,7 +89,7 @@ public boolean isLayoutPrepared() {
}

public boolean isNodeModulesPrepared() {
Path nodeModulesInstallDirPath = new File(nodeModulesDir(), "node_modules").toPath();
Path nodeModulesInstallDirPath = new File(nodeModulesDir(), NODE_MODULES).toPath();
if (!Files.isDirectory(nodeModulesInstallDirPath)) {
return false;
}
Expand Down
Loading

0 comments on commit 39c84f2

Please sign in to comment.