diff --git a/command-runner/src/main/scala/stryker4s/command/Stryker4sCommandRunner.scala b/command-runner/src/main/scala/stryker4s/command/Stryker4sCommandRunner.scala index 8e7a31ccc..0b18f247f 100644 --- a/command-runner/src/main/scala/stryker4s/command/Stryker4sCommandRunner.scala +++ b/command-runner/src/main/scala/stryker4s/command/Stryker4sCommandRunner.scala @@ -7,6 +7,7 @@ import stryker4s.command.config.ProcessRunnerConfig import stryker4s.command.runner.ProcessTestRunner import stryker4s.config.Config import stryker4s.log.Logger +import stryker4s.model.CompilerErrMsg import stryker4s.mutants.applymutants.ActiveMutationContext import stryker4s.mutants.applymutants.ActiveMutationContext.ActiveMutationContext import stryker4s.run.process.ProcessRunner @@ -21,12 +22,12 @@ class Stryker4sCommandRunner(processRunnerConfig: ProcessRunnerConfig, timeout: override def resolveTestRunners( tmpDir: Path - )(implicit config: Config): NonEmptyList[Resource[IO, TestRunner]] = { + )(implicit config: Config): Either[NonEmptyList[CompilerErrMsg], NonEmptyList[Resource[IO, TestRunner]]] = { val innerTestRunner = Resource.pure[IO, TestRunner](new ProcessTestRunner(processRunnerConfig.testRunner, ProcessRunner(), tmpDir)) val withTimeout = TestRunner.timeoutRunner(timeout, innerTestRunner) - NonEmptyList.of(withTimeout) + Right(NonEmptyList.of(withTimeout)) } } diff --git a/command-runner/src/main/scala/stryker4s/command/runner/ProcessTestRunner.scala b/command-runner/src/main/scala/stryker4s/command/runner/ProcessTestRunner.scala index 81a5a82aa..310046564 100644 --- a/command-runner/src/main/scala/stryker4s/command/runner/ProcessTestRunner.scala +++ b/command-runner/src/main/scala/stryker4s/command/runner/ProcessTestRunner.scala @@ -20,7 +20,7 @@ class ProcessTestRunner(command: Command, processRunner: ProcessRunner, tmpDir: } def runMutant(mutant: Mutant): IO[MutantRunResult] = { - val id = mutant.id + val id = mutant.id.globalId processRunner(command, tmpDir, ("ACTIVE_MUTATION", id.toString)).map { case Success(0) => Survived(mutant) case Success(_) => Killed(mutant) diff --git a/command-runner/src/test/scala/stryker4s/run/ProcessTestRunnerTest.scala b/command-runner/src/test/scala/stryker4s/run/ProcessTestRunnerTest.scala index 1d1e6643c..1187e0d6a 100644 --- a/command-runner/src/test/scala/stryker4s/run/ProcessTestRunnerTest.scala +++ b/command-runner/src/test/scala/stryker4s/run/ProcessTestRunnerTest.scala @@ -24,7 +24,7 @@ class ProcessTestRunnerTest extends Stryker4sIOSuite with MockitoIOSuite with Lo it("should return a Survived mutant on an exitcode 0 process") { val testProcessRunner = TestProcessRunner(Success(0)) - val mutant = Mutant(0, q"4", q"5", EmptyString) + val mutant = Mutant(MutantId(0), q"4", q"5", EmptyString) processTestRunner(testProcessRunner).runMutant(mutant).asserting { result => result shouldBe a[Survived] testProcessRunner.timesCalled.next() should equal(1) @@ -33,7 +33,7 @@ class ProcessTestRunnerTest extends Stryker4sIOSuite with MockitoIOSuite with Lo it("should return a Killed mutant on an exitcode 1 process") { val testProcessRunner = TestProcessRunner(Success(1)) - val mutant = Mutant(0, q"4", q"5", EmptyString) + val mutant = Mutant(MutantId(0), q"4", q"5", EmptyString) processTestRunner(testProcessRunner).runMutant(mutant).asserting { result => result shouldBe a[Killed] testProcessRunner.timesCalled.next() should equal(1) @@ -43,7 +43,7 @@ class ProcessTestRunnerTest extends Stryker4sIOSuite with MockitoIOSuite with Lo it("should return a TimedOut mutant on a TimedOut process") { val exception = new TimeoutException("Test") val testProcessRunner = TestProcessRunner(Failure(exception)) - val mutant = Mutant(0, q"4", q"5", EmptyString) + val mutant = Mutant(MutantId(0), q"4", q"5", EmptyString) processTestRunner(testProcessRunner).runMutant(mutant).asserting { result => result shouldBe a[TimedOut] testProcessRunner.timesCalled.next() should equal(1) diff --git a/core/src/main/scala/stryker4s/Stryker4s.scala b/core/src/main/scala/stryker4s/Stryker4s.scala index c01d4668f..83da915d7 100644 --- a/core/src/main/scala/stryker4s/Stryker4s.scala +++ b/core/src/main/scala/stryker4s/Stryker4s.scala @@ -7,14 +7,15 @@ import stryker4s.mutants.Mutator import stryker4s.run.MutantRunner import stryker4s.run.threshold.{ScoreStatus, ThresholdChecker} -class Stryker4s(fileSource: MutatesFileResolver, mutator: Mutator, runner: MutantRunner)(implicit config: Config) { +class Stryker4s(fileSource: MutatesFileResolver, mutator: Mutator, runner: MutantRunner)(implicit + config: Config +) { def run(): IO[ScoreStatus] = { val filesToMutate = fileSource.files for { - mutatedFiles <- mutator.mutate(filesToMutate) - metrics <- runner(mutatedFiles) + metrics <- runner(errors => mutator.mutate(filesToMutate, errors)) scoreStatus = ThresholdChecker.determineScoreStatus(metrics.mutationScore) } yield scoreStatus } diff --git a/core/src/main/scala/stryker4s/extension/exception/stryker4sException.scala b/core/src/main/scala/stryker4s/extension/exception/stryker4sException.scala index c89ec60ed..d9a5273e6 100644 --- a/core/src/main/scala/stryker4s/extension/exception/stryker4sException.scala +++ b/core/src/main/scala/stryker4s/extension/exception/stryker4sException.scala @@ -1,5 +1,7 @@ package stryker4s.extension.exception +import stryker4s.model.CompilerErrMsg + import scala.util.control.NoStackTrace sealed abstract class Stryker4sException(message: String) extends Exception(message) @@ -14,3 +16,11 @@ final case class TestSetupException(name: String) ) final case class MutationRunFailedException(message: String) extends Stryker4sException(message) + +final case class UnableToFixCompilerErrorsException(errs: List[CompilerErrMsg]) + extends Stryker4sException( + "Unable to remove non-compiling mutants in the mutated files. As a work-around you can exclude them in the stryker.conf. Please report this issue at https://github.com/stryker-mutator/stryker4s/issues\n" + + errs + .map(err => s"${err.path}: '${err.msg}'") + .mkString("\n") + ) diff --git a/core/src/main/scala/stryker4s/model/CompilerErrMsg.scala b/core/src/main/scala/stryker4s/model/CompilerErrMsg.scala new file mode 100644 index 000000000..45f1984e9 --- /dev/null +++ b/core/src/main/scala/stryker4s/model/CompilerErrMsg.scala @@ -0,0 +1,7 @@ +package stryker4s.model + +//This class is used to contain information about mutants that did not compile +//It essentially exists so that we don't have to pass around the SBT specific compiler exception +case class CompilerErrMsg(msg: String, path: String, line: Integer) { + override def toString: String = s"$path:L$line: '$msg'" +} diff --git a/core/src/main/scala/stryker4s/model/Mutant.scala b/core/src/main/scala/stryker4s/model/Mutant.scala index d8e3ed326..217249142 100644 --- a/core/src/main/scala/stryker4s/model/Mutant.scala +++ b/core/src/main/scala/stryker4s/model/Mutant.scala @@ -4,4 +4,8 @@ import scala.meta.{Term, Tree} import stryker4s.extension.mutationtype.Mutation -final case class Mutant(id: Int, original: Term, mutated: Term, mutationType: Mutation[_ <: Tree]) +case class MutantId(globalId: Int) extends AnyVal { + override def toString: String = globalId.toString +} + +final case class Mutant(id: MutantId, original: Term, mutated: Term, mutationType: Mutation[_ <: Tree]) diff --git a/core/src/main/scala/stryker4s/model/MutantRunResult.scala b/core/src/main/scala/stryker4s/model/MutantRunResult.scala index 1f40f5e52..e3f824f40 100644 --- a/core/src/main/scala/stryker4s/model/MutantRunResult.scala +++ b/core/src/main/scala/stryker4s/model/MutantRunResult.scala @@ -22,3 +22,5 @@ final case class NoCoverage(mutant: Mutant, description: Option[String] = None) final case class Error(mutant: Mutant, description: Option[String] = None) extends MutantRunResult final case class Ignored(mutant: Mutant, description: Option[String] = None) extends MutantRunResult + +final case class CompileError(mutant: Mutant, description: Option[String] = None) extends MutantRunResult diff --git a/core/src/main/scala/stryker4s/model/MutatedFile.scala b/core/src/main/scala/stryker4s/model/MutatedFile.scala index 0634b07b7..6a5a98f64 100644 --- a/core/src/main/scala/stryker4s/model/MutatedFile.scala +++ b/core/src/main/scala/stryker4s/model/MutatedFile.scala @@ -2,6 +2,17 @@ package stryker4s.model import fs2.io.file.Path -import scala.meta.Tree +import scala.meta._ -final case class MutatedFile(fileOrigin: Path, tree: Tree, mutants: Seq[Mutant], excludedMutants: Int) +final case class MutatedFile( + fileOrigin: Path, + tree: Tree, + mutants: Seq[Mutant], + nonCompilingMutants: Seq[Mutant], + excludedMutants: Int +) { + + def mutatedSource: String = { + tree.syntax + } +} diff --git a/core/src/main/scala/stryker4s/model/MutationsInSource.scala b/core/src/main/scala/stryker4s/model/MutationsInSource.scala index a93368aa8..41812d833 100644 --- a/core/src/main/scala/stryker4s/model/MutationsInSource.scala +++ b/core/src/main/scala/stryker4s/model/MutationsInSource.scala @@ -2,4 +2,4 @@ package stryker4s.model import scala.meta.Source -final case class MutationsInSource(source: Source, mutants: Seq[Mutant], excluded: Int) +final case class MutationsInSource(source: Source, mutants: Seq[Mutant], excluded: Int, fileName: String) diff --git a/core/src/main/scala/stryker4s/mutants/Mutator.scala b/core/src/main/scala/stryker4s/mutants/Mutator.scala index e41ff9114..317f24dc7 100644 --- a/core/src/main/scala/stryker4s/mutants/Mutator.scala +++ b/core/src/main/scala/stryker4s/mutants/Mutator.scala @@ -7,25 +7,29 @@ import fs2.io.file.Path import stryker4s.config.Config import stryker4s.extension.StreamExtensions._ import stryker4s.log.Logger -import stryker4s.model.{MutatedFile, MutationsInSource, SourceTransformations} +import stryker4s.model._ import stryker4s.mutants.applymutants.{MatchBuilder, StatementTransformer} import stryker4s.mutants.findmutants.MutantFinder -import scala.meta.Tree +import scala.meta._ -class Mutator(mutantFinder: MutantFinder, transformer: StatementTransformer, matchBuilder: MatchBuilder)(implicit +class Mutator( + mutantFinder: MutantFinder, + transformer: StatementTransformer, + matchBuilder: MatchBuilder +)(implicit config: Config, log: Logger ) { - def mutate(files: Stream[IO, Path]): IO[Seq[MutatedFile]] = { + def mutate(files: Stream[IO, Path], compileErrors: Seq[CompilerErrMsg] = Seq.empty): IO[Seq[MutatedFile]] = { + if (compileErrors.nonEmpty) { + log.debug("Trying to remove mutants that gave these errors:\n\t" + compileErrors.mkString("\n\t")) + } files .parEvalMapUnordered(config.concurrency)(p => findMutants(p).tupleLeft(p)) .map { case (file, mutationsInSource) => - val transformed = transformStatements(mutationsInSource) - val builtTree = buildMatches(transformed) - - MutatedFile(file, builtTree, mutationsInSource.mutants, mutationsInSource.excluded) + mutateFile(file, mutationsInSource, compileErrors) } .filterNot(mutatedFile => mutatedFile.mutants.isEmpty && mutatedFile.excludedMutants == 0) .compile @@ -33,6 +37,62 @@ class Mutator(mutantFinder: MutantFinder, transformer: StatementTransformer, mat .flatTap(logMutationResult) } + private def mutateFile( + file: Path, + mutationsInSource: MutationsInSource, + compileErrors: Seq[CompilerErrMsg] + ): MutatedFile = { + val transformed = transformStatements(mutationsInSource) + val builtTree = buildMatches(transformed) + + val mutatedFile = MutatedFile( + fileOrigin = file, + tree = builtTree, + mutants = mutationsInSource.mutants, + nonCompilingMutants = Seq.empty, + excludedMutants = mutationsInSource.excluded + ) + + if (compileErrors.isEmpty) + mutatedFile + else { + //If there are any compiler errors (i.e. we're currently retrying the mutation with the bad ones rolled back) + //Then we take the original tree built that didn't compile + //And then we search inside it to translate the compile errors to mutants + //Finally we rebuild it from scratch without those mutants + //This is not very performant, but you only pay the cost if there actually is a compiler error + val errorsInThisFile = compileErrors.filter(err => file.endsWith(err.path)) + if (errorsInThisFile.isEmpty) { + log.debug(s"No compiler errors in $file") + mutatedFile + } else { + log.debug(s"Found ${errorsInThisFile.mkString(" ")} in $file") + + val nonCompilingIds = errorsToIds( + errorsInThisFile, + mutatedFile.mutatedSource, + transformed.transformedStatements.flatMap(_.mutantStatements) + ) + log.debug(s"Removed mutant id[s] ${nonCompilingIds.mkString(";")} in $file") + + val (nonCompilingMutants, compilingMutants) = + mutationsInSource.mutants.partition(mut => nonCompilingIds.contains(mut.id)) + + val mutationsInSourceWithoutErrors = mutationsInSource.copy(mutants = compilingMutants) + val transformedWithoutErrors = transformStatements(mutationsInSourceWithoutErrors) + val builtTreeWithoutErrors = buildMatches(transformedWithoutErrors) + + MutatedFile( + fileOrigin = file, + tree = builtTreeWithoutErrors, + mutants = compilingMutants, + nonCompilingMutants = nonCompilingMutants, + excludedMutants = mutationsInSource.excluded + ) + } + } + } + /** Step 1: Find mutants in the found files */ private def findMutants(file: Path): IO[MutationsInSource] = mutantFinder.mutantsInFile(file) @@ -44,7 +104,7 @@ class Mutator(mutantFinder: MutantFinder, transformer: StatementTransformer, mat /** Step 3: Build pattern matches from transformed trees */ - private def buildMatches(transformedMutantsInSource: SourceTransformations): Tree = + private def buildMatches(transformedMutantsInSource: SourceTransformations) = matchBuilder.buildNewSource(transformedMutantsInSource) private def logMutationResult(mutatedFiles: Iterable[MutatedFile]): IO[Unit] = { @@ -70,4 +130,32 @@ class Mutator(mutantFinder: MutantFinder, transformer: StatementTransformer, mat } else IO.unit } } + + //Given compiler errors, return the mutants that caused it by searching for the matching case statement at that line + private def errorsToIds( + compileErrors: Seq[CompilerErrMsg], + mutatedFile: String, + mutants: Seq[Mutant] + ): Seq[MutantId] = { + val statementToMutIdMap = mutants.map { mutant => + matchBuilder.mutantToCase(mutant).structure -> mutant.id + }.toMap + + val lineToMutantId: Map[Int, MutantId] = mutatedFile + //Parsing the mutated tree again as a string is the only way to get the position info of the mutated statements + .parse[Source] + .getOrElse(throw new RuntimeException(s"Failed to parse $mutatedFile to remove non-compiling mutants")) + .collect { + case node: Case if statementToMutIdMap.contains(node.structure) => + val mutId = statementToMutIdMap(node.structure) + //+1 because scalameta uses zero-indexed line numbers + (node.pos.startLine to node.pos.endLine).map(i => i + 1 -> mutId) + } + .flatten + .toMap + + compileErrors.flatMap { err => + lineToMutantId.get(err.line) + } + } } diff --git a/core/src/main/scala/stryker4s/mutants/applymutants/CoverageMatchBuilder.scala b/core/src/main/scala/stryker4s/mutants/applymutants/CoverageMatchBuilder.scala index d47779bbd..ffc29c763 100644 --- a/core/src/main/scala/stryker4s/mutants/applymutants/CoverageMatchBuilder.scala +++ b/core/src/main/scala/stryker4s/mutants/applymutants/CoverageMatchBuilder.scala @@ -11,7 +11,7 @@ class CoverageMatchBuilder(mutationContext: ActiveMutationContext)(implicit log: // sbt-stryker4s-testrunner matches on Int instead of Option[Int] override def mutantToCase(mutant: Mutant): Case = - super.buildCase(mutant.mutated, p"${mutant.id}") + super.buildCase(mutant.mutated, p"${mutant.id.globalId}") override def defaultCase(transformedMutant: TransformedMutants): Case = withCoverage(super.defaultCase(transformedMutant), transformedMutant.mutantStatements) @@ -20,7 +20,7 @@ class CoverageMatchBuilder(mutationContext: ActiveMutationContext)(implicit log: * switch. `coverMutant` always returns true */ private def withCoverage(caze: Case, mutants: List[Mutant]): Case = { - val coverageCond = q"_root_.stryker4s.coverage.coverMutant(..${mutants.map(_.id).map(Lit.Int(_))})" + val coverageCond = q"_root_.stryker4s.coverage.coverMutant(..${mutants.map(_.id.globalId).map(Lit.Int(_))})" caze.copy(cond = Some(coverageCond)) } } diff --git a/core/src/main/scala/stryker4s/mutants/applymutants/MatchBuilder.scala b/core/src/main/scala/stryker4s/mutants/applymutants/MatchBuilder.scala index db7b25ba3..d0811a982 100644 --- a/core/src/main/scala/stryker4s/mutants/applymutants/MatchBuilder.scala +++ b/core/src/main/scala/stryker4s/mutants/applymutants/MatchBuilder.scala @@ -56,8 +56,8 @@ class MatchBuilder(mutationContext: ActiveMutationContext)(implicit log: Logger) q"($mutationContext match { ..case $cases })" } - protected def mutantToCase(mutant: Mutant): Case = - buildCase(mutant.mutated, p"Some(${mutant.id})") + def mutantToCase(mutant: Mutant): Case = + buildCase(mutant.mutated, p"Some(${mutant.id.globalId})") protected def defaultCase(transformedMutant: TransformedMutants): Case = buildCase(transformedMutant.originalStatement, p"_") @@ -72,6 +72,8 @@ class MatchBuilder(mutationContext: ActiveMutationContext)(implicit log: Logger) } .map { case (originalStatement, mutants) => TransformedMutants(originalStatement, mutants.toList) } .toSeq - .sortBy(_.mutantStatements.head.id) // Should be sorted so tree transformations are applied in order of discovery + .sortBy( + _.mutantStatements.head.id.globalId + ) // Should be sorted so tree transformations are applied in order of discovery } } diff --git a/core/src/main/scala/stryker4s/mutants/applymutants/StatementTransformer.scala b/core/src/main/scala/stryker4s/mutants/applymutants/StatementTransformer.scala index 2a821cd4a..ba06e44fa 100644 --- a/core/src/main/scala/stryker4s/mutants/applymutants/StatementTransformer.scala +++ b/core/src/main/scala/stryker4s/mutants/applymutants/StatementTransformer.scala @@ -11,7 +11,7 @@ class StatementTransformer { .groupBy(mutant => mutant.original) .map { case (original, mutants) => transformMutant(original, mutants) } .toSeq - .sortBy(_.mutantStatements.map(_.id).max) + .sortBy(_.mutantStatements.map(_.id.globalId).max) SourceTransformations(source, transformedMutants) } diff --git a/core/src/main/scala/stryker4s/mutants/findmutants/MutantFinder.scala b/core/src/main/scala/stryker4s/mutants/findmutants/MutantFinder.scala index b421164ef..286720ab1 100644 --- a/core/src/main/scala/stryker4s/mutants/findmutants/MutantFinder.scala +++ b/core/src/main/scala/stryker4s/mutants/findmutants/MutantFinder.scala @@ -15,7 +15,7 @@ class MutantFinder(matcher: MutantMatcher)(implicit config: Config, log: Logger) def mutantsInFile(filePath: Path): IO[MutationsInSource] = for { parsedSource <- parseFile(filePath) (included, excluded) <- IO(findMutants(parsedSource)) - } yield MutationsInSource(parsedSource, included, excluded) + } yield MutationsInSource(parsedSource, included, excluded, filePath.toString) def findMutants(source: Source): (Seq[Mutant], Int) = { val (ignored, included) = source.collect(matcher.allMatchers).flatten.partitionEither(identity) diff --git a/core/src/main/scala/stryker4s/mutants/findmutants/MutantMatcher.scala b/core/src/main/scala/stryker4s/mutants/findmutants/MutantMatcher.scala index 27b87a6e6..a8b3729e2 100644 --- a/core/src/main/scala/stryker4s/mutants/findmutants/MutantMatcher.scala +++ b/core/src/main/scala/stryker4s/mutants/findmutants/MutantMatcher.scala @@ -6,7 +6,7 @@ import stryker4s.config.Config import stryker4s.extension.PartialFunctionOps._ import stryker4s.extension.TreeExtensions.{GetMods, PathToRoot, TreeIsInExtension} import stryker4s.extension.mutationtype._ -import stryker4s.model.{IgnoredMutationReason, Mutant, MutationExcluded} +import stryker4s.model.{IgnoredMutationReason, Mutant, MutantId, MutationExcluded} import scala.meta._ @@ -115,8 +115,16 @@ class MutantMatcher()(implicit config: Config) { .map { mutated => if (matchExcluded(mutated) || isSuppressedByAnnotation(mutated, original)) Left(MutationExcluded()) - else - Right(Mutant(ids.next(), original, mutationToTerm(mutated), mutated)) + else { + Right( + Mutant( + MutantId(globalId = ids.next()), + original, + mutationToTerm(mutated), + mutated + ) + ) + } } } diff --git a/core/src/main/scala/stryker4s/report/mapper/MutantRunResultMapper.scala b/core/src/main/scala/stryker4s/report/mapper/MutantRunResultMapper.scala index 746eec441..3748b268c 100644 --- a/core/src/main/scala/stryker4s/report/mapper/MutantRunResultMapper.scala +++ b/core/src/main/scala/stryker4s/report/mapper/MutantRunResultMapper.scala @@ -39,7 +39,7 @@ trait MutantRunResultMapper { private def toMutantResult(runResult: MutantRunResult): MutantResult = { val mutant = runResult.mutant MutantResult( - mutant.id.toString, + mutant.id.globalId.toString, mutant.mutationType.mutationName, mutant.mutated.syntax, toLocation(mutant.original.pos), @@ -56,12 +56,13 @@ trait MutantRunResultMapper { private def toMutantStatus(mutant: MutantRunResult): MutantStatus = mutant match { - case _: Survived => MutantStatus.Survived - case _: Killed => MutantStatus.Killed - case _: NoCoverage => MutantStatus.NoCoverage - case _: TimedOut => MutantStatus.Timeout - case _: Error => MutantStatus.RuntimeError - case _: Ignored => MutantStatus.Ignored + case _: Survived => MutantStatus.Survived + case _: Killed => MutantStatus.Killed + case _: NoCoverage => MutantStatus.NoCoverage + case _: TimedOut => MutantStatus.Timeout + case _: Error => MutantStatus.RuntimeError + case _: Ignored => MutantStatus.Ignored + case _: CompileError => MutantStatus.CompileError } private def fileContentAsString(path: Path)(implicit config: Config): String = diff --git a/core/src/main/scala/stryker4s/run/MutantRunner.scala b/core/src/main/scala/stryker4s/run/MutantRunner.scala index a8b2e9adb..4a7342ba1 100644 --- a/core/src/main/scala/stryker4s/run/MutantRunner.scala +++ b/core/src/main/scala/stryker4s/run/MutantRunner.scala @@ -1,6 +1,8 @@ package stryker4s.run +import cats.data.NonEmptyList import cats.effect.{IO, Resource} +import cats.syntax.either._ import cats.syntax.functor._ import fs2.io.file.{Files, Path} import fs2.{text, Pipe, Stream} @@ -8,10 +10,10 @@ import mutationtesting.{Metrics, MetricsResult} import stryker4s.config.Config import stryker4s.extension.FileExtensions._ import stryker4s.extension.StreamExtensions._ -import stryker4s.extension.exception.InitialTestRunFailedException +import stryker4s.extension.exception.{InitialTestRunFailedException, UnableToFixCompilerErrorsException} import stryker4s.files.FilesFileResolver import stryker4s.log.Logger -import stryker4s.model._ +import stryker4s.model.{CompilerErrMsg, _} import stryker4s.report.mapper.MutantRunResultMapper import stryker4s.report.{FinishedRunEvent, MutantTestedEvent, Reporter} @@ -20,23 +22,47 @@ import scala.collection.immutable.SortedMap import scala.concurrent.duration._ class MutantRunner( - createTestRunnerPool: Path => Resource[IO, TestRunnerPool], + createTestRunnerPool: Path => Either[NonEmptyList[CompilerErrMsg], Resource[IO, TestRunnerPool]], fileResolver: FilesFileResolver, reporter: Reporter )(implicit config: Config, log: Logger) extends MutantRunResultMapper { - def apply(mutatedFiles: Seq[MutatedFile]): IO[MetricsResult] = - prepareEnv(mutatedFiles) - .flatMap(createTestRunnerPool) - .use { testRunnerPool => - testRunnerPool.loan - .use(initialTestRun) - .flatMap { coverageExclusions => - runMutants(mutatedFiles, testRunnerPool, coverageExclusions).timed + def apply(mutateFiles: Seq[CompilerErrMsg] => IO[Seq[MutatedFile]]): IO[MetricsResult] = { + mutateFiles(Seq.empty).flatMap { mutatedFiles => + run(mutatedFiles) + .flatMap { + case Right(metrics) => IO.pure(metrics.asRight) + case Left(errors) => + log.info("Attempting to remove mutants that gave a compile error...") + //Retry once with the non-compiling mutants removed + mutateFiles(errors.toList).flatMap(run) + } + .flatMap { + case Right(metrics) => IO.pure(metrics) + //Failed at remove the non-compiling mutants + case Left(errs) => IO.raiseError(UnableToFixCompilerErrorsException(errs.toList)) + } + } + } + + def run(mutatedFiles: Seq[MutatedFile]): IO[Either[NonEmptyList[CompilerErrMsg], MetricsResult]] = { + prepareEnv(mutatedFiles).use { path => + createTestRunnerPool(path) match { + case Left(errs) => IO.pure(errs.asLeft) + case Right(testRunnerPoolResource) => + testRunnerPoolResource.use { testRunnerPool => + testRunnerPool.loan + .use(initialTestRun) + .flatMap { coverageExclusions => + runMutants(mutatedFiles, testRunnerPool, coverageExclusions).timed + } + .flatMap(t => createAndReportResults(t._1, t._2)) + .map(Right(_)) } } - .flatMap(t => createAndReportResults(t._1, t._2)) + } + } def createAndReportResults(duration: FiniteDuration, runResults: Map[Path, Seq[MutantRunResult]]) = for { time <- IO.realTime @@ -88,7 +114,7 @@ class MutantRunner( .createDirectories(targetPath.parent.get) .as((mutatedFile, targetPath)) }.map { case (mutatedFile, targetPath) => - Stream(mutatedFile.tree.syntax) + Stream(mutatedFile.mutatedSource) .covary[IO] .through(text.utf8.encode) .through(Files[IO].writeAll(targetPath)) @@ -102,22 +128,35 @@ class MutantRunner( val allMutants = mutatedFiles.flatMap(m => m.mutants.toList.map(m.fileOrigin.relativePath -> _)) - val (staticMutants, rest) = allMutants.partition(m => coverageExclusions.staticMutants.contains(m._2.id)) + val (staticMutants, rest) = allMutants.partition(m => coverageExclusions.staticMutants.contains(m._2.id.globalId)) + val (noCoverageMutants, testableMutants) = - rest.partition(m => coverageExclusions.hasCoverage && !coverageExclusions.coveredMutants.contains(m._2.id)) + rest.partition(m => + coverageExclusions.hasCoverage && !coverageExclusions.coveredMutants.contains(m._2.id.globalId) + ) + + val compilerErrorMutants = + mutatedFiles.flatMap(m => m.nonCompilingMutants.toList.map(m.fileOrigin.relativePath -> _)) if (noCoverageMutants.nonEmpty) { log.info( s"${noCoverageMutants.size} mutants detected as having no code coverage. They will be skipped and marked as NoCoverage" ) - log.debug(s"NoCoverage mutant ids are: ${noCoverageMutants.map(_._2.id).mkString(", ")}") + log.debug(s"NoCoverage mutant ids are: ${noCoverageMutants.map(_._2.id.globalId).mkString(", ")}") } if (staticMutants.nonEmpty) { log.info( s"${staticMutants.size} mutants detected as static. They will be skipped and marked as Ignored" ) - log.debug(s"Static mutant ids are: ${staticMutants.map(_._2.id).mkString(", ")}") + log.debug(s"Static mutant ids are: ${staticMutants.map(_._2.id.globalId).mkString(", ")}") + } + + if (compilerErrorMutants.nonEmpty) { + log.info( + s"${compilerErrorMutants.size} mutants gave a compiler error. They will be marked as such in the report." + ) + log.debug(s"Non-compiling mutant ids are: ${compilerErrorMutants.map(_._2.id.globalId).mkString(", ")}") } def mapPureMutants[K, V, VV](l: Seq[(K, V)], f: V => VV) = @@ -127,6 +166,8 @@ class MutantRunner( val static = mapPureMutants(staticMutants, staticMutant(_)) // Map all no-coverage mutants val noCoverage = mapPureMutants(noCoverageMutants, (m: Mutant) => NoCoverage(m)) + // Map all no-compiling mutants + val noCompiling = mapPureMutants(compilerErrorMutants, (m: Mutant) => CompileError(m)) // Run all testable mutants val totalTestableMutants = testableMutants.size @@ -140,14 +181,14 @@ class MutantRunner( // Back to per-file structure implicit val pathOrdering: Ordering[Path] = implicitly[Ordering[nio.file.Path]].on[Path](_.toNioPath) - (static ++ noCoverage ++ testedMutants) + (static ++ noCoverage ++ noCompiling ++ testedMutants) .fold(SortedMap.empty[Path, Seq[MutantRunResult]]) { case (resultsMap, (path, result)) => val results = resultsMap.getOrElse(path, Seq.empty) :+ result resultsMap + (path -> results) } .compile .lastOrError - .map(_.map { case (k, v) => (k -> v.sortBy(_.mutant.id)) }) + .map(_.map { case (k, v) => (k -> v.sortBy(_.mutant.id.globalId)) }) } def initialTestRun(testRunner: TestRunner): IO[CoverageExclusions] = { diff --git a/core/src/main/scala/stryker4s/run/Stryker4sRunner.scala b/core/src/main/scala/stryker4s/run/Stryker4sRunner.scala index b4ccc7fbb..29fac67cd 100644 --- a/core/src/main/scala/stryker4s/run/Stryker4sRunner.scala +++ b/core/src/main/scala/stryker4s/run/Stryker4sRunner.scala @@ -3,9 +3,8 @@ package stryker4s.run import cats.data.NonEmptyList import cats.effect.{IO, Resource} import fs2.io.file.Path -import stryker4s.Stryker4s import stryker4s.config._ -import stryker4s.files.{ConfigFilesResolver, DiskFileIO, FilesFileResolver, GlobFileResolver, MutatesFileResolver} +import stryker4s.files._ import stryker4s.log.Logger import stryker4s.mutants.Mutator import stryker4s.mutants.applymutants.ActiveMutationContext.ActiveMutationContext @@ -15,6 +14,8 @@ import stryker4s.report._ import stryker4s.report.dashboard.DashboardConfigProvider import stryker4s.run.process.ProcessRunner import stryker4s.run.threshold.ScoreStatus +import stryker4s.Stryker4s +import stryker4s.model.CompilerErrMsg import sttp.client3.SttpBackend import sttp.client3.httpclient.fs2.HttpClientFs2Backend @@ -22,7 +23,7 @@ abstract class Stryker4sRunner(implicit log: Logger) { def run(): IO[ScoreStatus] = { implicit val config: Config = ConfigReader.readConfig() - val createTestRunnerPool = (path: Path) => ResourcePool(resolveTestRunners(path)) + val createTestRunnerPool = (path: Path) => resolveTestRunners(path).map(ResourcePool(_)) val stryker4s = new Stryker4s( resolveMutatesFileSource, @@ -61,7 +62,9 @@ abstract class Stryker4sRunner(implicit log: Logger) { def resolveMatchBuilder(implicit config: Config): MatchBuilder = new MatchBuilder(mutationActivation) - def resolveTestRunners(tmpDir: Path)(implicit config: Config): NonEmptyList[Resource[IO, stryker4s.run.TestRunner]] + def resolveTestRunners(tmpDir: Path)(implicit + config: Config + ): Either[NonEmptyList[CompilerErrMsg], NonEmptyList[Resource[IO, stryker4s.run.TestRunner]]] def resolveMutatesFileSource(implicit config: Config): MutatesFileResolver = new GlobFileResolver( diff --git a/core/src/test/resources/rollbackTest/TestObj1.scala b/core/src/test/resources/rollbackTest/TestObj1.scala new file mode 100644 index 000000000..fb7887b33 --- /dev/null +++ b/core/src/test/resources/rollbackTest/TestObj1.scala @@ -0,0 +1,16 @@ +package stryker4s.mutants + +import java.nio.file.{Files, Paths} + +object TestObj1 { + def test2(a: String): Boolean = { + Files.exists(Paths.get(a)) //Should not get mutated! + } + + def least(a: Int, b: Int): Int = { + (a, b) match { + case (a, b) if a < b => a + case (a, b) if a == b => 0 + } + } +} diff --git a/core/src/test/resources/rollbackTest/TestObj1MutatedWithoutForall.scala b/core/src/test/resources/rollbackTest/TestObj1MutatedWithoutForall.scala new file mode 100644 index 000000000..a27c19f74 --- /dev/null +++ b/core/src/test/resources/rollbackTest/TestObj1MutatedWithoutForall.scala @@ -0,0 +1,36 @@ +package stryker4s.mutants +import java.nio.file.{Files, Paths} +object TestObj1 { + def test2(a: String): Boolean = { + Files.exists(Paths.get(a)) + } + def least(a: Int, b: Int): Int = { + _root_.scala.sys.props.get("ACTIVE_MUTATION").map(_root_.java.lang.Integer.parseInt(_)) match { + case Some(1) => + (a, b) match { + case (a, b) if a <= b => a + case (a, b) if a == b => 0 + } + case Some(2) => + (a, b) match { + case (a, b) if a > b => a + case (a, b) if a == b => 0 + } + case Some(3) => + (a, b) match { + case (a, b) if a == b => a + case (a, b) if a == b => 0 + } + case Some(4) => + (a, b) match { + case (a, b) if a < b => a + case (a, b) if a != b => 0 + } + case _ => + (a, b) match { + case (a, b) if a < b => a + case (a, b) if a == b => 0 + } + } + } +} diff --git a/core/src/test/resources/rollbackTest/TestObj2.scala b/core/src/test/resources/rollbackTest/TestObj2.scala new file mode 100644 index 000000000..287ccf801 --- /dev/null +++ b/core/src/test/resources/rollbackTest/TestObj2.scala @@ -0,0 +1,9 @@ +package stryker4s.mutants + +object TestObj2 { + //spacing to get different mutant on exact same line as in TestObj1 + + def str(a: String): Boolean = { + a == "blah" + } +} diff --git a/core/src/test/resources/rollbackTest/TestObj2Mutated.scala b/core/src/test/resources/rollbackTest/TestObj2Mutated.scala new file mode 100644 index 000000000..d75f51fc1 --- /dev/null +++ b/core/src/test/resources/rollbackTest/TestObj2Mutated.scala @@ -0,0 +1,13 @@ +package stryker4s.mutants +object TestObj2 { + def str(a: String): Boolean = { + _root_.scala.sys.props.get("ACTIVE_MUTATION").map(_root_.java.lang.Integer.parseInt(_)) match { + case Some(5) => + a != "blah" + case Some(6) => + a == "" + case _ => + a == "blah" + } + } +} diff --git a/core/src/test/scala/stryker4s/extension/exception/Stryker4sExceptionTest.scala b/core/src/test/scala/stryker4s/extension/exception/Stryker4sExceptionTest.scala new file mode 100644 index 000000000..941b0d374 --- /dev/null +++ b/core/src/test/scala/stryker4s/extension/exception/Stryker4sExceptionTest.scala @@ -0,0 +1,58 @@ +package stryker4s.extension.exception + +import stryker4s.model.CompilerErrMsg +import stryker4s.testutil.Stryker4sSuite + +class Stryker4sExceptionTest extends Stryker4sSuite { + describe("UnableToBuildPatternMatchException") { + it("should have the correct message") { + UnableToBuildPatternMatchException().getMessage shouldBe "Unable to build pattern match" + } + } + + describe("InitialTestRunFailedException") { + it("should have the correct message") { + InitialTestRunFailedException("testMsg").getMessage shouldBe "testMsg" + } + } + + describe("TestSetupException") { + it("should have the correct message ") { + TestSetupException("ProjectName").getMessage shouldBe + "Could not setup mutation testing environment. Unable to resolve project ProjectName. This could be due to compile errors or misconfiguration of Stryker4s. See debug logs for more information." + } + } + + describe("MutationRunFailedException") { + it("should have the correct message") { + MutationRunFailedException("xyz").getMessage shouldBe "xyz" + } + } + describe("UnableToFixCompilerErrorsException") { + it("should have a nicely formatted message") { + val errs = List( + CompilerErrMsg( + msg = "value forall is not a member of object java.nio.file.Files", + path = "/src/main/scala/com/company/strykerTest/TestObj1.scala", + line = 123 + ), + CompilerErrMsg( + msg = "something something types", + path = "/src/main/scala/com/company/strykerTest/TestObj1.scala", + line = 2 + ), + CompilerErrMsg( + msg = "yet another error with symbols $#'%%$~@1", + path = "/src/main/scala/com/company/strykerTest/TestObj2.scala", + line = 10000 + ) + ) + + UnableToFixCompilerErrorsException(errs).getMessage shouldBe + """Unable to remove non-compiling mutants in the mutated files. As a work-around you can exclude them in the stryker.conf. Please report this issue at https://github.com/stryker-mutator/stryker4s/issues + |/src/main/scala/com/company/strykerTest/TestObj1.scala: 'value forall is not a member of object java.nio.file.Files' + |/src/main/scala/com/company/strykerTest/TestObj1.scala: 'something something types' + |/src/main/scala/com/company/strykerTest/TestObj2.scala: 'yet another error with symbols $#'%%$~@1'""".stripMargin + } + } +} diff --git a/core/src/test/scala/stryker4s/model/CompilerErrMsgTest.scala b/core/src/test/scala/stryker4s/model/CompilerErrMsgTest.scala new file mode 100644 index 000000000..bf461bc12 --- /dev/null +++ b/core/src/test/scala/stryker4s/model/CompilerErrMsgTest.scala @@ -0,0 +1,15 @@ +package stryker4s.model + +import stryker4s.testutil.Stryker4sSuite + +class CompilerErrMsgTest extends Stryker4sSuite { + describe("CompilerErrMsgTest") { + it("should have a nicely formatted toString") { + CompilerErrMsg( + msg = "value forall is not a member of object java.nio.file.Files", + path = "/src/main/scala/com/company/strykerTest/TestObj1.scala", + line = 123 + ).toString shouldBe "/src/main/scala/com/company/strykerTest/TestObj1.scala:L123: 'value forall is not a member of object java.nio.file.Files'" + } + } +} diff --git a/core/src/test/scala/stryker4s/model/MutantIdTest.scala b/core/src/test/scala/stryker4s/model/MutantIdTest.scala new file mode 100644 index 000000000..9d6c12a06 --- /dev/null +++ b/core/src/test/scala/stryker4s/model/MutantIdTest.scala @@ -0,0 +1,12 @@ +package stryker4s.model + +import stryker4s.testutil.Stryker4sSuite + +class MutantIdTest extends Stryker4sSuite { + describe("MutantId") { + it("should have a toString that returns a number") { + MutantId(1234).toString shouldBe "1234" + MutantId(-1).toString shouldBe "-1" + } + } +} diff --git a/core/src/test/scala/stryker4s/model/MutatedFileTest.scala b/core/src/test/scala/stryker4s/model/MutatedFileTest.scala new file mode 100644 index 000000000..2a9181543 --- /dev/null +++ b/core/src/test/scala/stryker4s/model/MutatedFileTest.scala @@ -0,0 +1,20 @@ +package stryker4s.model + +import fs2.io.file.Path +import stryker4s.testutil.Stryker4sSuite + +import scala.meta._ + +class MutatedFileTest extends Stryker4sSuite { + describe("MutatedFile") { + it("should return the new source codes as a string") { + MutatedFile( + Path("/blah/test"), + q"class blah(x: String) { def hi() = x }", + Seq.empty, + Seq.empty, + 0 + ).mutatedSource shouldBe "class blah(x: String) { def hi() = x }" + } + } +} diff --git a/core/src/test/scala/stryker4s/mutants/AddAllMutationsTest.scala b/core/src/test/scala/stryker4s/mutants/AddAllMutationsTest.scala index a3d63d234..abb83ca2a 100644 --- a/core/src/test/scala/stryker4s/mutants/AddAllMutationsTest.scala +++ b/core/src/test/scala/stryker4s/mutants/AddAllMutationsTest.scala @@ -87,7 +87,7 @@ class AddAllMutationsTest extends Stryker4sSuite with LogMatchers { .flatMap(_.mutantStatements) .foreach { mutantStatement => mutatedTree - .find(p"Some(${Lit.Int(mutantStatement.id)})") + .find(p"Some(${Lit.Int(mutantStatement.id.globalId)})") .getOrElse( fail { val mutant = foundMutants.find(_.id == mutantStatement.id).get diff --git a/core/src/test/scala/stryker4s/mutants/RollbackTest.scala b/core/src/test/scala/stryker4s/mutants/RollbackTest.scala new file mode 100644 index 000000000..118df2df6 --- /dev/null +++ b/core/src/test/scala/stryker4s/mutants/RollbackTest.scala @@ -0,0 +1,73 @@ +package stryker4s.mutants + +import stryker4s.config.Config +import stryker4s.extension.mutationtype.Forall +import stryker4s.model.CompilerErrMsg +import stryker4s.mutants.applymutants.{ActiveMutationContext, MatchBuilder, StatementTransformer} +import stryker4s.mutants.findmutants.{MutantFinder, MutantMatcher} +import stryker4s.scalatest.{FileUtil, LogMatchers} +import stryker4s.testutil.Stryker4sIOSuite +import stryker4s.testutil.stubs.TestFileResolver + +import scala.meta._ + +class RollbackTest extends Stryker4sIOSuite with LogMatchers { + describe("Mutator") { + it("should remove a non-compiling mutant") { + implicit val conf: Config = Config.default.copy( + baseDir = FileUtil.getResource("rollbackTest"), + concurrency = 1 //Concurrency 1 to make output order predictable + ) + + val testObj1Path = FileUtil.getResource("rollbackTest/TestObj1.scala") + val testObj2Path = FileUtil.getResource("rollbackTest/TestObj2.scala") + + val testFiles = Seq(testObj1Path, testObj2Path) + + val testSourceCollector = new TestFileResolver(testFiles) + + val mutator = new Mutator( + new MutantFinder(new MutantMatcher), + new StatementTransformer, + new MatchBuilder(ActiveMutationContext.sysProps) + ) + + val errs = List( + CompilerErrMsg("value forall is not a member of object java.nio.file.Files", "rollbackTest/TestObj1.scala", 7) + ) + + val ret = mutator.mutate(testSourceCollector.files, errs) + + ret.asserting { files => + val testObj1Mutated = files.head + val testObj2Mutated = files.last + + testObj1Mutated.fileOrigin shouldBe testObj1Path + testObj1Mutated.tree.structure shouldBe FileUtil + .getResource("rollbackTest/TestObj1MutatedWithoutForall.scala") + .toNioPath + .parse[Source] + .get + .structure + + testObj1Mutated.mutants.size shouldBe 4 + + testObj1Mutated.nonCompilingMutants.size shouldBe 1 + testObj1Mutated.nonCompilingMutants.head.mutationType shouldBe Forall + + testObj1Mutated.excludedMutants shouldBe 0 + + testObj2Mutated.fileOrigin shouldBe testObj2Path + testObj2Mutated.tree.structure shouldBe FileUtil + .getResource("rollbackTest/TestObj2Mutated.scala") + .toNioPath + .parse[Source] + .get + .structure + testObj2Mutated.mutants.size shouldBe 2 + testObj2Mutated.nonCompilingMutants.size shouldBe 0 + testObj2Mutated.excludedMutants shouldBe 0 + } + } + } +} diff --git a/core/src/test/scala/stryker4s/mutants/applymutants/CoverageMatchBuilderTest.scala b/core/src/test/scala/stryker4s/mutants/applymutants/CoverageMatchBuilderTest.scala index c5f83c361..4ba2ea9cd 100644 --- a/core/src/test/scala/stryker4s/mutants/applymutants/CoverageMatchBuilderTest.scala +++ b/core/src/test/scala/stryker4s/mutants/applymutants/CoverageMatchBuilderTest.scala @@ -2,7 +2,7 @@ package stryker4s.mutants.applymutants import stryker4s.extension.TreeExtensions.IsEqualExtension import stryker4s.extension.mutationtype.GreaterThan -import stryker4s.model.{Mutant, TransformedMutants} +import stryker4s.model.{Mutant, MutantId, TransformedMutants} import stryker4s.scalatest.LogMatchers import stryker4s.testutil.Stryker4sSuite @@ -15,7 +15,7 @@ class CoverageMatchBuilderTest extends Stryker4sSuite with LogMatchers { val ids = Iterator.from(0) val originalStatement = q"x >= 15" val mutants = List(q"x > 15", q"x <= 15") - .map(Mutant(ids.next(), originalStatement, _, GreaterThan)) + .map(Mutant(MutantId(ids.next()), originalStatement, _, GreaterThan)) val sut = new CoverageMatchBuilder(ActiveMutationContext.testRunner) // Act @@ -30,7 +30,7 @@ class CoverageMatchBuilderTest extends Stryker4sSuite with LogMatchers { val ids = Iterator.from(0) val originalStatement = q"x >= 15" val mutants = List(q"x > 15", q"x <= 15") - .map(Mutant(ids.next(), originalStatement, _, GreaterThan)) + .map(Mutant(MutantId(ids.next()), originalStatement, _, GreaterThan)) val sut = new CoverageMatchBuilder(ActiveMutationContext.testRunner) // Act diff --git a/core/src/test/scala/stryker4s/mutants/applymutants/MatchBuilderTest.scala b/core/src/test/scala/stryker4s/mutants/applymutants/MatchBuilderTest.scala index 7568baec4..5a6aa5c5f 100644 --- a/core/src/test/scala/stryker4s/mutants/applymutants/MatchBuilderTest.scala +++ b/core/src/test/scala/stryker4s/mutants/applymutants/MatchBuilderTest.scala @@ -1,11 +1,10 @@ package stryker4s.mutants.applymutants import scala.meta._ - import stryker4s.extension.TreeExtensions._ import stryker4s.extension.exception.UnableToBuildPatternMatchException import stryker4s.extension.mutationtype._ -import stryker4s.model.{Mutant, SourceTransformations, TransformedMutants} +import stryker4s.model.{Mutant, MutantId, SourceTransformations, TransformedMutants} import stryker4s.scalatest.LogMatchers import stryker4s.testutil.Stryker4sSuite @@ -17,7 +16,7 @@ class MatchBuilderTest extends Stryker4sSuite with LogMatchers { val ids = Iterator.from(0) val originalStatement = q"x >= 15" val mutants = List(q"x > 15", q"x <= 15") - .map(Mutant(ids.next(), originalStatement, _, GreaterThan)) + .map(Mutant(MutantId(ids.next()), originalStatement, _, GreaterThan)) val sut = new MatchBuilder(ActiveMutationContext.testRunner) // Act @@ -66,12 +65,12 @@ class MatchBuilderTest extends Stryker4sSuite with LogMatchers { val source = source"""class Foo { def bar: Boolean = 15 > 14 }""" val failedMutants = List( - Mutant(0, q"foo", q"bar", GreaterThan), - Mutant(1, q"baz", q"qux", GreaterThan) + Mutant(MutantId(0), q"foo", q"bar", GreaterThan), + Mutant(MutantId(1), q"baz", q"qux", GreaterThan) ) val successfulMutants = List( - Mutant(2, q">", q"15 < 14", GreaterThan), - Mutant(3, q">", q"15 <= 14", GreaterThan) + Mutant(MutantId(2), q">", q"15 < 14", GreaterThan), + Mutant(MutantId(3), q">", q"15 <= 14", GreaterThan) ) val transformed = TransformedMutants(q"14 < 15", failedMutants) val successfulTransformed = TransformedMutants(source.find(q"15 > 14").value, successfulMutants) @@ -379,7 +378,7 @@ class MatchBuilderTest extends Stryker4sSuite with LogMatchers { val topStatement = source.find(origStatement).value.topStatement() val mutant = mutants .map(m => topStatement.transformOnce { case orig if orig.isEqual(origStatement) => m }.get) - .map(m => Mutant(ids.next(), topStatement, m.asInstanceOf[Term], mutation)) + .map(m => Mutant(MutantId(ids.next()), topStatement, m.asInstanceOf[Term], mutation)) .toList TransformedMutants(topStatement, mutant) diff --git a/core/src/test/scala/stryker4s/mutants/applymutants/StatementTransformerTest.scala b/core/src/test/scala/stryker4s/mutants/applymutants/StatementTransformerTest.scala index d146c3a74..1b79015d6 100644 --- a/core/src/test/scala/stryker4s/mutants/applymutants/StatementTransformerTest.scala +++ b/core/src/test/scala/stryker4s/mutants/applymutants/StatementTransformerTest.scala @@ -1,11 +1,10 @@ package stryker4s.mutants.applymutants import scala.meta._ - import stryker4s.extension.ImplicitMutationConversion.mutationToTree import stryker4s.extension.TreeExtensions._ import stryker4s.extension.mutationtype._ -import stryker4s.model.Mutant +import stryker4s.model.{Mutant, MutantId} import stryker4s.testutil.Stryker4sSuite class StatementTransformerTest extends Stryker4sSuite { @@ -64,7 +63,7 @@ class StatementTransformerTest extends Stryker4sSuite { val originalTopTree = q"val x: Boolean = 15 >= 5" val originalTree = originalTopTree.find(q">=").value val mutants = List(EqualTo, GreaterThan, LesserThanEqualTo) - .map(Mutant(0, originalTree, _, GreaterThanEqualTo)) + .map(Mutant(MutantId(0), originalTree, _, GreaterThanEqualTo)) // Act val transformedMutant = sut.transformMutant(originalTree, mutants) @@ -84,7 +83,7 @@ class StatementTransformerTest extends Stryker4sSuite { val source = "object Foo { def bar: Boolean = 15 >= 4 }".parse[Source].get val origTree = source.find(q">=").value val mutants = List(EqualTo, GreaterThan, LesserThanEqualTo) - .map(Mutant(0, origTree, _, GreaterThanEqualTo)) + .map(Mutant(MutantId(0), origTree, _, GreaterThanEqualTo)) // Act val result = sut.transformSource(source, mutants) @@ -103,11 +102,11 @@ class StatementTransformerTest extends Stryker4sSuite { val firstOrigTree = source.find(q">=").value val firstMutants: Seq[Mutant] = List(EqualTo, GreaterThan, LesserThanEqualTo) - .map(Mutant(0, firstOrigTree, _, GreaterThanEqualTo)) + .map(Mutant(MutantId(0), firstOrigTree, _, GreaterThanEqualTo)) val secOrigTree = source.find(q"<").value val secondMutants: Seq[Mutant] = List(LesserThanEqualTo, GreaterThan, EqualTo) - .map(Mutant(0, secOrigTree, _, GreaterThanEqualTo)) + .map(Mutant(MutantId(0), secOrigTree, _, GreaterThanEqualTo)) val statements = firstMutants ++ secondMutants diff --git a/core/src/test/scala/stryker4s/mutants/findmutants/MutantMatcherTest.scala b/core/src/test/scala/stryker4s/mutants/findmutants/MutantMatcherTest.scala index 748e6fc5d..bdaa7e2da 100644 --- a/core/src/test/scala/stryker4s/mutants/findmutants/MutantMatcherTest.scala +++ b/core/src/test/scala/stryker4s/mutants/findmutants/MutantMatcherTest.scala @@ -540,7 +540,7 @@ class MutantMatcherTest extends Stryker4sSuite { val mutants = sut.TermExtensions(GreaterThan).~~>(LesserThan, GreaterThanEqualTo, EqualTo).collect { case Right(v) => v } - mutants.map(_.id) should contain theSameElementsAs List(0, 1, 2) + mutants.map(_.id.globalId) should contain theSameElementsAs List(0, 1, 2) } } diff --git a/core/src/test/scala/stryker4s/report/mapper/MutantRunResultMapperTest.scala b/core/src/test/scala/stryker4s/report/mapper/MutantRunResultMapperTest.scala index f9870ff08..333aaf823 100644 --- a/core/src/test/scala/stryker4s/report/mapper/MutantRunResultMapperTest.scala +++ b/core/src/test/scala/stryker4s/report/mapper/MutantRunResultMapperTest.scala @@ -7,7 +7,7 @@ import stryker4s.config.{Config, Thresholds => ConfigThresholds} import stryker4s.extension.FileExtensions._ import stryker4s.extension.ImplicitMutationConversion._ import stryker4s.extension.mutationtype._ -import stryker4s.model.{Killed, Mutant, Survived} +import stryker4s.model.{Killed, Mutant, MutantId, Survived} import stryker4s.scalatest.FileUtil import stryker4s.testutil.Stryker4sSuite @@ -78,6 +78,6 @@ class MutantRunResultMapperTest extends Stryker4sSuite with Inside { import scala.meta._ val parsed = file.toNioPath.parse[Source] val foundOrig = parsed.get.find(original).value - Mutant(id, foundOrig, category.tree, category) + Mutant(MutantId(id), foundOrig, category.tree, category) } } diff --git a/core/src/test/scala/stryker4s/run/MutantRunnerTest.scala b/core/src/test/scala/stryker4s/run/MutantRunnerTest.scala index 10ff8620a..f07b1cda3 100644 --- a/core/src/test/scala/stryker4s/run/MutantRunnerTest.scala +++ b/core/src/test/scala/stryker4s/run/MutantRunnerTest.scala @@ -1,9 +1,10 @@ package stryker4s.run +import cats.effect.IO import org.mockito.captor.ArgCaptor import stryker4s.config.Config import stryker4s.extension.mutationtype.EmptyString -import stryker4s.model.{Killed, Mutant, MutatedFile, Survived} +import stryker4s.model._ import stryker4s.report.{FinishedRunEvent, Reporter} import stryker4s.scalatest.{FileUtil, LogMatchers} import stryker4s.testutil.stubs.{TestFileResolver, TestRunnerStub} @@ -21,17 +22,17 @@ class MutantRunnerTest extends Stryker4sIOSuite with MockitoIOSuite with LogMatc val reporterMock = mock[Reporter] whenF(reporterMock.onRunFinished(any[FinishedRunEvent])).thenReturn(()) when(reporterMock.mutantTested).thenReturn(_.drain) - val mutant = Mutant(3, q"0", q"zero", EmptyString) - val secondMutant = Mutant(1, q"1", q"one", EmptyString) - val thirdMutant = Mutant(2, q"5", q"5", EmptyString) + val mutant = Mutant(MutantId(3), q"0", q"zero", EmptyString) + val secondMutant = Mutant(MutantId(1), q"1", q"one", EmptyString) + val thirdMutant = Mutant(MutantId(2), q"5", q"5", EmptyString) val testRunner = TestRunnerStub.withResults(Killed(mutant), Killed(secondMutant), Survived(thirdMutant)) val sut = new MutantRunner(testRunner, fileCollectorMock, reporterMock) val file = FileUtil.getResource("scalaFiles/simpleFile.scala") val mutants = Seq(mutant, secondMutant, thirdMutant) - val mutatedFile = MutatedFile(file, q"def foo = 4", mutants, 0) + val mutatedFile = MutatedFile(file, q"def foo = 4", mutants, Seq.empty, 0) - sut(List(mutatedFile)).asserting { result => + sut(_ => IO.pure(List(mutatedFile))).asserting { result => val captor = ArgCaptor[FinishedRunEvent] verify(reporterMock, times(1)).onRunFinished(captor.capture) val runReport = captor.value.report.files.loneElement @@ -47,5 +48,52 @@ class MutantRunnerTest extends Stryker4sIOSuite with MockitoIOSuite with LogMatc result.survived shouldBe 1 } } + + it("should return a mutationScore of 66.67 when 2 of 3 mutants are killed and 1 doesn't compile.") { + val fileCollectorMock = new TestFileResolver(Seq.empty) + val reporterMock = mock[Reporter] + whenF(reporterMock.onRunFinished(any[FinishedRunEvent])).thenReturn(()) + when(reporterMock.mutantTested).thenReturn(_.drain) + + val mutant = Mutant(MutantId(3), q"0", q"zero", EmptyString) + val secondMutant = Mutant(MutantId(1), q"1", q"one", EmptyString) + val thirdMutant = Mutant(MutantId(2), q"5", q"5", EmptyString) + val nonCompilingMutant = Mutant(MutantId(4), q"7", q"2", EmptyString) + + val errs = List(CompilerErrMsg("blah", "xyz", 123)) + val testRunner = + TestRunnerStub.withInitialCompilerError( + errs, + Killed(mutant), + Killed(secondMutant), + Survived(thirdMutant), + CompileError(nonCompilingMutant) + ) + + val sut = new MutantRunner(testRunner, fileCollectorMock, reporterMock) + val file = FileUtil.getResource("scalaFiles/simpleFile.scala") + val mutants = Seq(mutant, secondMutant, thirdMutant) + val mutatedFile = MutatedFile(file, q"def foo = 4", mutants, Seq(nonCompilingMutant), 0) + + sut(_ => IO(List(mutatedFile))).asserting { result => + val captor = ArgCaptor[FinishedRunEvent] + verify(reporterMock, times(1)).onRunFinished(captor.capture) + val runReport = captor.value.report.files.loneElement + + "Setting up mutated environment..." shouldBe loggedAsInfo + "Starting initial test run..." shouldBe loggedAsInfo + "Initial test run succeeded! Testing mutants..." shouldBe loggedAsInfo + "Attempting to remove mutants that gave a compile error..." shouldBe loggedAsInfo + + runReport._1 shouldBe "simpleFile.scala" + runReport._2.mutants.map(_.id) shouldBe List("1", "2", "3", "4") + result.mutationScore shouldBe ((2d / 3d) * 100) + result.totalMutants shouldBe 4 + result.totalInvalid shouldBe 1 + result.killed shouldBe 2 + result.survived shouldBe 1 + result.compileErrors shouldBe 1 + } + } } } diff --git a/core/src/test/scala/stryker4s/testutil/TestData.scala b/core/src/test/scala/stryker4s/testutil/TestData.scala index 841ff46a0..eec5c2b10 100644 --- a/core/src/test/scala/stryker4s/testutil/TestData.scala +++ b/core/src/test/scala/stryker4s/testutil/TestData.scala @@ -4,14 +4,14 @@ import fs2.io.file.Path import mutationtesting.{Metrics, MetricsResult, MutationTestResult, Thresholds} import stryker4s.config.Config import stryker4s.extension.mutationtype.GreaterThan -import stryker4s.model.Mutant +import stryker4s.model.{Mutant, MutantId} import stryker4s.report.FinishedRunEvent import scala.concurrent.duration._ import scala.meta._ trait TestData { - def createMutant = Mutant(0, q"<", q">", GreaterThan) + def createMutant = Mutant(MutantId(0), q"<", q">", GreaterThan) def createMutationTestResult = MutationTestResult(thresholds = Thresholds(100, 0), files = Map.empty) diff --git a/core/src/test/scala/stryker4s/testutil/stubs/TestRunnerStub.scala b/core/src/test/scala/stryker4s/testutil/stubs/TestRunnerStub.scala index 44edd3ac3..10c581825 100644 --- a/core/src/test/scala/stryker4s/testutil/stubs/TestRunnerStub.scala +++ b/core/src/test/scala/stryker4s/testutil/stubs/TestRunnerStub.scala @@ -5,8 +5,8 @@ import cats.effect.{IO, Resource} import cats.syntax.applicativeError._ import fs2.io.file.Path import stryker4s.extension.mutationtype.LesserThan -import stryker4s.model.{InitialTestRunResult, Killed, Mutant, MutantRunResult, NoCoverageInitialTestRun} -import stryker4s.run.{ResourcePool, TestRunner} +import stryker4s.model._ +import stryker4s.run.{ResourcePool, TestRunner, TestRunnerPool} import scala.meta._ @@ -24,8 +24,26 @@ class TestRunnerStub(results: Seq[() => MutantRunResult]) extends TestRunner { object TestRunnerStub { - def resource = withResults(Killed(Mutant(0, q">", q"<", LesserThan))) + def resource = withResults(Killed(Mutant(MutantId(0), q">", q"<", LesserThan))) + + def withResults(mutants: MutantRunResult*) = (_: Path) => makeResults(mutants) + + def withInitialCompilerError( + errs: List[CompilerErrMsg], + mutants: MutantRunResult* + ): Path => Either[NonEmptyList[CompilerErrMsg], Resource[IO, TestRunnerPool]] = { + var firstRun = true + (_: Path) => + if (firstRun) { + firstRun = false + Left(NonEmptyList.fromListUnsafe(errs)) + } else { + makeResults(mutants) + } + } - def withResults(mutants: MutantRunResult*) = (_: Path) => - ResourcePool(NonEmptyList.of(Resource.pure[IO, TestRunner](new TestRunnerStub(mutants.map(() => _))))) + private def makeResults( + mutants: Seq[MutantRunResult] + ): Either[NonEmptyList[CompilerErrMsg], Resource[IO, TestRunnerPool]] = + Right(ResourcePool(NonEmptyList.of(Resource.pure[IO, TestRunner](new TestRunnerStub(mutants.map(() => _)))))) } diff --git a/maven/src/main/scala/stryker4s/maven/Stryker4sMavenRunner.scala b/maven/src/main/scala/stryker4s/maven/Stryker4sMavenRunner.scala index c2ec622aa..fb06309de 100644 --- a/maven/src/main/scala/stryker4s/maven/Stryker4sMavenRunner.scala +++ b/maven/src/main/scala/stryker4s/maven/Stryker4sMavenRunner.scala @@ -8,6 +8,7 @@ import org.apache.maven.shared.invoker.Invoker import stryker4s.config.Config import stryker4s.log.Logger import stryker4s.maven.runner.MavenTestRunner +import stryker4s.model.CompilerErrMsg import stryker4s.mutants.applymutants.ActiveMutationContext.{envVar, ActiveMutationContext} import stryker4s.run.Stryker4sRunner @@ -19,14 +20,14 @@ class Stryker4sMavenRunner(project: MavenProject, invoker: Invoker)(implicit log override def resolveTestRunners( tmpDir: Path - )(implicit config: Config): NonEmptyList[Resource[IO, MavenTestRunner]] = { + )(implicit config: Config): Either[NonEmptyList[CompilerErrMsg], NonEmptyList[Resource[IO, MavenTestRunner]]] = { val goals = List("test") val properties = new Properties(project.getProperties) setTestProperties(properties, config.testFilter) invoker.setWorkingDirectory(tmpDir.toNioPath.toFile()) - NonEmptyList.of(Resource.pure[IO, MavenTestRunner](new MavenTestRunner(project, invoker, properties, goals))) + Right(NonEmptyList.of(Resource.pure[IO, MavenTestRunner](new MavenTestRunner(project, invoker, properties, goals)))) } private def setTestProperties(properties: Properties, testFilter: Seq[String]): Unit = { diff --git a/maven/src/test/scala/stryker4s/maven/Stryker4sMavenRunnerTest.scala b/maven/src/test/scala/stryker4s/maven/Stryker4sMavenRunnerTest.scala index 9dba154b4..f747d446f 100644 --- a/maven/src/test/scala/stryker4s/maven/Stryker4sMavenRunnerTest.scala +++ b/maven/src/test/scala/stryker4s/maven/Stryker4sMavenRunnerTest.scala @@ -24,6 +24,8 @@ class Stryker4sMavenRunnerTest extends Stryker4sSuite with MockitoSugar { sut .resolveTestRunners(tmpDir) + .right + .get .head .use(result => { verify(invokerMock).setWorkingDirectory(eqTo(tmpDir.toNioPath.toFile())) @@ -42,6 +44,8 @@ class Stryker4sMavenRunnerTest extends Stryker4sSuite with MockitoSugar { sut .resolveTestRunners(tmpDir) + .right + .get .head .use(result => { result.properties.getProperty("test") should equal(expectedTestFilter.mkString(", ")) @@ -62,6 +66,8 @@ class Stryker4sMavenRunnerTest extends Stryker4sSuite with MockitoSugar { sut .resolveTestRunners(tmpDir) + .right + .get .head .use(result => IO.pure(result.properties.getProperty("test") should equal(s"*OtherTest, $expectedTestFilter"))) .unsafeRunSync() @@ -78,6 +84,8 @@ class Stryker4sMavenRunnerTest extends Stryker4sSuite with MockitoSugar { sut .resolveTestRunners(tmpDir) + .right + .get .head .use(result => IO.pure(result.properties.getProperty("wildcardSuites") should equal(s"*OtherTest,$expectedTestFilter")) diff --git a/maven/src/test/scala/stryker4s/maven/runner/MavenTestRunnerTest.scala b/maven/src/test/scala/stryker4s/maven/runner/MavenTestRunnerTest.scala index 60dfe6e7f..3358add56 100644 --- a/maven/src/test/scala/stryker4s/maven/runner/MavenTestRunnerTest.scala +++ b/maven/src/test/scala/stryker4s/maven/runner/MavenTestRunnerTest.scala @@ -9,7 +9,7 @@ import org.mockito.captor.ArgCaptor import org.mockito.scalatest.MockitoSugar import stryker4s.config.Config import stryker4s.extension.mutationtype.LesserThan -import stryker4s.model.{Killed, Mutant, NoCoverageInitialTestRun, Survived} +import stryker4s.model.{Killed, Mutant, MutantId, NoCoverageInitialTestRun, Survived} import stryker4s.testutil.Stryker4sSuite import java.{util => ju} @@ -81,7 +81,7 @@ class MavenTestRunnerTest extends Stryker4sSuite with MockitoSugar { when(invokerMock.execute(any[InvocationRequest])).thenReturn(mockResult) val sut = new MavenTestRunner(new MavenProject(), invokerMock, properties, goals) - val result = sut.runMutant(Mutant(1, q">", q"<", LesserThan)).unsafeRunSync() + val result = sut.runMutant(Mutant(MutantId(1), q">", q"<", LesserThan)).unsafeRunSync() result shouldBe a[Killed] } @@ -93,7 +93,7 @@ class MavenTestRunnerTest extends Stryker4sSuite with MockitoSugar { when(invokerMock.execute(any[InvocationRequest])).thenReturn(mockResult) val sut = new MavenTestRunner(new MavenProject(), invokerMock, properties, goals) - val result = sut.runMutant(Mutant(1, q">", q"<", LesserThan)).unsafeRunSync() + val result = sut.runMutant(Mutant(MutantId(1), q">", q"<", LesserThan)).unsafeRunSync() result shouldBe a[Survived] } @@ -109,7 +109,7 @@ class MavenTestRunnerTest extends Stryker4sSuite with MockitoSugar { val sut = new MavenTestRunner(project, invokerMock, project.getProperties(), goals) - sut.runMutant(Mutant(1, q">", q"<", LesserThan)).unsafeRunSync() + sut.runMutant(Mutant(MutantId(1), q">", q"<", LesserThan)).unsafeRunSync() verify(invokerMock).execute(captor) val invokedRequest = captor.value @@ -132,7 +132,7 @@ class MavenTestRunnerTest extends Stryker4sSuite with MockitoSugar { mavenProject.getActiveProfiles.add(profile) val sut = new MavenTestRunner(mavenProject, invokerMock, properties, goals) - sut.runMutant(Mutant(1, q">", q"<", LesserThan)).unsafeRunSync() + sut.runMutant(Mutant(MutantId(1), q">", q"<", LesserThan)).unsafeRunSync() verify(invokerMock).execute(captor) val invokedRequest = captor.value diff --git a/sbt/src/main/scala/stryker4s/sbt/Stryker4sSbtRunner.scala b/sbt/src/main/scala/stryker4s/sbt/Stryker4sSbtRunner.scala index 1b675e027..2ed9c730a 100644 --- a/sbt/src/main/scala/stryker4s/sbt/Stryker4sSbtRunner.scala +++ b/sbt/src/main/scala/stryker4s/sbt/Stryker4sSbtRunner.scala @@ -9,7 +9,9 @@ import sbt.internal.LogManager import stryker4s.config.{Config, TestFilter} import stryker4s.extension.FileExtensions._ import stryker4s.extension.exception.TestSetupException +import stryker4s.files.{FilesFileResolver, MutatesFileResolver, SbtFilesResolver, SbtMutatesResolver} import stryker4s.log.Logger +import stryker4s.model.CompilerErrMsg import stryker4s.mutants.applymutants.ActiveMutationContext.ActiveMutationContext import stryker4s.mutants.applymutants.{ActiveMutationContext, CoverageMatchBuilder, MatchBuilder} import stryker4s.run.{Stryker4sRunner, TestRunner} @@ -18,10 +20,6 @@ import stryker4s.sbt.runner.{LegacySbtTestRunner, SbtTestRunner} import java.io.{File => JFile, PrintStream} import scala.concurrent.duration.FiniteDuration -import stryker4s.files.SbtMutatesResolver -import stryker4s.files.MutatesFileResolver -import stryker4s.files.FilesFileResolver -import stryker4s.files.SbtFilesResolver /** This Runner run Stryker mutations in a single SBT session * @@ -45,7 +43,7 @@ class Stryker4sSbtRunner( def resolveTestRunners( tmpDir: Path - )(implicit config: Config): NonEmptyList[Resource[IO, TestRunner]] = { + )(implicit config: Config): Either[NonEmptyList[CompilerErrMsg], NonEmptyList[Resource[IO, TestRunner]]] = { def setupLegacySbtTestRunner( settings: Seq[Def.Setting[_]], extracted: Extracted @@ -69,7 +67,7 @@ class Stryker4sSbtRunner( def setupSbtTestRunner( settings: Seq[Def.Setting[_]], extracted: Extracted - ): NonEmptyList[Resource[IO, TestRunner]] = { + ): Either[NonEmptyList[CompilerErrMsg], NonEmptyList[Resource[IO, TestRunner]]] = { val stryker4sVersion = this.getClass().getPackage().getImplementationVersion() log.debug(s"Resolved stryker4s version $stryker4sVersion") @@ -78,44 +76,82 @@ class Stryker4sSbtRunner( "io.stryker-mutator" %% "sbt-stryker4s-testrunner" % stryker4sVersion ) val newState = extracted.appendWithSession(fullSettings, state) - def extractTaskValue[T](task: TaskKey[T], name: String) = + + def extractTaskValue[T](task: TaskKey[T], name: String) = { + Project.runTask(task, newState) match { case Some((_, Value(result))) => result case other => log.debug(s"Expected $name but got $other") - throw new TestSetupException(name) + throw TestSetupException(name) } + } - val classpath = extractTaskValue(Test / fullClasspath, "classpath").map(_.data.getPath()) - - val javaOpts = extractTaskValue(Test / javaOptions, "javaOptions") - - val frameworks = extractTaskValue(Test / loadedTestFrameworks, "test frameworks").values.toSeq - - val testGroups = extractTaskValue(Test / testGrouping, "testGrouping").map { group => - if (config.testFilter.isEmpty) group - else { - val testFilter = new TestFilter() - val filteredTests = group.tests.filter(t => testFilter.filter(t.name)) - group.copy(tests = filteredTests) + //SBT returns any errors as a Incomplete case class, which can contain other Incomplete instances + //You have to recursively search through them to get the real exception + def getRootCause(i: Incomplete): Seq[Throwable] = { + i.directCause match { + case None => + i.causes.flatMap(getRootCause) + case Some(cause) => + cause +: i.causes.flatMap(getRootCause) } } - val concurrency = if (config.debug.debugTestRunner) { - log.warn( - "'debug.debug-test-runner' config is 'true', creating 1 test-runner with debug arguments enabled on port 8000." - ) - 1 - } else { - log.info(s"Creating ${config.concurrency} test-runners") - config.concurrency + //See if the mutations compile, and if not extract the errors + val compilerErrors = Project.runTask(Compile / Keys.compile, newState) match { + case Some((_, Inc(cause))) => + val compileErrors = (getRootCause(cause) collect { case exception: sbt.internal.inc.CompileFailed => + exception.problems.flatMap { e => + for { + path <- e.position().sourceFile().asScala + pathStr = tmpDir.relativize(Path(path.absolutePath)).toString + line <- e.position().line().asScala + } yield CompilerErrMsg(e.message(), pathStr, line) + }.toSeq + }).flatten.toList + + NonEmptyList.fromList(compileErrors) + case _ => + None } - val portStart = 13336 - val portRanges = NonEmptyList.fromListUnsafe((1 to concurrency).map(_ + portStart).toList) - - portRanges.map { port => - SbtTestRunner.create(classpath, javaOpts, frameworks, testGroups, port, sharedTimeout) + compilerErrors match { + case Some(errors) => Left(errors) + case None => + val classpath = extractTaskValue(Test / fullClasspath, "classpath").map(_.data.getPath()) + + val javaOpts = extractTaskValue(Test / javaOptions, "javaOptions") + + val frameworks = extractTaskValue(Test / loadedTestFrameworks, "test frameworks").values.toSeq + + val testGroups = extractTaskValue(Test / testGrouping, "testGrouping").map { group => + if (config.testFilter.isEmpty) group + else { + val testFilter = new TestFilter() + val filteredTests = group.tests.filter(t => testFilter.filter(t.name)) + group.copy(tests = filteredTests) + } + } + + val concurrency = if (config.debug.debugTestRunner) { + log.warn( + "'debug.debug-test-runner' config is 'true', creating 1 test-runner with debug arguments enabled on port 8000." + ) + 1 + } else { + log.info(s"Creating ${config.concurrency} test-runners") + config.concurrency + } + + val portStart = 13336 + val portRanges = NonEmptyList.fromListUnsafe((1 to concurrency).map(_ + portStart).toList) + + Right( + portRanges.map { port => + SbtTestRunner.create(classpath, javaOpts, frameworks, testGroups, port, sharedTimeout) + } + ) } } @@ -164,9 +200,10 @@ class Stryker4sSbtRunner( val (settings, extracted) = extractSbtProject(tmpDir) - if (config.legacyTestRunner) - setupLegacySbtTestRunner(settings, extracted) - else + if (config.legacyTestRunner) { + //No compiler error handling in the legacy runner + Right(setupLegacySbtTestRunner(settings, extracted)) + } else setupSbtTestRunner(settings, extracted) } diff --git a/sbt/src/main/scala/stryker4s/sbt/runner/LegacySbtTestRunner.scala b/sbt/src/main/scala/stryker4s/sbt/runner/LegacySbtTestRunner.scala index e4905fb47..ce58578b7 100644 --- a/sbt/src/main/scala/stryker4s/sbt/runner/LegacySbtTestRunner.scala +++ b/sbt/src/main/scala/stryker4s/sbt/runner/LegacySbtTestRunner.scala @@ -23,7 +23,7 @@ class LegacySbtTestRunner(initialState: State, settings: Seq[Def.Setting[_]], ex def runMutant(mutant: Mutant): IO[MutantRunResult] = { val mutationState = - extracted.appendWithSession(settings :+ mutationSetting(mutant.id), initialState) + extracted.appendWithSession(settings :+ mutationSetting(mutant.id.globalId), initialState) runTests( mutationState, onError = { diff --git a/sbt/src/main/scala/stryker4s/sbt/runner/ProcessTestRunner.scala b/sbt/src/main/scala/stryker4s/sbt/runner/ProcessTestRunner.scala index e295d8922..6196213dd 100644 --- a/sbt/src/main/scala/stryker4s/sbt/runner/ProcessTestRunner.scala +++ b/sbt/src/main/scala/stryker4s/sbt/runner/ProcessTestRunner.scala @@ -20,7 +20,7 @@ import scala.util.control.NonFatal class ProcessTestRunner(testProcess: TestRunnerConnection) extends TestRunner { override def runMutant(mutant: Mutant): IO[MutantRunResult] = { - val message = StartTestRun(mutant.id) + val message = StartTestRun(mutant.id.globalId) testProcess.sendMessage(message).map { case _: TestsSuccessful => Survived(mutant) case _: TestsUnsuccessful => Killed(mutant) diff --git a/sbt/src/sbt-test/sbt-stryker4s/test-1/src/main/scala/example/TestObj1.scala b/sbt/src/sbt-test/sbt-stryker4s/test-1/src/main/scala/example/TestObj1.scala new file mode 100644 index 000000000..8cc638cc0 --- /dev/null +++ b/sbt/src/sbt-test/sbt-stryker4s/test-1/src/main/scala/example/TestObj1.scala @@ -0,0 +1,14 @@ +package example + +import java.nio.file.{Files, Paths} + +object TestObj { + + def mutatesOkay(s: String) = s.exists(_ == 'a') + + def test2(a: String): Boolean = { + Files.exists(Paths.get(a)) //Should not get mutated! + } + + def alsoMutatesOkay(s: String) = s.exists(_ == 'b') +} diff --git a/sbt/src/sbt-test/sbt-stryker4s/test-1/src/main/scala/example/TestObj2.scala b/sbt/src/sbt-test/sbt-stryker4s/test-1/src/main/scala/example/TestObj2.scala new file mode 100644 index 000000000..d52e088d5 --- /dev/null +++ b/sbt/src/sbt-test/sbt-stryker4s/test-1/src/main/scala/example/TestObj2.scala @@ -0,0 +1,9 @@ +package example + +object TestObj2 { + //spacing to get exact same line as in TestObj + + def str(a: String): Boolean = { + a == "blah" + } +} diff --git a/sbt/src/sbt-test/sbt-stryker4s/test-1/src/test/scala/example/TestObj1Spec.scala b/sbt/src/sbt-test/sbt-stryker4s/test-1/src/test/scala/example/TestObj1Spec.scala new file mode 100644 index 000000000..b2df64ee2 --- /dev/null +++ b/sbt/src/sbt-test/sbt-stryker4s/test-1/src/test/scala/example/TestObj1Spec.scala @@ -0,0 +1,21 @@ +package example + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class Spec extends AnyFlatSpec with Matchers { + it should "check for a's in mutatesOkay" in { + TestObj.mutatesOkay(" a ") shouldBe true + TestObj.mutatesOkay(" b ") shouldBe false + } + + it should "return false if a file does not exists" in { + TestObj.test2("/home/blah/fake") shouldBe false + } + + it should "check for b's in alsoMutatesOkay" in { + TestObj.alsoMutatesOkay(" b ") shouldBe true + TestObj.alsoMutatesOkay(" a ") shouldBe false + } + +} diff --git a/sbt/src/sbt-test/sbt-stryker4s/test-1/src/test/scala/example/TestObj2Spec.scala b/sbt/src/sbt-test/sbt-stryker4s/test-1/src/test/scala/example/TestObj2Spec.scala new file mode 100644 index 000000000..597857063 --- /dev/null +++ b/sbt/src/sbt-test/sbt-stryker4s/test-1/src/test/scala/example/TestObj2Spec.scala @@ -0,0 +1,11 @@ +package example + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class Spec2 extends AnyFlatSpec with Matchers { + it should "check the string" in { + TestObj2.str("hi") shouldBe false + TestObj2.str("blah") shouldBe true + } +} diff --git a/sbt/src/sbt-test/sbt-stryker4s/test-1/stryker4s.conf b/sbt/src/sbt-test/sbt-stryker4s/test-1/stryker4s.conf index 003a54bb1..314049447 100644 --- a/sbt/src/sbt-test/sbt-stryker4s/test-1/stryker4s.conf +++ b/sbt/src/sbt-test/sbt-stryker4s/test-1/stryker4s.conf @@ -2,10 +2,10 @@ stryker4s { files: [ "*", "!global" ] reporters: ["console", "json", "html"] thresholds: { - # Should be 66,66%. Break if lower than that - high: 66 - low: 65 - break: 64 + # Should be 88%. Something is broken if it's lower than that + high: 90 + low: 89 + break: 88 } test-filter: ["!*IgnoreMeTest"] }