Support for scenario-level parallel execution (#277)

* import tests from PR 119

* small code cleanup

* Prototype to support method-level parallelization for MsTest

* Add SystemTests for parallel execution

* Remove obsolete Generator unit test

* make sure that all remaining after feature hooks are called

* remove invalid expectations of specs test about feature context

* Support method-level parallel execution for NUnit

* Support new execution structure for xUnit

* code cleanup

* generate featureInfo as field

* slow down parallel test to avoid false errors (tests are so fast on Linux CI that they don't run in parallel)

* fix VB compatibility

* update CHANGELOG

* update CHANGELOG

* remove obsolete specs feature

* Update docs

* fix CHANGELOG

* refactor Log class generation for system tests

* Improve System Tests parallel handling

* apply hinting to choose the test runners in more optimal way

* fix unit test warning

* Add extra check to ensure scenario-level parallelization

* Use feature info to optimize test runner reuse

* fix a comment

* fix typo, make docs clearer

CHANGELOG.md

@@ -1,7 +1,9 @@
 # [vNext]
 
 ## Improvements:
+
 * Upgrade to Gherkin v30 from v29 (see [Gherkin changelog](https://github.com/cucumber/gherkin/blob/main/CHANGELOG.md)) (#305)
+* Support scenario-level (method-level) parallel execution (#119, #277)
 
 ## Bug fixes:
 

Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.cs

@@ -6,21 +6,23 @@
 using global::System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 
+[assembly: NUnit.Framework.FixtureLifeCycle(NUnit.Framework.LifeCycle.InstancePerTestCase)]
+
 [GeneratedCode("Reqnroll", "REQNROLL_VERSION")]
 [global::NUnit.Framework.SetUpFixture]
-public class PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks
+public static class PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks
 {
     [global::NUnit.Framework.OneTimeSetUp]
     [MethodImpl(MethodImplOptions.NoInlining)]
-    public async Task AssemblyInitializeAsync()
+    public static async Task AssemblyInitializeAsync()
     {
         var currentAssembly = typeof(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly;
         await global::Reqnroll.TestRunnerManager.OnTestRunStartAsync(currentAssembly);
     }
 
     [global::NUnit.Framework.OneTimeTearDown]
     [MethodImpl(MethodImplOptions.NoInlining)]
-    public async ValueTask AssemblyCleanupAsync()
+    public static async ValueTask AssemblyCleanupAsync()
     {
         var currentAssembly = typeof(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly;
         await global::Reqnroll.TestRunnerManager.OnTestRunEndAsync(currentAssembly);

Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.vb

@@ -7,19 +7,21 @@ Imports System.CodeDom.Compiler
 Imports System.Reflection
 Imports System.Runtime.CompilerServices
 
+<assembly: NUnit.Framework.FixtureLifeCycle(NUnit.Framework.LifeCycle.InstancePerTestCase)>
+
 <GeneratedCode("Reqnroll", "REQNROLL_VERSION")>
 <SetUpFixture>
 Public NotInheritable Class PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks
     <OneTimeSetUp>
     <MethodImpl(MethodImplOptions.NoInlining)>
-    Public Async Function AssemblyInitializeAsync() As Task
+    Public Shared Async Function AssemblyInitializeAsync() As Task
         Dim currentAssembly As Assembly = GetType(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly
         Await Global.Reqnroll.TestRunnerManager.OnTestRunStartAsync(currentAssembly)
     End Function
 
     <OneTimeTearDown>
     <MethodImpl(MethodImplOptions.NoInlining)>
-    Public Async Function AssemblyCleanupAsync() As Task
+    Public Shared Async Function AssemblyCleanupAsync() As Task
         Dim currentAssembly As Assembly = GetType(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly
         Await Global.Reqnroll.TestRunnerManager.OnTestRunEndAsync(currentAssembly)
     End Function

Reqnroll.Generator/CodeDom/CodeDomHelper.cs

@@ -298,6 +298,18 @@ public string GetGlobalizedTypeName(Type type)
             // Global namespaces not yet supported in VB
             return type.FullName!;
         }
+
+        public CodeExpression CreateOptionalArgumentExpression(string parameterName, CodeVariableReferenceExpression valueExpression)
+        {
+            switch (TargetLanguage)
+            {
+                case CodeDomProviderLanguage.CSharp:
+                    return new CodeSnippetExpression($"{parameterName}: {valueExpression.VariableName}");
+                case CodeDomProviderLanguage.VB:
+                    return new CodeSnippetExpression($"{parameterName} := {valueExpression.VariableName}");
+            }
+            return valueExpression;
+        }
     }
 }
 

Reqnroll.Generator/Generation/GeneratorConstants.cs

@@ -13,6 +13,7 @@ public class GeneratorConstants
         public const string TESTCLASS_CLEANUP_NAME = "FeatureTearDownAsync";
         public const string BACKGROUND_NAME = "FeatureBackgroundAsync";
         public const string TESTRUNNER_FIELD = "testRunner";
+        public const string FEATUREINFO_FIELD = "featureInfo";
         public const string REQNROLL_NAMESPACE = "Reqnroll";
         public const string SCENARIO_OUTLINE_EXAMPLE_TAGS_PARAMETER = "exampleTags";
         public const string SCENARIO_TAGS_VARIABLE_NAME = "tagsOfScenario";

Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs

@@ -1,5 +1,6 @@
 using System;
 using System.CodeDom;
+using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using System.Linq;
 using System.Reflection;
@@ -12,6 +13,7 @@
 
 namespace Reqnroll.Generator.Generation
 {
+    [SuppressMessage("ReSharper", "BitwiseOperatorOnEnumWithoutFlags")]
     public class UnitTestFeatureGenerator : IFeatureGenerator
     {
         private readonly CodeDomHelper _codeDomHelper;
@@ -149,11 +151,8 @@ private void SetupTestClass(TestClassGenerationContext generationContext)
                 _testGeneratorProvider.SetTestClassCategories(generationContext, featureCategories);
             }
 
-            var featureTagsField = new CodeMemberField(typeof(string[]), GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME);
-            featureTagsField.Attributes |= MemberAttributes.Static;
-            featureTagsField.InitExpression = _scenarioPartHelper.GetStringArrayExpression(generationContext.Feature.Tags);
-
-            generationContext.TestClass.Members.Add(featureTagsField);
+            DeclareFeatureTagsField(generationContext);
+            DeclareFeatureInfoMember(generationContext);
         }
 
         private CodeMemberField DeclareTestRunnerMember(CodeTypeDeclaration type)
@@ -163,6 +162,33 @@ private CodeMemberField DeclareTestRunnerMember(CodeTypeDeclaration type)
             return testRunnerField;
         }
 
+        private void DeclareFeatureTagsField(TestClassGenerationContext generationContext)
+        {
+            var featureTagsField = new CodeMemberField(typeof(string[]), GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME);
+            featureTagsField.Attributes |= MemberAttributes.Static;
+            featureTagsField.InitExpression = _scenarioPartHelper.GetStringArrayExpression(generationContext.Feature.Tags);
+            generationContext.TestClass.Members.Add(featureTagsField);
+        }
+
+        private void DeclareFeatureInfoMember(TestClassGenerationContext generationContext)
+        {
+            var featureInfoField = new CodeMemberField(
+                _codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), GeneratorConstants.FEATUREINFO_FIELD);
+            featureInfoField.Attributes |= MemberAttributes.Static;
+            featureInfoField.InitExpression = new CodeObjectCreateExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)),
+                new CodeObjectCreateExpression(typeof(CultureInfo),
+                                               new CodePrimitiveExpression(generationContext.Feature.Language)),
+                new CodePrimitiveExpression(generationContext.Document.DocumentLocation?.FeatureFolderPath),
+                new CodePrimitiveExpression(generationContext.Feature.Name),
+                new CodePrimitiveExpression(generationContext.Feature.Description),
+                new CodeFieldReferenceExpression(
+                    new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(ProgrammingLanguage))),
+                    _codeDomHelper.TargetLanguage.ToString()),
+                new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME));
+
+            generationContext.TestClass.Members.Add(featureInfoField);
+        }
+
         private void SetupTestClassInitializeMethod(TestClassGenerationContext generationContext)
         {
             var testClassInitializeMethod = generationContext.TestClassInitializeMethod;
@@ -173,42 +199,6 @@ private void SetupTestClassInitializeMethod(TestClassGenerationContext generatio
             _codeDomHelper.MarkCodeMemberMethodAsAsync(testClassInitializeMethod);
 
             _testGeneratorProvider.SetTestClassInitializeMethod(generationContext);
-
-            //testRunner = TestRunnerManager.GetTestRunnerForAssembly(null, [test_worker_id]);
-            var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression();
-
-            var getTestRunnerExpression = new CodeMethodInvokeExpression(
-                new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))),
-                nameof(TestRunnerManager.GetTestRunnerForAssembly));
-
-            testClassInitializeMethod.Statements.Add(
-                new CodeAssignStatement(
-                    testRunnerField,
-                    getTestRunnerExpression));
-
-            //FeatureInfo featureInfo = new FeatureInfo("xxxx");
-            testClassInitializeMethod.Statements.Add(
-                new CodeVariableDeclarationStatement(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), "featureInfo",
-                    new CodeObjectCreateExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)),
-                        new CodeObjectCreateExpression(typeof(CultureInfo),
-                            new CodePrimitiveExpression(generationContext.Feature.Language)),
-                        new CodePrimitiveExpression(generationContext.Document.DocumentLocation?.FeatureFolderPath),
-                        new CodePrimitiveExpression(generationContext.Feature.Name),
-                        new CodePrimitiveExpression(generationContext.Feature.Description),
-                        new CodeFieldReferenceExpression(
-                            new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(Reqnroll.ProgrammingLanguage))),
-                            _codeDomHelper.TargetLanguage.ToString()),
-                        new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME))));
-
-            //await testRunner.OnFeatureStartAsync(featureInfo);
-            var onFeatureStartExpression = new CodeMethodInvokeExpression(
-                testRunnerField,
-                nameof(ITestRunner.OnFeatureStartAsync),
-                new CodeVariableReferenceExpression("featureInfo"));
-
-            _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureStartExpression);
-
-            testClassInitializeMethod.Statements.Add(onFeatureStartExpression);
         }
 
         private void SetupTestClassCleanupMethod(TestClassGenerationContext generationContext)
@@ -221,30 +211,6 @@ private void SetupTestClassCleanupMethod(TestClassGenerationContext generationCo
             _codeDomHelper.MarkCodeMemberMethodAsAsync(testClassCleanupMethod);
 
             _testGeneratorProvider.SetTestClassCleanupMethod(generationContext);
-
-            var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression();
-
-            // await testRunner.OnFeatureEndAsync();
-            var expression = new CodeMethodInvokeExpression(
-                testRunnerField,
-                nameof(ITestRunner.OnFeatureEndAsync));
-
-            _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression);
-
-            testClassCleanupMethod.Statements.Add(expression);
-
-            // 
-            testClassCleanupMethod.Statements.Add(
-                new CodeMethodInvokeExpression(
-                    new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))),
-                    nameof(TestRunnerManager.ReleaseTestRunner),
-                    testRunnerField));
-
-            // testRunner = null;
-            testClassCleanupMethod.Statements.Add(
-                new CodeAssignStatement(
-                    testRunnerField,
-                    new CodePrimitiveExpression(null)));
         }
 
         private void SetupTestInitializeMethod(TestClassGenerationContext generationContext)
@@ -257,6 +223,78 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont
             _codeDomHelper.MarkCodeMemberMethodAsAsync(testInitializeMethod);
 
             _testGeneratorProvider.SetTestInitializeMethod(generationContext);
+
+            // Obtain the test runner for executing a single test
+            // testRunner = global::Reqnroll.TestRunnerManager.GetTestRunnerForAssembly(featureHint: featureInfo);
+
+            var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression();
+
+            var getTestRunnerExpression = new CodeMethodInvokeExpression(
+                new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))),
+                nameof(TestRunnerManager.GetTestRunnerForAssembly),
+                _codeDomHelper.CreateOptionalArgumentExpression("featureHint", 
+                    new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD)));
+
+            testInitializeMethod.Statements.Add(
+                new CodeAssignStatement(
+                    testRunnerField,
+                    getTestRunnerExpression));
+
+
+            // "Finish" current feature if needed
+
+            var featureContextExpression = new CodePropertyReferenceExpression(
+                testRunnerField,
+                "FeatureContext");
+
+            var onFeatureEndAsyncExpression = new CodeMethodInvokeExpression(
+                testRunnerField,
+                nameof(ITestRunner.OnFeatureEndAsync));
+            _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureEndAsyncExpression);
+
+            //if (testRunner.FeatureContext != null && !testRunner.FeatureContext.FeatureInfo.Equals(featureInfo))
+            //  await testRunner.OnFeatureEndAsync(); // finish if different
+            testInitializeMethod.Statements.Add(
+                new CodeConditionStatement(
+                    new CodeBinaryOperatorExpression(
+                        new CodeBinaryOperatorExpression(
+                            featureContextExpression,
+                            CodeBinaryOperatorType.IdentityInequality,
+                            new CodePrimitiveExpression(null)),
+                        CodeBinaryOperatorType.BooleanAnd,
+                        new CodeBinaryOperatorExpression(
+                            new CodeMethodInvokeExpression(
+                                new CodePropertyReferenceExpression(
+                                    featureContextExpression,
+                                    "FeatureInfo"),
+                                nameof(object.Equals),
+                                new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD)),
+                            CodeBinaryOperatorType.ValueEquality,
+                            new CodePrimitiveExpression(false))),
+                    new CodeExpressionStatement(
+                        onFeatureEndAsyncExpression)));
+
+
+            // "Start" the feature if needed
+
+            //if (testRunner.FeatureContext == null) {
+            //  await testRunner.OnFeatureStartAsync(featureInfo);
+            //}
+
+            var onFeatureStartExpression = new CodeMethodInvokeExpression(
+                testRunnerField,
+                nameof(ITestRunner.OnFeatureStartAsync),
+                new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD));
+            _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureStartExpression);
+
+            testInitializeMethod.Statements.Add(
+                new CodeConditionStatement(
+                    new CodeBinaryOperatorExpression(
+                        featureContextExpression,
+                        CodeBinaryOperatorType.IdentityEquality,
+                        new CodePrimitiveExpression(null)),
+                    new CodeExpressionStatement(
+                        onFeatureStartExpression)));
         }
 
         private void SetupTestCleanupMethod(TestClassGenerationContext generationContext)
@@ -280,6 +318,14 @@ private void SetupTestCleanupMethod(TestClassGenerationContext generationContext
             _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression);
 
             testCleanupMethod.Statements.Add(expression);
+
+            // "Release" the TestRunner, so that other threads can pick it up
+            // TestRunnerManager.ReleaseTestRunner(testRunner);
+            testCleanupMethod.Statements.Add(
+                new CodeMethodInvokeExpression(
+                    new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))),
+                    nameof(TestRunnerManager.ReleaseTestRunner),
+                    testRunnerField));
         }
 
         private void SetupScenarioInitializeMethod(TestClassGenerationContext generationContext)