From 920059394f3fa154d5ffecd6f5350b01ea1860d9 Mon Sep 17 00:00:00 2001 From: Chris Holt Date: Wed, 8 Nov 2017 15:36:54 -0600 Subject: [PATCH] Improvements for load tester #390 --- nuget/canopy.integration.nuspec | 2 +- nuget/canopy.nuspec | 2 +- paket.dependencies | 2 +- paket.lock | 2 +- src/canopy.integration/loadTest.fs | 53 ++++++++++++++++++++---------- 5 files changed, 40 insertions(+), 21 deletions(-) diff --git a/nuget/canopy.integration.nuspec b/nuget/canopy.integration.nuspec index 257e4193..acf8f6c7 100644 --- a/nuget/canopy.integration.nuspec +++ b/nuget/canopy.integration.nuspec @@ -12,7 +12,7 @@ @releaseNotes@ @tags@ - + diff --git a/nuget/canopy.nuspec b/nuget/canopy.nuspec index 76085a90..f72ed2d4 100644 --- a/nuget/canopy.nuspec +++ b/nuget/canopy.nuspec @@ -12,7 +12,7 @@ @releaseNotes@ @tags@ - + diff --git a/paket.dependencies b/paket.dependencies index 99564968..c4b9cdd6 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -3,7 +3,7 @@ source https://www.nuget.org/api/v2/ framework: net452 nuget NuGet.CommandLine -nuget FSharp.Core >= 3.0.2 lowest_matching:true +nuget FSharp.Core >= 4.0.0.1 lowest_matching:true nuget Selenium.WebDriver nuget FAKE nuget FSharp.Formatting diff --git a/paket.lock b/paket.lock index 707a2172..a22de5cd 100644 --- a/paket.lock +++ b/paket.lock @@ -3,7 +3,7 @@ NUGET remote: https://www.nuget.org/api/v2 FAKE (4.63.2) FSharp.Compiler.Service (2.0.0.6) - FSharp.Core (3.0.2) + FSharp.Core (4.0.0.1) FSharp.Data (2.4.2) FSharp.Formatting (2.14.4) FSharp.Compiler.Service (2.0.0.6) diff --git a/src/canopy.integration/loadTest.fs b/src/canopy.integration/loadTest.fs index ec52ff3a..7f6aca96 100644 --- a/src/canopy.integration/loadTest.fs +++ b/src/canopy.integration/loadTest.fs @@ -33,7 +33,7 @@ type Job = type private Result = { - Description : string + Task : Task Min : float Max : float Average: float @@ -48,7 +48,7 @@ type private Worker = | Retire type private Reporter = - | WorkerDone of description:string * timing:float + | WorkerDone of Task * timing:float | Retire of AsyncReplyChannel type private Manager = @@ -65,7 +65,7 @@ let private time f = //worker ctor //workers just run the action and send the timing informatoin to the reporter -let private newWorker (manager : actor) (reporter : actor) description action : actor = +let private newWorker (manager : actor) (reporter : actor) task : actor = actor.Start(fun self -> let rec loop () = async { @@ -74,8 +74,8 @@ let private newWorker (manager : actor) (reporter : actor) de | Worker.Retire -> return () | Worker.Do -> - let timing = time action - reporter.Post(Reporter.WorkerDone(description, timing)) + let timing = time task.Action + reporter.Post(Reporter.WorkerDone(task, timing)) manager.Post(Manager.WorkerDone) return! loop () } @@ -84,7 +84,7 @@ let private newWorker (manager : actor) (reporter : actor) de //reporter ctor //reporter recieves timing information about tasks from workers and aggregates it let private newReporter () : actor = - let mutable results : (string * float) list = [] + let mutable results : (Task * float) list = [] actor.Start(fun self -> let rec loop () = async { @@ -93,11 +93,12 @@ let private newReporter () : actor = | Reporter.Retire replyChannel -> let finalResults = results - |> Seq.groupBy (fun (description, timing) -> description) - |> Seq.map (fun (description, pairs) -> description, pairs |> Seq.map (fun (_, timings) -> timings)) - |> Seq.map (fun (description, timings) -> + |> Seq.groupBy (fun (task, timing) -> task.Description) + |> Seq.sortBy (fun (description, _) -> description) + |> Seq.map (fun (description, pairs) -> description, pairs |> Seq.head |> fst, pairs |> Seq.map (fun (_, timings) -> timings)) + |> Seq.map (fun (description, task, timings) -> { - Description = description + Task = task Min = Seq.min timings Max = Seq.max timings Average = Seq.average timings @@ -154,7 +155,7 @@ let private runTasksOnce job = manager.Post(Initialize(job.Tasks.Length)) job.Tasks - |> List.map (fun task -> newWorker manager reporter task.Description task.Action) + |> List.map (fun task -> newWorker manager reporter task) |> List.iter (fun worker -> worker.Post(Do); worker.Post(Worker.Retire)) manager.PostAndReply(fun replyChannel -> Manager.Retire replyChannel) |> ignore @@ -195,12 +196,28 @@ let rec private iterateWorkers timingsAndWorkers (sw : System.Diagnostics.Stopwa System.Threading.Thread.Sleep(1) iterateWorkers timingsAndWorkers sw -let private printResults results = +let private printJob job = printfn "Job: (load %A) (minutes %A) (acceptableRatioPercent %A) (warmup %A) (baseline %A)" job.Load job.Minutes job.AcceptableRatioPercent job.Warmup job.Baseline + +let private printBaseline baselineResults = + printfn "" + if baselineResults |> List.length = 0 then printfn "No Baseline" + else + printfn "Baseline ms" + printfn "--------------------------------------------------------------------------------" + baselineResults + |> List.iter (fun result -> + let description = result.Task.Description.PadRight(70, ' ') + let value = (sprintf "%.1f" result.Average).PadLeft(10, ' ') + printfn "%s%s" description value) + +let private printResults results load = + printfn "" printfn "Task MIN ms MAX ms AVG ms" printfn "--------------------------------------------------------------------------------" results |> List.iter (fun result -> - let description = result.Description.PadRight(50, ' ') + let temp = (sprintf "%s x%i" result.Task.Description (result.Task.Frequency * load)) + let description = temp.Substring(0, System.Math.Min(temp.Length, 50)).PadRight(50, ' ') let min = (sprintf "%.1f" result.Min).PadLeft(10, ' ') let max = (sprintf "%.1f" result.Max).PadLeft(10, ' ') let avg = (sprintf "%.1f" result.Average).PadLeft(10, ' ') @@ -210,10 +227,10 @@ let private runBaseline job baselineResults results = if job.Baseline = true then results |> List.map (fun result -> - let baselineResult = baselineResults |> List.find(fun baselineResult -> result.Description = baselineResult.Description) + let baselineResult = baselineResults |> List.find(fun baselineResult -> result.Task.Description = baselineResult.Task.Description) let threshold = baselineResult.Average * (float job.AcceptableRatioPercent / 100.0) if result.Average > threshold then - Some (sprintf "FAILED: Average of %.1f exceeded threshold of %.1f for %s" result.Average threshold result.Description) + Some (sprintf "FAILED: Average of %.1f exceeded threshold of %.1f for %s" result.Average threshold result.Task.Description) else None) |> List.choose id else [] @@ -236,7 +253,7 @@ let runLoadTest job = if job.Baseline = true then baselineResults <- baseline job //create all the workers and create the time they should execute - let timingsAndWorkers = createTimeline job |> List.map (fun (timing, task) -> timing, newWorker manager reporter task.Description task.Action) + let timingsAndWorkers = createTimeline job |> List.map (fun (timing, task) -> timing, newWorker manager reporter task) manager.Post(Initialize(timingsAndWorkers.Length)) //loop and look at head and see if its time has passed and if it has then @@ -245,6 +262,8 @@ let runLoadTest job = manager.PostAndReply(fun replyChannel -> Manager.Retire replyChannel) |> ignore let results = reporter.PostAndReply(fun replyChannel -> Reporter.Retire replyChannel) - printResults results + printJob job + printBaseline baselineResults + printResults results job.Load runBaseline job baselineResults results |> failIfFailure