Refactor the test benchmark app to make the example easier to follow (f…

…lutter#7640)

A breaking change to this package will follow that changes the structure of setting the initial path for the benchmark run. This was originally done in flutter#7632, which was closed because there were more changes needed to get the tests to pass. This cleanup is part of that work.

This PR
- moves the benchmark clients to `test_app/benchmark` following pub package guidance for naming conventions
- regenerates the test_app web assets using `flutter create` to ensure the web configuration is up to date
- adds more structure to the benchmark test_infra setup to provide a more thorough example of what a user of this package might need to do. I followed the structure that is used in DevTools benchmark tests.

packages/web_benchmarks/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.1.0-wip
+
+* Restructure the `testing/test_app` to make the example benchmarks easier to follow.
+
 ## 2.0.2
 
 * Updates minimum supported SDK version to Flutter 3.19/Dart 3.3.

packages/web_benchmarks/README.md

@@ -5,14 +5,14 @@ benchmarks in Chrome.
 
 # Writing a benchmark
 
-An example benchmark can be found in [testing/web_benchmark_test.dart][1].
+An example benchmark can be found in [testing/test_app/benchmark/web_benchmark_test.dart][1].
 
 A web benchmark is made of two parts: a client and a server. The client is code
 that runs in the browser together with the benchmark code. The server serves the
 app's code and assets. Additionally, the server communicates with the browser to
 extract the performance traces.
 
-[1]: https://github.com/flutter/packages/blob/master/packages/web_benchmarks/testing/web_benchmarks_test.dart
+[1]: https://github.com/flutter/packages/blob/master/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart
 
 # Analyzing benchmark results
 

packages/web_benchmarks/lib/server.dart

@@ -32,8 +32,6 @@ const int defaultChromeDebugPort = 10000;
 /// can be different (and typically is) from the production entry point of the
 /// app.
 ///
-/// If [useCanvasKit] is true, builds the app in CanvasKit mode.
-///
 /// [benchmarkServerPort] is the port this benchmark server serves the app on.
 /// By default uses [defaultBenchmarkServerPort].
 ///
@@ -42,6 +40,13 @@ const int defaultChromeDebugPort = 10000;
 ///
 /// If [headless] is true, runs Chrome without UI. In particular, this is
 /// useful in environments (e.g. CI) that doesn't have a display.
+///
+/// If [treeShakeIcons] is false, '--no-tree-shake-icons' will be passed as a
+/// build argument when building the benchmark app.
+///
+/// [compilationOptions] specify the compiler and renderer to use for the
+/// benchmark app. This can either use dart2wasm & skwasm or
+/// dart2js & canvaskit.
 Future<BenchmarkResults> serveWebBenchmark({
   required io.Directory benchmarkAppDirectory,
   required String entryPoint,

packages/web_benchmarks/lib/src/runner.dart

@@ -39,15 +39,20 @@ class BenchmarkServer {
   /// can be different (and typically is) from the production entry point of the
   /// app.
   ///
-  /// If [useCanvasKit] is true, builds the app in CanvasKit mode.
-  ///
   /// [benchmarkServerPort] is the port this benchmark server serves the app on.
   ///
   /// [chromeDebugPort] is the port Chrome uses for DevTool Protocol used to
   /// extract tracing data.
   ///
   /// If [headless] is true, runs Chrome without UI. In particular, this is
   /// useful in environments (e.g. CI) that doesn't have a display.
+  ///
+  /// If [treeShakeIcons] is false, '--no-tree-shake-icons' will be passed as a
+  /// build argument when building the benchmark app.
+  ///
+  /// [compilationOptions] specify the compiler and renderer to use for the
+  /// benchmark app. This can either use dart2wasm & skwasm or
+  /// dart2js & canvaskit.
   BenchmarkServer({
     required this.benchmarkAppDirectory,
     required this.entryPoint,

packages/web_benchmarks/pubspec.yaml

@@ -2,7 +2,7 @@ name: web_benchmarks
 description: A benchmark harness for performance-testing Flutter apps in Chrome.
 repository: https://github.com/flutter/packages/tree/main/packages/web_benchmarks
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+web_benchmarks%22
-version: 2.0.2
+version: 2.1.0-wip
 
 environment:
   sdk: ^3.3.0

packages/web_benchmarks/testing/test_app/.gitignore

@@ -5,9 +5,12 @@
 *.swp
 .DS_Store
 .atom/
+.build/
 .buildlog/
 .history
 .svn/
+.swiftpm/
+migrate_working_dir/
 
 # IntelliJ related
 *.iml
@@ -26,7 +29,6 @@
 .dart_tool/
 .flutter-plugins
 .flutter-plugins-dependencies
-.packages
 .pub-cache/
 .pub/
 /build/
@@ -36,3 +38,8 @@ app.*.symbols
 
 # Obfuscation related
 app.*.map.json
+
+# Android Studio will place build artifacts here
+/android/app/debug
+/android/app/profile
+/android/app/release

packages/web_benchmarks/testing/test_app/.metadata

@@ -4,7 +4,27 @@
 # This file should be version controlled and should not be manually edited.
 
 version:
-  revision: d26268bb9e6d713a73d6148da7fa75936d442741
-  channel: master
+  revision: "0cd170798c6462aec738d4c749ce3a5fff1c80cf"
+  channel: "master"
 
 project_type: app
+
+# Tracks metadata for the flutter migrate command
+migration:
+  platforms:
+    - platform: root
+      create_revision: 0cd170798c6462aec738d4c749ce3a5fff1c80cf
+      base_revision: 0cd170798c6462aec738d4c749ce3a5fff1c80cf
+    - platform: web
+      create_revision: 0cd170798c6462aec738d4c749ce3a5fff1c80cf
+      base_revision: 0cd170798c6462aec738d4c749ce3a5fff1c80cf
+
+  # User provided section
+
+  # List of Local paths (relative to this file) that should be
+  # ignored by the migrate tool.
+  #
+  # Files that are not part of the templates will be ignored by default.
+  unmanaged_files:
+    - 'lib/main.dart'
+    - 'ios/Runner.xcodeproj/project.pbxproj'

packages/web_benchmarks/testing/test_app/benchmark/test_infra/automator.dart

@@ -0,0 +1,146 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// ignore_for_file: avoid_print
+
+import 'dart:async';
+
+import 'package:flutter/foundation.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:test_app/home_page.dart' show aboutPageKey, textKey;
+import 'package:test_app/main.dart';
+import 'package:web/web.dart';
+import 'package:web_benchmarks/client.dart';
+
+import 'common.dart';
+
+/// A class that automates the test web app.
+class Automator {
+  Automator({
+    required this.benchmark,
+    required this.stopWarmingUpCallback,
+    required this.profile,
+  });
+
+  /// The current benchmark.
+  final BenchmarkName benchmark;
+
+  /// A function to call when warm-up is finished.
+  ///
+  /// This function is intended to ask `Recorder` to mark the warm-up phase
+  /// as over.
+  final void Function() stopWarmingUpCallback;
+
+  /// The profile collected for the running benchmark
+  final Profile profile;
+
+  /// Whether the automation has ended.
+  bool finished = false;
+
+  /// A widget controller for automation.
+  late LiveWidgetController controller;
+
+  Widget createWidget() {
+    Future<void>.delayed(const Duration(milliseconds: 400), automate);
+    return const MyApp();
+  }
+
+  Future<void> automate() async {
+    await warmUp();
+
+    switch (benchmark) {
+      case BenchmarkName.appNavigate:
+        await _handleAppNavigate();
+      case BenchmarkName.appScroll:
+        await _handleAppScroll();
+      case BenchmarkName.appTap:
+        await _handleAppTap();
+      case BenchmarkName.simpleCompilationCheck:
+        _handleSimpleCompilationCheck();
+      case BenchmarkName.simpleInitialPageCheck:
+        _handleSimpleInitialPageCheck();
+    }
+
+    // At the end of the test, mark as finished.
+    finished = true;
+  }
+
+  /// Warm up the animation.
+  Future<void> warmUp() async {
+    // Let animation stop.
+    await animationStops();
+
+    // Set controller.
+    controller = LiveWidgetController(WidgetsBinding.instance);
+
+    await controller.pumpAndSettle();
+
+    // When warm-up finishes, inform the recorder.
+    stopWarmingUpCallback();
+  }
+
+  Future<void> _handleAppNavigate() async {
+    for (int i = 0; i < 10; ++i) {
+      print('Testing round $i...');
+      await controller.tap(find.byKey(aboutPageKey));
+      await animationStops();
+      await controller.tap(find.byType(BackButton));
+      await animationStops();
+    }
+  }
+
+  Future<void> _handleAppScroll() async {
+    final ScrollableState scrollable =
+        Scrollable.of(find.byKey(textKey).evaluate().single);
+    await scrollable.position.animateTo(
+      30000,
+      curve: Curves.linear,
+      duration: const Duration(seconds: 20),
+    );
+  }
+
+  Future<void> _handleAppTap() async {
+    for (int i = 0; i < 10; ++i) {
+      print('Testing round $i...');
+      await controller.tap(find.byIcon(Icons.add));
+      await animationStops();
+    }
+  }
+
+  void _handleSimpleCompilationCheck() {
+    // Record whether we are in wasm mode or not. Ideally, we'd have a more
+    // first-class way to add metadata like this, but this will work for us to
+    // pass information about the environment back to the server for the
+    // purposes of our own tests.
+    profile.extraData['isWasm'] = kIsWasm ? 1 : 0;
+  }
+
+  void _handleSimpleInitialPageCheck() {
+    // Record whether the URL contains the expected initial page so we can
+    // verify the behavior of setting the `initialPage` on the benchmark server.
+    final bool containsExpectedPage =
+        window.location.toString().contains(testBenchmarkInitialPage);
+    profile.extraData['expectedUrl'] = containsExpectedPage ? 1 : 0;
+  }
+}
+
+const Duration _animationCheckingInterval = Duration(milliseconds: 50);
+
+Future<void> animationStops() async {
+  if (!WidgetsBinding.instance.hasScheduledFrame) {
+    return;
+  }
+
+  final Completer<void> stopped = Completer<void>();
+
+  Timer.periodic(_animationCheckingInterval, (Timer timer) {
+    if (!WidgetsBinding.instance.hasScheduledFrame) {
+      stopped.complete();
+      timer.cancel();
+    }
+  });
+
+  await stopped.future;
+}

packages/web_benchmarks/testing/test_app/benchmark/test_infra/client/app_client.dart

@@ -0,0 +1,21 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:web_benchmarks/client.dart';
+
+import '../common.dart';
+import '../recorder.dart';
+
+Future<void> main() async {
+  await runBenchmarks(
+    <String, RecorderFactory>{
+      BenchmarkName.appNavigate.name: () =>
+          TestAppRecorder(benchmark: BenchmarkName.appNavigate),
+      BenchmarkName.appScroll.name: () =>
+          TestAppRecorder(benchmark: BenchmarkName.appScroll),
+      BenchmarkName.appTap.name: () =>
+          TestAppRecorder(benchmark: BenchmarkName.appTap),
+    },
+  );
+}

packages/web_benchmarks/testing/test_app/benchmark/test_infra/client/simple_compilation_client.dart

@@ -0,0 +1,18 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:web_benchmarks/client.dart';
+
+import '../common.dart';
+import '../recorder.dart';
+
+Future<void> main() async {
+  await runBenchmarks(
+    <String, RecorderFactory>{
+      BenchmarkName.simpleCompilationCheck.name: () => TestAppRecorder(
+            benchmark: BenchmarkName.simpleCompilationCheck,
+          ),
+    },
+  );
+}

packages/web_benchmarks/testing/test_app/benchmark/test_infra/client/simple_initial_page_client.dart

@@ -0,0 +1,18 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:web_benchmarks/client.dart';
+
+import '../common.dart';
+import '../recorder.dart';
+
+Future<void> main() async {
+  await runBenchmarks(
+    <String, RecorderFactory>{
+      BenchmarkName.simpleInitialPageCheck.name: () => TestAppRecorder(
+            benchmark: BenchmarkName.simpleInitialPageCheck,
+          ),
+    },
+  );
+}

packages/web_benchmarks/testing/test_app/benchmark/test_infra/common.dart

@@ -0,0 +1,13 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+const String testBenchmarkInitialPage = 'index.html#about';
+
+enum BenchmarkName {
+  appNavigate,
+  appScroll,
+  appTap,
+  simpleInitialPageCheck,
+  simpleCompilationCheck;
+}

packages/web_benchmarks/testing/test_app/benchmark/test_infra/recorder.dart

@@ -0,0 +1,38 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter/material.dart';
+import 'package:web_benchmarks/client.dart';
+
+import 'automator.dart';
+import 'common.dart';
+
+/// A recorder that measures frame building durations for the test app.
+class TestAppRecorder extends WidgetRecorder {
+  TestAppRecorder({required this.benchmark})
+      : super(name: benchmark.name, useCustomWarmUp: true);
+
+  /// The name of the benchmark to be run.
+  ///
+  /// See `common.dart` for the list of the names of all benchmarks.
+  final BenchmarkName benchmark;
+
+  Automator? _automator;
+  bool get _finished => _automator?.finished ?? false;
+
+  /// Whether we should continue recording.
+  @override
+  bool shouldContinue() => !_finished || profile.shouldContinue();
+
+  /// Creates the [Automator] widget.
+  @override
+  Widget createWidget() {
+    _automator = Automator(
+      benchmark: benchmark,
+      stopWarmingUpCallback: profile.stopWarmingUp,
+      profile: profile,
+    );
+    return _automator!.createWidget();
+  }
+}