1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5/// @docImport 'package:flutter_driver/flutter_driver.dart';
6///
7/// @docImport 'integration_test_driver_extended.dart';
8library;
9
10import 'dart:async';
11import 'dart:developer' as developer;
12import 'dart:io' show HttpClient, SocketException, WebSocket;
13import 'dart:ui';
14
15import 'package:flutter/foundation.dart';
16import 'package:flutter/rendering.dart';
17import 'package:flutter/services.dart';
18import 'package:flutter/widgets.dart';
19import 'package:flutter_test/flutter_test.dart';
20import 'package:vm_service/vm_service.dart' as vm;
21
22import 'common.dart';
23import 'src/callback.dart' as driver_actions;
24import 'src/channel.dart';
25import 'src/extension.dart';
26
27export 'src/vm_service_golden_client.dart';
28
29const String _success = 'success';
30
31/// Whether results should be reported to the native side over the method
32/// channel.
33///
34/// This is enabled by default for use by native test frameworks like Android
35/// instrumentation or XCTest. When running with the Flutter Tool through
36/// `flutter test integration_test` though, it will be disabled as the Flutter
37/// tool will be responsible for collection of test results.
38const bool _shouldReportResultsToNative = bool.fromEnvironment(
39 'INTEGRATION_TEST_SHOULD_REPORT_RESULTS_TO_NATIVE',
40 defaultValue: true,
41);
42
43/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
44/// on a channel to adapt them to native instrumentation test format.
45class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding
46 implements IntegrationTestResults {
47 /// Sets up a listener to report that the tests are finished when everything is
48 /// torn down.
49 IntegrationTestWidgetsFlutterBinding() {
50 tearDownAll(() async {
51 if (!_allTestsPassed.isCompleted) {
52 _allTestsPassed.complete(failureMethodsDetails.isEmpty);
53 }
54 callbackManager.cleanup();
55
56 // TODO(jiahaog): Print the message directing users to run with
57 // `flutter test` when Web is supported.
58 if (!_shouldReportResultsToNative || kIsWeb) {
59 return;
60 }
61
62 try {
63 await integrationTestChannel.invokeMethod<void>('allTestsFinished', <String, dynamic>{
64 'results': results.map<String, dynamic>((String name, Object result) {
65 if (result is Failure) {
66 return MapEntry<String, dynamic>(name, result.details);
67 }
68 return MapEntry<String, Object>(name, result);
69 }),
70 });
71 } on MissingPluginException {
72 debugPrint(r'''
73Warning: integration_test plugin was not detected.
74
75If you're running the tests with `flutter drive`, please make sure your tests
76are in the `integration_test/` directory of your package and use
77`flutter test $path_to_test` to run it instead.
78
79If you're running the tests with Android instrumentation or XCTest, this means
80that you are not capturing test results properly! See the following link for
81how to set up the integration_test plugin:
82
83https://docs.flutter.dev/testing/integration-tests
84''');
85 }
86 });
87
88 final TestExceptionReporter oldTestExceptionReporter = reportTestException;
89 reportTestException = (FlutterErrorDetails details, String testDescription) {
90 results[testDescription] = Failure(testDescription, details.toString());
91 oldTestExceptionReporter(details, testDescription);
92 };
93 }
94
95 @override
96 bool get overrideHttpClient => false;
97
98 @override
99 bool get registerTestTextInput => false;
100
101 Size? _surfaceSize;
102
103 // This flag is used to print warning messages when tracking performance
104 // under debug mode.
105 static bool _firstRun = false;
106
107 @override
108 Future<void> setSurfaceSize(Size? size) {
109 return TestAsyncUtils.guard<void>(() async {
110 assert(inTest);
111 if (_surfaceSize == size) {
112 return;
113 }
114 _surfaceSize = size;
115 handleMetricsChanged();
116 });
117 }
118
119 @override
120 ViewConfiguration createViewConfigurationFor(RenderView renderView) {
121 final FlutterView view = renderView.flutterView;
122 final Size? surfaceSize = view == platformDispatcher.implicitView ? _surfaceSize : null;
123 return TestViewConfiguration.fromView(
124 size: surfaceSize ?? view.physicalSize / view.devicePixelRatio,
125 view: view,
126 );
127 }
128
129 @override
130 Completer<bool> get allTestsPassed => _allTestsPassed;
131 final Completer<bool> _allTestsPassed = Completer<bool>();
132
133 @override
134 List<Failure> get failureMethodsDetails => results.values.whereType<Failure>().toList();
135
136 @override
137 void initInstances() {
138 super.initInstances();
139 _instance = this;
140 }
141
142 /// The singleton instance of this object.
143 ///
144 /// Provides access to the features exposed by this class. The binding must
145 /// be initialized before using this getter; this is typically done by calling
146 /// [IntegrationTestWidgetsFlutterBinding.ensureInitialized].
147 static IntegrationTestWidgetsFlutterBinding get instance => BindingBase.checkInstance(_instance);
148 static IntegrationTestWidgetsFlutterBinding? _instance;
149
150 /// Returns an instance of the [IntegrationTestWidgetsFlutterBinding], creating and
151 /// initializing it if necessary.
152 ///
153 /// See also:
154 ///
155 /// * [WidgetsFlutterBinding.ensureInitialized], the equivalent in the widgets framework.
156 static IntegrationTestWidgetsFlutterBinding ensureInitialized() {
157 if (_instance == null) {
158 IntegrationTestWidgetsFlutterBinding();
159 }
160 return _instance!;
161 }
162
163 /// Test results that will be populated after the tests have completed.
164 ///
165 /// Keys are the test descriptions, and values are either [_success] or
166 /// a [Failure].
167 @visibleForTesting
168 Map<String, Object> results = <String, Object>{};
169
170 /// The extra data for the reported result.
171 ///
172 /// The values in `reportData` must be json-serializable objects or `null`.
173 /// If it's `null`, no extra data is attached to the result.
174 ///
175 /// The default value is `null`.
176 @override
177 Map<String, dynamic>? reportData;
178
179 /// Manages callbacks received from driver side and commands send to driver
180 /// side.
181 final CallbackManager callbackManager = driver_actions.callbackManager;
182
183 /// Takes a screenshot.
184 ///
185 /// On Android, you need to call `convertFlutterSurfaceToImage()`, and
186 /// pump a frame before taking a screenshot.
187 Future<List<int>> takeScreenshot(String screenshotName, [Map<String, Object?>? args]) async {
188 reportData ??= <String, dynamic>{};
189 reportData!['screenshots'] ??= <dynamic>[];
190 final Map<String, dynamic> data = await callbackManager.takeScreenshot(screenshotName, args);
191 assert(data.containsKey('bytes'));
192
193 (reportData!['screenshots']! as List<dynamic>).add(data);
194 return data['bytes']! as List<int>;
195 }
196
197 /// Android only. Converts the Flutter surface to an image view.
198 /// Be aware that if you are conducting a perf test, you may not want to call
199 /// this method since the this is an expensive operation that affects the
200 /// rendering of a Flutter app.
201 ///
202 /// Once the screenshot is taken, call `revertFlutterImage()` to restore
203 /// the original Flutter surface.
204 Future<void> convertFlutterSurfaceToImage() async {
205 await callbackManager.convertFlutterSurfaceToImage();
206 }
207
208 /// The callback function to response the driver side input.
209 @visibleForTesting
210 Future<Map<String, dynamic>> callback(Map<String, String> params) async {
211 return callbackManager.callback(params, this /* as IntegrationTestResults */);
212 }
213
214 // Emulates the Flutter driver extension, returning 'pass' or 'fail'.
215 @override
216 void initServiceExtensions() {
217 super.initServiceExtensions();
218
219 if (kIsWeb) {
220 registerWebServiceExtension(callback);
221 }
222
223 registerServiceExtension(name: 'driver', callback: callback);
224 }
225
226 @override
227 Future<void> runTest(
228 Future<void> Function() testBody,
229 VoidCallback invariantTester, {
230 String description = '',
231 @Deprecated(
232 'This parameter has no effect. Use the `timeout` parameter on `testWidgets` instead. '
233 'This feature was deprecated after v2.6.0-1.0.pre.',
234 )
235 Duration? timeout,
236 }) async {
237 await super.runTest(testBody, invariantTester, description: description);
238 results[description] ??= _success;
239 }
240
241 // Do not paint a description label because it could show up in screenshots
242 // of the integration test.
243 @override
244 void setLabel(String value) {}
245
246 vm.VmService? _vmService;
247
248 /// Initialize the [vm.VmService] settings for the timeline.
249 @visibleForTesting
250 Future<void> enableTimeline({
251 List<String> streams = const <String>['all'],
252 @visibleForTesting vm.VmService? vmService,
253 @visibleForTesting HttpClient? httpClient,
254 }) async {
255 assert(streams.isNotEmpty);
256 if (vmService != null) {
257 _vmService = vmService;
258 }
259 if (_vmService == null) {
260 final developer.ServiceProtocolInfo info = await developer.Service.getInfo();
261 assert(info.serverUri != null);
262 final String address = 'ws://localhost:${info.serverUri!.port}${info.serverUri!.path}ws';
263 try {
264 _vmService = await _vmServiceConnectUri(address, httpClient: httpClient);
265 } on SocketException catch (e, s) {
266 throw StateError(
267 'Failed to connect to VM Service at $address.\n'
268 'This may happen if DDS is enabled. If this test was launched via '
269 '`flutter drive`, try adding `--no-dds`.\n'
270 'The original exception was:\n'
271 '$e\n$s',
272 );
273 }
274 }
275 await _vmService!.setVMTimelineFlags(streams);
276 }
277
278 /// Runs [action] and returns a [vm.Timeline] trace for it.
279 ///
280 /// Waits for the `Future` returned by [action] to complete prior to stopping
281 /// the trace.
282 ///
283 /// The `streams` parameter limits the recorded timeline event streams to only
284 /// the ones listed. By default, all streams are recorded.
285 /// See `timeline_streams` in
286 /// [Dart-SDK/runtime/vm/timeline.cc](https://github.com/dart-lang/sdk/blob/main/runtime/vm/timeline.cc)
287 ///
288 /// If [retainPriorEvents] is true, retains events recorded prior to calling
289 /// [action]. Otherwise, prior events are cleared before calling [action]. By
290 /// default, prior events are cleared.
291 Future<vm.Timeline> traceTimeline(
292 Future<dynamic> Function() action, {
293 List<String> streams = const <String>['all'],
294 bool retainPriorEvents = false,
295 }) async {
296 await enableTimeline(streams: streams);
297 if (retainPriorEvents) {
298 await action();
299 return _vmService!.getVMTimeline();
300 }
301
302 await _vmService!.clearVMTimeline();
303 final vm.Timestamp startTime = await _vmService!.getVMTimelineMicros();
304 await action();
305 final vm.Timestamp endTime = await _vmService!.getVMTimelineMicros();
306 return _vmService!.getVMTimeline(
307 timeOriginMicros: startTime.timestamp,
308 timeExtentMicros: endTime.timestamp,
309 );
310 }
311
312 /// This is a convenience method that calls [traceTimeline] and sends the
313 /// result back to the host for the [flutter_driver] style tests.
314 ///
315 /// This records the timeline during `action` and adds the result to
316 /// [reportData] with `reportKey`. The [reportData] contains extra information
317 /// from the test other than test success/fail. It will be passed back to the
318 /// host and be processed by the [ResponseDataCallback] defined in
319 /// [integrationDriver]. By default it will be written to
320 /// `build/integration_response_data.json` with the key `timeline`.
321 ///
322 /// For tests with multiple calls of this method, `reportKey` needs to be a
323 /// unique key, otherwise the later result will override earlier one. Tests
324 /// that call this multiple times must also provide a custom
325 /// [ResponseDataCallback] to decide where and how to write the output
326 /// timelines. For example,
327 ///
328 /// ```dart
329 /// import 'package:integration_test/integration_test_driver.dart';
330 ///
331 /// Future<void> main() {
332 /// return integrationDriver(
333 /// responseDataCallback: (Map<String, dynamic>? data) async {
334 /// if (data != null) {
335 /// for (final MapEntry<String, dynamic> entry in data.entries) {
336 /// print('Writing ${entry.key} to the disk.');
337 /// await writeResponseData(
338 /// entry.value as Map<String, dynamic>,
339 /// testOutputFilename: entry.key,
340 /// );
341 /// }
342 /// }
343 /// },
344 /// );
345 /// }
346 /// ```
347 ///
348 /// The `streams` and `retainPriorEvents` parameters are passed as-is to
349 /// [traceTimeline].
350 Future<void> traceAction(
351 Future<dynamic> Function() action, {
352 List<String> streams = const <String>['all'],
353 bool retainPriorEvents = false,
354 String reportKey = 'timeline',
355 }) async {
356 final vm.Timeline timeline = await traceTimeline(
357 action,
358 streams: streams,
359 retainPriorEvents: retainPriorEvents,
360 );
361 reportData ??= <String, dynamic>{};
362 reportData![reportKey] = timeline.toJson();
363 }
364
365 Future<_GarbageCollectionInfo> _runAndGetGCInfo(Future<void> Function() action) async {
366 if (kIsWeb) {
367 await action();
368 return const _GarbageCollectionInfo();
369 }
370
371 final vm.Timeline timeline = await traceTimeline(action, streams: <String>['GC']);
372
373 final int oldGenGCCount = timeline.traceEvents!.where((vm.TimelineEvent event) {
374 return event.json!['cat'] == 'GC' && event.json!['name'] == 'CollectOldGeneration';
375 }).length;
376 final int newGenGCCount = timeline.traceEvents!.where((vm.TimelineEvent event) {
377 return event.json!['cat'] == 'GC' && event.json!['name'] == 'CollectNewGeneration';
378 }).length;
379 return _GarbageCollectionInfo(oldCount: oldGenGCCount, newCount: newGenGCCount);
380 }
381
382 /// Watches the [FrameTiming] during `action` and report it to the binding
383 /// with key `reportKey`.
384 ///
385 /// This can be used to implement performance tests previously using
386 /// [traceAction] and [TimelineSummary] from [flutter_driver]
387 Future<void> watchPerformance(
388 Future<void> Function() action, {
389 String reportKey = 'performance',
390 }) async {
391 assert(() {
392 if (_firstRun) {
393 debugPrint(kDebugWarning);
394 _firstRun = false;
395 }
396 return true;
397 }());
398
399 // The engine could batch FrameTimings and send them only once per second.
400 // Delay for a sufficient time so either old FrameTimings are flushed and not
401 // interfering our measurements here, or new FrameTimings are all reported.
402 // TODO(CareF): remove this when flush FrameTiming is readily in engine.
403 // See https://github.com/flutter/flutter/issues/64808
404 // and https://github.com/flutter/flutter/issues/67593
405 final List<FrameTiming> frameTimings = <FrameTiming>[];
406 Future<void> delayForFrameTimings() async {
407 int count = 0;
408 while (frameTimings.isEmpty) {
409 count++;
410 await Future<void>.delayed(const Duration(seconds: 2));
411 if (count > 20) {
412 debugPrint('delayForFrameTimings is taking longer than expected...');
413 }
414 }
415 }
416
417 await Future<void>.delayed(const Duration(seconds: 2)); // flush old FrameTimings
418 final TimingsCallback watcher = frameTimings.addAll;
419 addTimingsCallback(watcher);
420 final _GarbageCollectionInfo gcInfo = await _runAndGetGCInfo(action);
421
422 await delayForFrameTimings(); // make sure all FrameTimings are reported
423 removeTimingsCallback(watcher);
424
425 final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(
426 frameTimings,
427 newGenGCCount: gcInfo.newCount,
428 oldGenGCCount: gcInfo.oldCount,
429 );
430 reportData ??= <String, dynamic>{};
431 reportData![reportKey] = frameTimes.summary;
432 }
433
434 @override
435 Timeout defaultTestTimeout = Timeout.none;
436
437 @override
438 void reportExceptionNoticed(FlutterErrorDetails exception) {
439 // This method is called to log errors as they happen, and they will also
440 // be eventually logged again at the end of the tests. The superclass
441 // behavior is specific to the "live" execution semantics of
442 // [LiveTestWidgetsFlutterBinding] so users don't have to wait until tests
443 // finish to see the stack traces.
444 //
445 // Disable this because Integration Tests follow the semantics of
446 // [AutomatedTestWidgetsFlutterBinding] that does not log the stack traces
447 // live, and avoids the doubly logged stack trace.
448 // TODO(jiahaog): Integration test binding should not inherit from
449 // `LiveTestWidgetsFlutterBinding` https://github.com/flutter/flutter/issues/81534
450 }
451}
452
453@immutable
454class _GarbageCollectionInfo {
455 const _GarbageCollectionInfo({this.oldCount = -1, this.newCount = -1});
456
457 final int oldCount;
458 final int newCount;
459}
460
461// Connect to the given uri and return a new [VmService] instance.
462//
463// Copied from vm_service_io so that we can pass a custom [HttpClient] for
464// testing. Currently, the WebSocket API reuses an HttpClient that
465// is created before the test can change the HttpOverrides.
466Future<vm.VmService> _vmServiceConnectUri(String wsUri, {HttpClient? httpClient}) async {
467 final WebSocket socket = await WebSocket.connect(wsUri, customClient: httpClient);
468 final StreamController<dynamic> controller = StreamController<dynamic>();
469 final Completer<void> streamClosedCompleter = Completer<void>();
470
471 socket.listen(
472 (dynamic data) => controller.add(data),
473 onDone: () => streamClosedCompleter.complete(),
474 );
475
476 return vm.VmService(
477 controller.stream,
478 (String message) => socket.add(message),
479 disposeHandler: () => socket.close(),
480 streamClosed: streamClosedCompleter.future,
481 );
482}
483