1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5/// @docImport 'package:flutter_driver/flutter_driver.dart';
6///
7/// @docImport 'integration_test_driver_extended.dart';
8library;
9
10import 'dart:async';
11import 'dart:developer' as developer;
12import 'dart:io' show HttpClient, SocketException, WebSocket;
13import 'dart:ui';
14
15import 'package:flutter/foundation.dart';
16import 'package:flutter/rendering.dart';
17import 'package:flutter/services.dart';
18import 'package:flutter/widgets.dart';
19import 'package:flutter_test/flutter_test.dart';
20import 'package:vm_service/vm_service.dart' as vm;
21
22import 'common.dart';
23import 'src/callback.dart' as driver_actions;
24import 'src/channel.dart';
25import 'src/extension.dart';
26
27const String _success = 'success';
28
29/// Whether results should be reported to the native side over the method
30/// channel.
31///
32/// This is enabled by default for use by native test frameworks like Android
33/// instrumentation or XCTest. When running with the Flutter Tool through
34/// `flutter test integration_test` though, it will be disabled as the Flutter
35/// tool will be responsible for collection of test results.
36const bool _shouldReportResultsToNative = bool.fromEnvironment(
37 'INTEGRATION_TEST_SHOULD_REPORT_RESULTS_TO_NATIVE',
38 defaultValue: true,
39);
40
41/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
42/// on a channel to adapt them to native instrumentation test format.
43class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding implements IntegrationTestResults {
44 /// Sets up a listener to report that the tests are finished when everything is
45 /// torn down.
46 IntegrationTestWidgetsFlutterBinding() {
47 tearDownAll(() async {
48 if (!_allTestsPassed.isCompleted) {
49 _allTestsPassed.complete(failureMethodsDetails.isEmpty);
50 }
51 callbackManager.cleanup();
52
53 // TODO(jiahaog): Print the message directing users to run with
54 // `flutter test` when Web is supported.
55 if (!_shouldReportResultsToNative || kIsWeb) {
56 return;
57 }
58
59 try {
60 await integrationTestChannel.invokeMethod<void>(
61 'allTestsFinished',
62 <String, dynamic>{
63 'results': results.map<String, dynamic>((String name, Object result) {
64 if (result is Failure) {
65 return MapEntry<String, dynamic>(name, result.details);
66 }
67 return MapEntry<String, Object>(name, result);
68 }),
69 },
70 );
71 } on MissingPluginException {
72 debugPrint(r'''
73Warning: integration_test plugin was not detected.
74
75If you're running the tests with `flutter drive`, please make sure your tests
76are in the `integration_test/` directory of your package and use
77`flutter test $path_to_test` to run it instead.
78
79If you're running the tests with Android instrumentation or XCTest, this means
80that you are not capturing test results properly! See the following link for
81how to set up the integration_test plugin:
82
83https://docs.flutter.dev/testing/integration-tests
84''');
85 }
86 });
87
88 final TestExceptionReporter oldTestExceptionReporter = reportTestException;
89 reportTestException =
90 (FlutterErrorDetails details, String testDescription) {
91 results[testDescription] = Failure(testDescription, details.toString());
92 oldTestExceptionReporter(details, testDescription);
93 };
94 }
95
96 @override
97 bool get overrideHttpClient => false;
98
99 @override
100 bool get registerTestTextInput => false;
101
102 Size? _surfaceSize;
103
104 // This flag is used to print warning messages when tracking performance
105 // under debug mode.
106 static bool _firstRun = false;
107
108 @override
109 Future<void> setSurfaceSize(Size? size) {
110 return TestAsyncUtils.guard<void>(() async {
111 assert(inTest);
112 if (_surfaceSize == size) {
113 return;
114 }
115 _surfaceSize = size;
116 handleMetricsChanged();
117 });
118 }
119
120 @override
121 ViewConfiguration createViewConfigurationFor(RenderView renderView) {
122 final FlutterView view = renderView.flutterView;
123 final Size? surfaceSize = view == platformDispatcher.implicitView ? _surfaceSize : null;
124 return TestViewConfiguration.fromView(
125 size: surfaceSize ?? view.physicalSize / view.devicePixelRatio,
126 view: view,
127 );
128 }
129
130 @override
131 Completer<bool> get allTestsPassed => _allTestsPassed;
132 final Completer<bool> _allTestsPassed = Completer<bool>();
133
134 @override
135 List<Failure> get failureMethodsDetails => results.values.whereType<Failure>().toList();
136
137 @override
138 void initInstances() {
139 super.initInstances();
140 _instance = this;
141 }
142
143 /// The singleton instance of this object.
144 ///
145 /// Provides access to the features exposed by this class. The binding must
146 /// be initialized before using this getter; this is typically done by calling
147 /// [IntegrationTestWidgetsFlutterBinding.ensureInitialized].
148 static IntegrationTestWidgetsFlutterBinding get instance => BindingBase.checkInstance(_instance);
149 static IntegrationTestWidgetsFlutterBinding? _instance;
150
151 /// Returns an instance of the [IntegrationTestWidgetsFlutterBinding], creating and
152 /// initializing it if necessary.
153 ///
154 /// See also:
155 ///
156 /// * [WidgetsFlutterBinding.ensureInitialized], the equivalent in the widgets framework.
157 static IntegrationTestWidgetsFlutterBinding ensureInitialized() {
158 if (_instance == null) {
159 IntegrationTestWidgetsFlutterBinding();
160 }
161 return _instance!;
162 }
163
164 /// Test results that will be populated after the tests have completed.
165 ///
166 /// Keys are the test descriptions, and values are either [_success] or
167 /// a [Failure].
168 @visibleForTesting
169 Map<String, Object> results = <String, Object>{};
170
171 /// The extra data for the reported result.
172 ///
173 /// The values in `reportData` must be json-serializable objects or `null`.
174 /// If it's `null`, no extra data is attached to the result.
175 ///
176 /// The default value is `null`.
177 @override
178 Map<String, dynamic>? reportData;
179
180 /// Manages callbacks received from driver side and commands send to driver
181 /// side.
182 final CallbackManager callbackManager = driver_actions.callbackManager;
183
184 /// Takes a screenshot.
185 ///
186 /// On Android, you need to call `convertFlutterSurfaceToImage()`, and
187 /// pump a frame before taking a screenshot.
188 Future<List<int>> takeScreenshot(String screenshotName, [Map<String, Object?>? args]) async {
189 reportData ??= <String, dynamic>{};
190 reportData!['screenshots'] ??= <dynamic>[];
191 final Map<String, dynamic> data = await callbackManager.takeScreenshot(screenshotName, args);
192 assert(data.containsKey('bytes'));
193
194 (reportData!['screenshots']! as List<dynamic>).add(data);
195 return data['bytes']! as List<int>;
196 }
197
198 /// Android only. Converts the Flutter surface to an image view.
199 /// Be aware that if you are conducting a perf test, you may not want to call
200 /// this method since the this is an expensive operation that affects the
201 /// rendering of a Flutter app.
202 ///
203 /// Once the screenshot is taken, call `revertFlutterImage()` to restore
204 /// the original Flutter surface.
205 Future<void> convertFlutterSurfaceToImage() async {
206 await callbackManager.convertFlutterSurfaceToImage();
207 }
208
209 /// The callback function to response the driver side input.
210 @visibleForTesting
211 Future<Map<String, dynamic>> callback(Map<String, String> params) async {
212 return callbackManager.callback(
213 params, this /* as IntegrationTestResults */);
214 }
215
216 // Emulates the Flutter driver extension, returning 'pass' or 'fail'.
217 @override
218 void initServiceExtensions() {
219 super.initServiceExtensions();
220
221 if (kIsWeb) {
222 registerWebServiceExtension(callback);
223 }
224
225 registerServiceExtension(name: 'driver', callback: callback);
226 }
227
228 @override
229 Future<void> runTest(
230 Future<void> Function() testBody,
231 VoidCallback invariantTester, {
232 String description = '',
233 @Deprecated(
234 'This parameter has no effect. Use the `timeout` parameter on `testWidgets` instead. '
235 'This feature was deprecated after v2.6.0-1.0.pre.'
236 )
237 Duration? timeout,
238 }) async {
239 await super.runTest(
240 testBody,
241 invariantTester,
242 description: description,
243 );
244 results[description] ??= _success;
245 }
246
247 // Do not paint a description label because it could show up in screenshots
248 // of the integration test.
249 @override
250 void setLabel(String value) {}
251
252 vm.VmService? _vmService;
253
254 /// Initialize the [vm.VmService] settings for the timeline.
255 @visibleForTesting
256 Future<void> enableTimeline({
257 List<String> streams = const <String>['all'],
258 @visibleForTesting vm.VmService? vmService,
259 @visibleForTesting HttpClient? httpClient,
260 }) async {
261 assert(streams.isNotEmpty);
262 if (vmService != null) {
263 _vmService = vmService;
264 }
265 if (_vmService == null) {
266 final developer.ServiceProtocolInfo info = await developer.Service.getInfo();
267 assert(info.serverUri != null);
268 final String address = 'ws://localhost:${info.serverUri!.port}${info.serverUri!.path}ws';
269 try {
270 _vmService = await _vmServiceConnectUri(address, httpClient: httpClient);
271 } on SocketException catch (e, s) {
272 throw StateError(
273 'Failed to connect to VM Service at $address.\n'
274 'This may happen if DDS is enabled. If this test was launched via '
275 '`flutter drive`, try adding `--no-dds`.\n'
276 'The original exception was:\n'
277 '$e\n$s',
278 );
279 }
280 }
281 await _vmService!.setVMTimelineFlags(streams);
282 }
283
284 /// Runs [action] and returns a [vm.Timeline] trace for it.
285 ///
286 /// Waits for the `Future` returned by [action] to complete prior to stopping
287 /// the trace.
288 ///
289 /// The `streams` parameter limits the recorded timeline event streams to only
290 /// the ones listed. By default, all streams are recorded.
291 /// See `timeline_streams` in
292 /// [Dart-SDK/runtime/vm/timeline.cc](https://github.com/dart-lang/sdk/blob/main/runtime/vm/timeline.cc)
293 ///
294 /// If [retainPriorEvents] is true, retains events recorded prior to calling
295 /// [action]. Otherwise, prior events are cleared before calling [action]. By
296 /// default, prior events are cleared.
297 Future<vm.Timeline> traceTimeline(
298 Future<dynamic> Function() action, {
299 List<String> streams = const <String>['all'],
300 bool retainPriorEvents = false,
301 }) async {
302 await enableTimeline(streams: streams);
303 if (retainPriorEvents) {
304 await action();
305 return _vmService!.getVMTimeline();
306 }
307
308 await _vmService!.clearVMTimeline();
309 final vm.Timestamp startTime = await _vmService!.getVMTimelineMicros();
310 await action();
311 final vm.Timestamp endTime = await _vmService!.getVMTimelineMicros();
312 return _vmService!.getVMTimeline(
313 timeOriginMicros: startTime.timestamp,
314 timeExtentMicros: endTime.timestamp,
315 );
316 }
317
318 /// This is a convenience method that calls [traceTimeline] and sends the
319 /// result back to the host for the [flutter_driver] style tests.
320 ///
321 /// This records the timeline during `action` and adds the result to
322 /// [reportData] with `reportKey`. The [reportData] contains extra information
323 /// from the test other than test success/fail. It will be passed back to the
324 /// host and be processed by the [ResponseDataCallback] defined in
325 /// [integrationDriver]. By default it will be written to
326 /// `build/integration_response_data.json` with the key `timeline`.
327 ///
328 /// For tests with multiple calls of this method, `reportKey` needs to be a
329 /// unique key, otherwise the later result will override earlier one. Tests
330 /// that call this multiple times must also provide a custom
331 /// [ResponseDataCallback] to decide where and how to write the output
332 /// timelines. For example,
333 ///
334 /// ```dart
335 /// import 'package:integration_test/integration_test_driver.dart';
336 ///
337 /// Future<void> main() {
338 /// return integrationDriver(
339 /// responseDataCallback: (Map<String, dynamic>? data) async {
340 /// if (data != null) {
341 /// for (final MapEntry<String, dynamic> entry in data.entries) {
342 /// print('Writing ${entry.key} to the disk.');
343 /// await writeResponseData(
344 /// entry.value as Map<String, dynamic>,
345 /// testOutputFilename: entry.key,
346 /// );
347 /// }
348 /// }
349 /// },
350 /// );
351 /// }
352 /// ```
353 ///
354 /// The `streams` and `retainPriorEvents` parameters are passed as-is to
355 /// [traceTimeline].
356 Future<void> traceAction(
357 Future<dynamic> Function() action, {
358 List<String> streams = const <String>['all'],
359 bool retainPriorEvents = false,
360 String reportKey = 'timeline',
361 }) async {
362 final vm.Timeline timeline = await traceTimeline(
363 action,
364 streams: streams,
365 retainPriorEvents: retainPriorEvents,
366 );
367 reportData ??= <String, dynamic>{};
368 reportData![reportKey] = timeline.toJson();
369 }
370
371 Future<_GarbageCollectionInfo> _runAndGetGCInfo(Future<void> Function() action) async {
372 if (kIsWeb) {
373 await action();
374 return const _GarbageCollectionInfo();
375 }
376
377 final vm.Timeline timeline = await traceTimeline(
378 action,
379 streams: <String>['GC'],
380 );
381
382 final int oldGenGCCount = timeline.traceEvents!.where((vm.TimelineEvent event) {
383 return event.json!['cat'] == 'GC' && event.json!['name'] == 'CollectOldGeneration';
384 }).length;
385 final int newGenGCCount = timeline.traceEvents!.where((vm.TimelineEvent event) {
386 return event.json!['cat'] == 'GC' && event.json!['name'] == 'CollectNewGeneration';
387 }).length;
388 return _GarbageCollectionInfo(
389 oldCount: oldGenGCCount,
390 newCount: newGenGCCount,
391 );
392 }
393
394 /// Watches the [FrameTiming] during `action` and report it to the binding
395 /// with key `reportKey`.
396 ///
397 /// This can be used to implement performance tests previously using
398 /// [traceAction] and [TimelineSummary] from [flutter_driver]
399 Future<void> watchPerformance(
400 Future<void> Function() action, {
401 String reportKey = 'performance',
402 }) async {
403 assert(() {
404 if (_firstRun) {
405 debugPrint(kDebugWarning);
406 _firstRun = false;
407 }
408 return true;
409 }());
410
411 // The engine could batch FrameTimings and send them only once per second.
412 // Delay for a sufficient time so either old FrameTimings are flushed and not
413 // interfering our measurements here, or new FrameTimings are all reported.
414 // TODO(CareF): remove this when flush FrameTiming is readily in engine.
415 // See https://github.com/flutter/flutter/issues/64808
416 // and https://github.com/flutter/flutter/issues/67593
417 final List<FrameTiming> frameTimings = <FrameTiming>[];
418 Future<void> delayForFrameTimings() async {
419 int count = 0;
420 while (frameTimings.isEmpty) {
421 count++;
422 await Future<void>.delayed(const Duration(seconds: 2));
423 if (count > 20) {
424 debugPrint('delayForFrameTimings is taking longer than expected...');
425 }
426 }
427 }
428
429 await Future<void>.delayed(const Duration(seconds: 2)); // flush old FrameTimings
430 final TimingsCallback watcher = frameTimings.addAll;
431 addTimingsCallback(watcher);
432 final _GarbageCollectionInfo gcInfo = await _runAndGetGCInfo(action);
433
434 await delayForFrameTimings(); // make sure all FrameTimings are reported
435 removeTimingsCallback(watcher);
436
437 final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(
438 frameTimings,
439 newGenGCCount: gcInfo.newCount,
440 oldGenGCCount: gcInfo.oldCount,
441 );
442 reportData ??= <String, dynamic>{};
443 reportData![reportKey] = frameTimes.summary;
444 }
445
446 @override
447 Timeout defaultTestTimeout = Timeout.none;
448
449 @override
450 Widget wrapWithDefaultView(Widget rootWidget) {
451 // This is a workaround where screenshots of root widgets have incorrect
452 // bounds.
453 // TODO(jiahaog): Remove when https://github.com/flutter/flutter/issues/66006 is fixed.
454 return super.wrapWithDefaultView(RepaintBoundary(child: rootWidget));
455 }
456
457 @override
458 void reportExceptionNoticed(FlutterErrorDetails exception) {
459 // This method is called to log errors as they happen, and they will also
460 // be eventually logged again at the end of the tests. The superclass
461 // behavior is specific to the "live" execution semantics of
462 // [LiveTestWidgetsFlutterBinding] so users don't have to wait until tests
463 // finish to see the stack traces.
464 //
465 // Disable this because Integration Tests follow the semantics of
466 // [AutomatedTestWidgetsFlutterBinding] that does not log the stack traces
467 // live, and avoids the doubly logged stack trace.
468 // TODO(jiahaog): Integration test binding should not inherit from
469 // `LiveTestWidgetsFlutterBinding` https://github.com/flutter/flutter/issues/81534
470 }
471}
472
473@immutable
474class _GarbageCollectionInfo {
475 const _GarbageCollectionInfo({this.oldCount = -1, this.newCount = -1});
476
477 final int oldCount;
478 final int newCount;
479}
480
481// Connect to the given uri and return a new [VmService] instance.
482//
483// Copied from vm_service_io so that we can pass a custom [HttpClient] for
484// testing. Currently, the WebSocket API reuses an HttpClient that
485// is created before the test can change the HttpOverrides.
486Future<vm.VmService> _vmServiceConnectUri(
487 String wsUri, {
488 HttpClient? httpClient,
489}) async {
490 final WebSocket socket = await WebSocket.connect(wsUri, customClient: httpClient);
491 final StreamController<dynamic> controller = StreamController<dynamic>();
492 final Completer<void> streamClosedCompleter = Completer<void>();
493
494 socket.listen(
495 (dynamic data) => controller.add(data),
496 onDone: () => streamClosedCompleter.complete(),
497 );
498
499 return vm.VmService(
500 controller.stream,
501 (String message) => socket.add(message),
502 disposeHandler: () => socket.close(),
503 streamClosed: streamClosedCompleter.future,
504 );
505}
506

Provided by KDAB

Privacy Policy
Learn more about Flutter for embedded and desktop on industrialflutter.com