1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5/// @docImport 'package:flutter_driver/flutter_driver.dart';
6///
7/// @docImport 'integration_test_driver_extended.dart';
8library;
9
10import 'dart:async';
11import 'dart:developer' as developer;
12import 'dart:io' show HttpClient, SocketException, WebSocket;
13import 'dart:ui';
14
15import 'package:flutter/foundation.dart';
16import 'package:flutter/rendering.dart';
17import 'package:flutter/services.dart';
18import 'package:flutter/widgets.dart';
19import 'package:flutter_test/flutter_test.dart';
20import 'package:vm_service/vm_service.dart' as vm;
21
22import 'common.dart';
23import 'src/callback.dart' as driver_actions;
24import 'src/channel.dart';
25import 'src/extension.dart';
26
27export 'src/vm_service_golden_client.dart';
28
29const String _success = 'success';
30
31/// Whether results should be reported to the native side over the method
32/// channel.
33///
34/// This is enabled by default for use by native test frameworks like Android
35/// instrumentation or XCTest. When running with the Flutter Tool through
36/// `flutter test integration_test` though, it will be disabled as the Flutter
37/// tool will be responsible for collection of test results.
38const bool _shouldReportResultsToNative = bool.fromEnvironment(
39 'INTEGRATION_TEST_SHOULD_REPORT_RESULTS_TO_NATIVE',
40 defaultValue: true,
41);
42
43/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
44/// on a channel to adapt them to native instrumentation test format.
45class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding
46 implements IntegrationTestResults {
47 /// Sets up a listener to report that the tests are finished when everything is
48 /// torn down.
49 IntegrationTestWidgetsFlutterBinding() {
50 tearDownAll(() async {
51 if (!_allTestsPassed.isCompleted) {
52 _allTestsPassed.complete(failureMethodsDetails.isEmpty);
53 }
54 callbackManager.cleanup();
55
56 // TODO(jiahaog): Print the message directing users to run with
57 // `flutter test` when Web is supported.
58 if (!_shouldReportResultsToNative || kIsWeb) {
59 return;
60 }
61
62 try {
63 await integrationTestChannel.invokeMethod<void>('allTestsFinished', <String, dynamic>{
64 'results': results.map<String, dynamic>((String name, Object result) {
65 if (result is Failure) {
66 return MapEntry<String, dynamic>(name, result.details);
67 }
68 return MapEntry<String, Object>(name, result);
69 }),
70 });
71 } on MissingPluginException {
72 debugPrint(r'''
73Warning: integration_test plugin was not detected.
74
75If you're running the tests with `flutter drive`, please make sure your tests
76are in the `integration_test/` directory of your package and use
77`flutter test $path_to_test` to run it instead.
78
79If you're running the tests with Android instrumentation or XCTest, this means
80that you are not capturing test results properly! See the following link for
81how to set up the integration_test plugin:
82
83https://docs.flutter.dev/testing/integration-tests
84''');
85 }
86 });
87
88 final TestExceptionReporter oldTestExceptionReporter = reportTestException;
89 reportTestException = (FlutterErrorDetails details, String testDescription) {
90 results[testDescription] = Failure(testDescription, details.toString());
91 oldTestExceptionReporter(details, testDescription);
92 };
93 }
94
95 @override
96 bool get overrideHttpClient => false;
97
98 @override
99 bool get registerTestTextInput => false;
100
101 Size? _surfaceSize;
102
103 // This flag is used to print warning messages when tracking performance
104 // under debug mode.
105 static bool _firstRun = false;
106
107 @override
108 Future<void> setSurfaceSize(Size? size) {
109 return TestAsyncUtils.guard<void>(() async {
110 assert(inTest);
111 if (_surfaceSize == size) {
112 return;
113 }
114 _surfaceSize = size;
115 handleMetricsChanged();
116 });
117 }
118
119 @override
120 ViewConfiguration createViewConfigurationFor(RenderView renderView) {
121 final FlutterView view = renderView.flutterView;
122 final Size? surfaceSize = view == platformDispatcher.implicitView ? _surfaceSize : null;
123 return TestViewConfiguration.fromView(
124 size: surfaceSize ?? view.physicalSize / view.devicePixelRatio,
125 view: view,
126 );
127 }
128
129 @override
130 Completer<bool> get allTestsPassed => _allTestsPassed;
131 final Completer<bool> _allTestsPassed = Completer<bool>();
132
133 @override
134 List<Failure> get failureMethodsDetails => results.values.whereType<Failure>().toList();
135
136 @override
137 void initInstances() {
138 super.initInstances();
139 _instance = this;
140 }
141
142 /// The singleton instance of this object.
143 ///
144 /// Provides access to the features exposed by this class. The binding must
145 /// be initialized before using this getter; this is typically done by calling
146 /// [IntegrationTestWidgetsFlutterBinding.ensureInitialized].
147 static IntegrationTestWidgetsFlutterBinding get instance => BindingBase.checkInstance(_instance);
148 static IntegrationTestWidgetsFlutterBinding? _instance;
149
150 /// Returns an instance of the [IntegrationTestWidgetsFlutterBinding], creating and
151 /// initializing it if necessary.
152 ///
153 /// See also:
154 ///
155 /// * [WidgetsFlutterBinding.ensureInitialized], the equivalent in the widgets framework.
156 static IntegrationTestWidgetsFlutterBinding ensureInitialized() {
157 if (_instance == null) {
158 IntegrationTestWidgetsFlutterBinding();
159 }
160 return _instance!;
161 }
162
163 /// Test results that will be populated after the tests have completed.
164 ///
165 /// Keys are the test descriptions, and values are either [_success] or
166 /// a [Failure].
167 @visibleForTesting
168 Map<String, Object> results = <String, Object>{};
169
170 /// The extra data for the reported result.
171 ///
172 /// The values in `reportData` must be json-serializable objects or `null`.
173 /// If it's `null`, no extra data is attached to the result.
174 ///
175 /// The default value is `null`.
176 @override
177 Map<String, dynamic>? reportData;
178
179 /// Manages callbacks received from driver side and commands send to driver
180 /// side.
181 final CallbackManager callbackManager = driver_actions.callbackManager;
182
183 /// Takes a screenshot.
184 ///
185 /// On Android, you need to call `convertFlutterSurfaceToImage()`, and
186 /// pump a frame before taking a screenshot.
187 Future<List<int>> takeScreenshot(String screenshotName, [Map<String, Object?>? args]) async {
188 reportData ??= <String, dynamic>{};
189 reportData!['screenshots'] ??= <dynamic>[];
190 final Map<String, dynamic> data = await callbackManager.takeScreenshot(screenshotName, args);
191 assert(data.containsKey('bytes'));
192
193 (reportData!['screenshots']! as List<dynamic>).add(data);
194 return data['bytes']! as List<int>;
195 }
196
197 /// Android only. Converts the Flutter surface to an image view.
198 /// Be aware that if you are conducting a perf test, you may not want to call
199 /// this method since the this is an expensive operation that affects the
200 /// rendering of a Flutter app.
201 ///
202 /// Once the screenshot is taken, call `revertFlutterImage()` to restore
203 /// the original Flutter surface.
204 Future<void> convertFlutterSurfaceToImage() async {
205 await callbackManager.convertFlutterSurfaceToImage();
206 }
207
208 /// The callback function to response the driver side input.
209 @visibleForTesting
210 Future<Map<String, dynamic>> callback(Map<String, String> params) async {
211 return callbackManager.callback(params, this /* as IntegrationTestResults */);
212 }
213
214 // Emulates the Flutter driver extension, returning 'pass' or 'fail'.
215 @override
216 void initServiceExtensions() {
217 super.initServiceExtensions();
218
219 if (kIsWeb) {
220 registerWebServiceExtension(callback);
221 }
222
223 registerServiceExtension(name: 'driver', callback: callback);
224 }
225
226 @override
227 Future<void> runTest(
228 Future<void> Function() testBody,
229 VoidCallback invariantTester, {
230 String description = '',
231 @Deprecated(
232 'This parameter has no effect. Use the `timeout` parameter on `testWidgets` instead. '
233 'This feature was deprecated after v2.6.0-1.0.pre.',
234 )
235 Duration? timeout,
236 }) async {
237 await super.runTest(testBody, invariantTester, description: description);
238 results[description] ??= _success;
239 }
240
241 // Do not paint a description label because it could show up in screenshots
242 // of the integration test.
243 @override
244 void setLabel(String value) {}
245
246 vm.VmService? _vmService;
247
248 /// Initialize the [vm.VmService] settings for the timeline.
249 @visibleForTesting
250 Future<void> enableTimeline({
251 List<String> streams = const <String>['all'],
252 @visibleForTesting vm.VmService? vmService,
253 @visibleForTesting HttpClient? httpClient,
254 }) async {
255 assert(streams.isNotEmpty);
256 if (vmService != null) {
257 _vmService = vmService;
258 }
259 if (_vmService == null) {
260 final developer.ServiceProtocolInfo info = await developer.Service.getInfo();
261 assert(info.serverUri != null);
262 final String address = 'ws://localhost:${info.serverUri!.port}${info.serverUri!.path}ws';
263 try {
264 _vmService = await _vmServiceConnectUri(address, httpClient: httpClient);
265 } on SocketException catch (e, s) {
266 throw StateError(
267 'Failed to connect to VM Service at $address.\n'
268 'This may happen if DDS is enabled. If this test was launched via '
269 '`flutter drive`, try adding `--no-dds`.\n'
270 'The original exception was:\n'
271 '$e\n$s',
272 );
273 }
274 }
275 await _vmService!.setVMTimelineFlags(streams);
276 }
277
278 /// Runs [action] and returns a [vm.Timeline] trace for it.
279 ///
280 /// Waits for the `Future` returned by [action] to complete prior to stopping
281 /// the trace.
282 ///
283 /// The `streams` parameter limits the recorded timeline event streams to only
284 /// the ones listed. By default, all streams are recorded.
285 /// See `timeline_streams` in
286 /// [Dart-SDK/runtime/vm/timeline.cc](https://github.com/dart-lang/sdk/blob/main/runtime/vm/timeline.cc)
287 ///
288 /// If [retainPriorEvents] is true, retains events recorded prior to calling
289 /// [action]. Otherwise, prior events are cleared before calling [action]. By
290 /// default, prior events are cleared.
291 Future<vm.Timeline> traceTimeline(
292 Future<dynamic> Function() action, {
293 List<String> streams = const <String>['all'],
294 bool retainPriorEvents = false,
295 }) async {
296 await enableTimeline(streams: streams);
297 if (retainPriorEvents) {
298 await action();
299 return _vmService!.getVMTimeline();
300 }
301
302 await _vmService!.clearVMTimeline();
303 final vm.Timestamp startTime = await _vmService!.getVMTimelineMicros();
304 await action();
305 final vm.Timestamp endTime = await _vmService!.getVMTimelineMicros();
306 return _vmService!.getVMTimeline(
307 timeOriginMicros: startTime.timestamp,
308 timeExtentMicros: endTime.timestamp,
309 );
310 }
311
312 /// This is a convenience method that calls [traceTimeline] and sends the
313 /// result back to the host for the [flutter_driver] style tests.
314 ///
315 /// This records the timeline during `action` and adds the result to
316 /// [reportData] with `reportKey`. The [reportData] contains extra information
317 /// from the test other than test success/fail. It will be passed back to the
318 /// host and be processed by the [ResponseDataCallback] defined in
319 /// [integrationDriver]. By default it will be written to
320 /// `build/integration_response_data.json` with the key `timeline`.
321 ///
322 /// For tests with multiple calls of this method, `reportKey` needs to be a
323 /// unique key, otherwise the later result will override earlier one. Tests
324 /// that call this multiple times must also provide a custom
325 /// [ResponseDataCallback] to decide where and how to write the output
326 /// timelines. For example,
327 ///
328 /// ```dart
329 /// import 'package:integration_test/integration_test_driver.dart';
330 ///
331 /// Future<void> main() {
332 /// return integrationDriver(
333 /// responseDataCallback: (Map<String, dynamic>? data) async {
334 /// if (data != null) {
335 /// for (final MapEntry<String, dynamic> entry in data.entries) {
336 /// print('Writing ${entry.key} to the disk.');
337 /// await writeResponseData(
338 /// entry.value as Map<String, dynamic>,
339 /// testOutputFilename: entry.key,
340 /// );
341 /// }
342 /// }
343 /// },
344 /// );
345 /// }
346 /// ```
347 ///
348 /// The `streams` and `retainPriorEvents` parameters are passed as-is to
349 /// [traceTimeline].
350 Future<void> traceAction(
351 Future<dynamic> Function() action, {
352 List<String> streams = const <String>['all'],
353 bool retainPriorEvents = false,
354 String reportKey = 'timeline',
355 }) async {
356 final vm.Timeline timeline = await traceTimeline(
357 action,
358 streams: streams,
359 retainPriorEvents: retainPriorEvents,
360 );
361 reportData ??= <String, dynamic>{};
362 reportData![reportKey] = timeline.toJson();
363 }
364
365 Future<_GarbageCollectionInfo> _runAndGetGCInfo(Future<void> Function() action) async {
366 if (kIsWeb) {
367 await action();
368 return const _GarbageCollectionInfo();
369 }
370
371 final vm.Timeline timeline = await traceTimeline(action, streams: <String>['GC']);
372
373 final int oldGenGCCount =
374 timeline.traceEvents!.where((vm.TimelineEvent event) {
375 return event.json!['cat'] == 'GC' && event.json!['name'] == 'CollectOldGeneration';
376 }).length;
377 final int newGenGCCount =
378 timeline.traceEvents!.where((vm.TimelineEvent event) {
379 return event.json!['cat'] == 'GC' && event.json!['name'] == 'CollectNewGeneration';
380 }).length;
381 return _GarbageCollectionInfo(oldCount: oldGenGCCount, newCount: newGenGCCount);
382 }
383
384 /// Watches the [FrameTiming] during `action` and report it to the binding
385 /// with key `reportKey`.
386 ///
387 /// This can be used to implement performance tests previously using
388 /// [traceAction] and [TimelineSummary] from [flutter_driver]
389 Future<void> watchPerformance(
390 Future<void> Function() action, {
391 String reportKey = 'performance',
392 }) async {
393 assert(() {
394 if (_firstRun) {
395 debugPrint(kDebugWarning);
396 _firstRun = false;
397 }
398 return true;
399 }());
400
401 // The engine could batch FrameTimings and send them only once per second.
402 // Delay for a sufficient time so either old FrameTimings are flushed and not
403 // interfering our measurements here, or new FrameTimings are all reported.
404 // TODO(CareF): remove this when flush FrameTiming is readily in engine.
405 // See https://github.com/flutter/flutter/issues/64808
406 // and https://github.com/flutter/flutter/issues/67593
407 final List<FrameTiming> frameTimings = <FrameTiming>[];
408 Future<void> delayForFrameTimings() async {
409 int count = 0;
410 while (frameTimings.isEmpty) {
411 count++;
412 await Future<void>.delayed(const Duration(seconds: 2));
413 if (count > 20) {
414 debugPrint('delayForFrameTimings is taking longer than expected...');
415 }
416 }
417 }
418
419 await Future<void>.delayed(const Duration(seconds: 2)); // flush old FrameTimings
420 final TimingsCallback watcher = frameTimings.addAll;
421 addTimingsCallback(watcher);
422 final _GarbageCollectionInfo gcInfo = await _runAndGetGCInfo(action);
423
424 await delayForFrameTimings(); // make sure all FrameTimings are reported
425 removeTimingsCallback(watcher);
426
427 final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(
428 frameTimings,
429 newGenGCCount: gcInfo.newCount,
430 oldGenGCCount: gcInfo.oldCount,
431 );
432 reportData ??= <String, dynamic>{};
433 reportData![reportKey] = frameTimes.summary;
434 }
435
436 @override
437 Timeout defaultTestTimeout = Timeout.none;
438
439 @override
440 void reportExceptionNoticed(FlutterErrorDetails exception) {
441 // This method is called to log errors as they happen, and they will also
442 // be eventually logged again at the end of the tests. The superclass
443 // behavior is specific to the "live" execution semantics of
444 // [LiveTestWidgetsFlutterBinding] so users don't have to wait until tests
445 // finish to see the stack traces.
446 //
447 // Disable this because Integration Tests follow the semantics of
448 // [AutomatedTestWidgetsFlutterBinding] that does not log the stack traces
449 // live, and avoids the doubly logged stack trace.
450 // TODO(jiahaog): Integration test binding should not inherit from
451 // `LiveTestWidgetsFlutterBinding` https://github.com/flutter/flutter/issues/81534
452 }
453}
454
455@immutable
456class _GarbageCollectionInfo {
457 const _GarbageCollectionInfo({this.oldCount = -1, this.newCount = -1});
458
459 final int oldCount;
460 final int newCount;
461}
462
463// Connect to the given uri and return a new [VmService] instance.
464//
465// Copied from vm_service_io so that we can pass a custom [HttpClient] for
466// testing. Currently, the WebSocket API reuses an HttpClient that
467// is created before the test can change the HttpOverrides.
468Future<vm.VmService> _vmServiceConnectUri(String wsUri, {HttpClient? httpClient}) async {
469 final WebSocket socket = await WebSocket.connect(wsUri, customClient: httpClient);
470 final StreamController<dynamic> controller = StreamController<dynamic>();
471 final Completer<void> streamClosedCompleter = Completer<void>();
472
473 socket.listen(
474 (dynamic data) => controller.add(data),
475 onDone: () => streamClosedCompleter.complete(),
476 );
477
478 return vm.VmService(
479 controller.stream,
480 (String message) => socket.add(message),
481 disposeHandler: () => socket.close(),
482 streamClosed: streamClosedCompleter.future,
483 );
484}
485

Provided by KDAB

Privacy Policy
Learn more about Flutter for embedded and desktop on industrialflutter.com