1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5import 'package:meta/meta.dart';
6import 'package:package_config/package_config_types.dart';
7
8import '../asset.dart';
9import '../base/common.dart';
10import '../base/file_system.dart';
11import '../build_info.dart';
12import '../bundle_builder.dart';
13import '../devfs.dart';
14import '../device.dart';
15import '../globals.dart' as globals;
16import '../native_assets.dart';
17import '../project.dart';
18import '../runner/flutter_command.dart';
19import '../runner/flutter_command_runner.dart';
20import '../test/coverage_collector.dart';
21import '../test/event_printer.dart';
22import '../test/runner.dart';
23import '../test/test_time_recorder.dart';
24import '../test/test_wrapper.dart';
25import '../test/watcher.dart';
26import '../web/compile.dart';
27import '../web/web_constants.dart';
28
29/// The name of the directory where Integration Tests are placed.
30///
31/// When there are test files specified for the test command that are part of
32/// this directory, *relative to the package root*, the files will be executed
33/// as Integration Tests.
34const _kIntegrationTestDirectory = 'integration_test';
35
36/// A command to run tests.
37///
38/// This command has two modes of execution:
39///
40/// ## Unit / Widget Tests
41///
42/// These tests run in the Flutter Tester, which is a desktop-based Flutter
43/// embedder. In this mode, tests are quick to compile and run.
44///
45/// By default, if no flags are passed to the `flutter test` command, the Tool
46/// will recursively find all files within the `test/` directory that end with
47/// the `*_test.dart` suffix, and run them in a single invocation.
48///
49/// See:
50/// - https://flutter.dev/to/unit-testing
51/// - https://flutter.dev/to/widget-testing
52///
53/// ## Integration Tests
54///
55/// These tests run in a connected Flutter Device, similar to `flutter run`. As
56/// a result, iteration is slower because device-based artifacts have to be
57/// built.
58///
59/// Integration tests should be placed in the `integration_test/` directory of
60/// your package. To run these tests, use `flutter test integration_test`.
61///
62/// See:
63/// - https://flutter.dev/to/integration-testing
64class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts {
65 TestCommand({
66 bool verboseHelp = false,
67 this.testWrapper = const TestWrapper(),
68 this.testRunner = const FlutterTestRunner(),
69 this.verbose = false,
70 this.nativeAssetsBuilder,
71 }) {
72 requiresPubspecYaml();
73 usesPubOption();
74 usesFrontendServerStarterPathOption(verboseHelp: verboseHelp);
75 usesTrackWidgetCreation(verboseHelp: verboseHelp);
76 addEnableExperimentation(hide: !verboseHelp);
77 usesDartDefineOption();
78 usesDeviceUserOption();
79 usesFlavorOption();
80 addEnableImpellerFlag(verboseHelp: verboseHelp);
81 addMachineOutputFlag(verboseHelp: verboseHelp);
82 addEnableFlutterGpuFlag(verboseHelp: verboseHelp);
83
84 argParser
85 ..addFlag(
86 'experimental-faster-testing',
87 negatable: false,
88 hide: !verboseHelp,
89 help: 'Run each test in a separate lightweight Flutter Engine to speed up testing.',
90 )
91 ..addMultiOption(
92 'name',
93 help: 'A regular expression matching substrings of the names of tests to run.',
94 valueHelp: 'regexp',
95 splitCommas: false,
96 )
97 ..addMultiOption(
98 'plain-name',
99 help: 'A plain-text substring of the names of tests to run.',
100 valueHelp: 'substring',
101 splitCommas: false,
102 )
103 ..addOption(
104 'tags',
105 abbr: 't',
106 help:
107 'Run only tests associated with the specified tags. See: https://pub.dev/packages/test#tagging-tests',
108 )
109 ..addOption(
110 'exclude-tags',
111 abbr: 'x',
112 help:
113 'Run only tests that do not have the specified tags. See: https://pub.dev/packages/test#tagging-tests',
114 )
115 ..addFlag(
116 'start-paused',
117 negatable: false,
118 help:
119 'Start in a paused mode and wait for a debugger to connect.\n'
120 'You must specify a single test file to run, explicitly.\n'
121 'Instructions for connecting with a debugger are printed to the '
122 'console once the test has started.',
123 )
124 ..addFlag('fail-fast', help: 'Stop running tests after the first failure.')
125 ..addFlag('run-skipped', help: 'Run skipped tests instead of skipping them.')
126 ..addFlag(
127 'disable-service-auth-codes',
128 negatable: false,
129 hide: !verboseHelp,
130 help:
131 '(deprecated) Allow connections to the VM service without using authentication codes. '
132 '(Not recommended! This can open your device to remote code execution attacks!)',
133 )
134 ..addFlag('coverage', negatable: false, help: 'Whether to collect coverage information.')
135 ..addFlag(
136 'merge-coverage',
137 negatable: false,
138 help:
139 'Whether to merge coverage data with "coverage/lcov.base.info".\n'
140 'Implies collecting coverage data. (Requires lcov.)',
141 )
142 ..addFlag(
143 'branch-coverage',
144 negatable: false,
145 help:
146 'Whether to collect branch coverage information. '
147 'Implies collecting coverage data.',
148 )
149 ..addFlag(
150 'ipv6',
151 negatable: false,
152 hide: !verboseHelp,
153 help: 'Whether to use IPv6 for the test harness server socket.',
154 )
155 ..addOption(
156 'coverage-path',
157 defaultsTo: 'coverage/lcov.info',
158 help: 'Where to store coverage information (if coverage is enabled).',
159 )
160 ..addMultiOption(
161 'coverage-package',
162 help:
163 'A regular expression matching packages names '
164 'to include in the coverage report (if coverage is enabled). '
165 'If unset, matches the current package name.',
166 valueHelp: 'package-name-regexp',
167 splitCommas: false,
168 )
169 ..addFlag(
170 'update-goldens',
171 negatable: false,
172 help:
173 'Whether "matchesGoldenFile()" calls within your test methods should ' // flutter_ignore: golden_tag (see analyze.dart)
174 'update the golden files rather than test for an existing match.',
175 )
176 ..addOption(
177 'concurrency',
178 abbr: 'j',
179 help:
180 'The number of concurrent test processes to run. This will be ignored '
181 'when running integration tests.',
182 valueHelp: 'jobs',
183 )
184 ..addFlag(
185 'test-assets',
186 defaultsTo: true,
187 help:
188 'Whether to build the assets bundle for testing. '
189 'This takes additional time before running the tests. '
190 'Consider using "--no-test-assets" if assets are not required.',
191 )
192 // --platform is not supported to be used by Flutter developers. It only
193 // exists to test the Flutter framework itself and may be removed entirely
194 // in the future. Developers should either use plain `flutter test`, or
195 // `package:integration_test` instead.
196 ..addOption(
197 'platform',
198 allowed: const <String>['tester', 'chrome'],
199 hide: !verboseHelp,
200 defaultsTo: 'tester',
201 help: 'Selects the test backend.',
202 allowedHelp: <String, String>{
203 'tester': 'Run tests using the VM-based test environment.',
204 'chrome':
205 '(deprecated) Run tests using the Google Chrome web browser. '
206 'This value is intended for testing the Flutter framework '
207 'itself and may be removed at any time.',
208 },
209 )
210 ..addOption(
211 'test-randomize-ordering-seed',
212 help:
213 'The seed to randomize the execution order of test cases within test files. '
214 'Must be a 32bit unsigned integer or the string "random", '
215 'which indicates that a seed should be selected randomly. '
216 'By default, tests run in the order they are declared.',
217 )
218 ..addOption(
219 'total-shards',
220 help:
221 'Tests can be sharded with the "--total-shards" and "--shard-index" '
222 'arguments, allowing you to split up your test suites and run '
223 'them separately.',
224 )
225 ..addOption(
226 'shard-index',
227 help:
228 'Tests can be sharded with the "--total-shards" and "--shard-index" '
229 'arguments, allowing you to split up your test suites and run '
230 'them separately.',
231 )
232 ..addFlag(
233 'enable-vmservice',
234 hide: !verboseHelp,
235 help:
236 'Enables the VM service without "--start-paused". This flag is '
237 'intended for use with tests that will use "dart:developer" to '
238 'interact with the VM service at runtime.\n'
239 'This flag is ignored if "--start-paused" or coverage are requested, as '
240 'the VM service will be enabled in those cases regardless.',
241 )
242 ..addOption(
243 'reporter',
244 abbr: 'r',
245 help:
246 'Set how to print test results. If unset, value will default to either compact or expanded.',
247 allowed: <String>['compact', 'expanded', 'failures-only', 'github', 'json', 'silent'],
248 allowedHelp: <String, String>{
249 'compact': 'A single line, updated continuously (the default).',
250 'expanded':
251 'A separate line for each update. May be preferred when logging to a file or in continuous integration.',
252 'failures-only': 'A separate line for failing tests, with no output for passing tests.',
253 'github':
254 'A custom reporter for GitHub Actions (the default reporter when running on GitHub Actions).',
255 'json': 'A machine-readable format. See: https://dart.dev/go/test-docs/json_reporter.md',
256 'silent':
257 'A reporter with no output. May be useful when only the exit code is meaningful.',
258 },
259 )
260 ..addOption(
261 'file-reporter',
262 help:
263 'Enable an additional reporter writing test results to a file.\n'
264 'Should be in the form <reporter>:<filepath>, '
265 'Example: "json:reports/tests.json".',
266 )
267 ..addOption(
268 'timeout',
269 help:
270 'The default timeout for individual tests, specified either in '
271 'seconds (e.g. "60s"), as a multiplier of the default test timeout '
272 '(e.g. "2x"), or as the string "none" to disable test timeouts '
273 'entirely. This value does not apply to the default test suite '
274 'loading timeout.',
275 )
276 ..addFlag(
277 'ignore-timeouts',
278 help:
279 'Ignore all timeouts. Useful when testing a big application '
280 'that requires a longer time to compile (e.g. running integration '
281 'tests for a Flutter app).',
282 negatable: false,
283 )
284 ..addFlag(
285 FlutterOptions.kWebWasmFlag,
286 help: 'Compile to WebAssembly rather than JavaScript.\n$kWasmMoreInfo',
287 negatable: false,
288 );
289
290 addDdsOptions(verboseHelp: verboseHelp);
291 usesFatalWarningsOption(verboseHelp: verboseHelp);
292 }
293
294 /// The interface for starting and configuring the tester.
295 final TestWrapper testWrapper;
296
297 /// Interface for running the tester process.
298 final FlutterTestRunner testRunner;
299
300 final TestCompilerNativeAssetsBuilder? nativeAssetsBuilder;
301
302 final bool verbose;
303
304 @visibleForTesting
305 bool get isIntegrationTest => _isIntegrationTest;
306 var _isIntegrationTest = false;
307
308 final _testFileUris = <Uri>{};
309
310 bool get isWeb => stringArg('platform') == 'chrome';
311 bool get useWasm => boolArg(FlutterOptions.kWebWasmFlag);
312
313 @override
314 Future<Set<DevelopmentArtifact>> get requiredArtifacts async {
315 final Set<DevelopmentArtifact> results = _isIntegrationTest
316 // Use [DeviceBasedDevelopmentArtifacts].
317 ? await super.requiredArtifacts
318 : <DevelopmentArtifact>{};
319 if (isWeb) {
320 results.add(DevelopmentArtifact.web);
321 }
322 return results;
323 }
324
325 @override
326 String get name => 'test';
327
328 @override
329 String get description => 'Run Flutter unit tests for the current project.';
330
331 @override
332 String get category => FlutterCommandCategory.project;
333
334 @override
335 Future<FlutterCommandResult> verifyThenRunCommand(String? commandPath) {
336 final List<Uri> testUris = argResults!.rest.map(_parseTestArgument).toList();
337 if (testUris.isEmpty) {
338 // We don't scan the entire package, only the test/ subdirectory, so that
339 // files with names like "hit_test.dart" don't get run.
340 final Directory testDir = globals.fs.directory('test');
341 if (!testDir.existsSync()) {
342 throwToolExit('Test directory "${testDir.path}" not found.');
343 }
344 _testFileUris.addAll(_findTests(testDir).map(Uri.file));
345 if (_testFileUris.isEmpty) {
346 throwToolExit(
347 'Test directory "${testDir.path}" does not appear to contain any test files.\n'
348 'Test files must be in that directory and end with the pattern "_test.dart".',
349 );
350 }
351 } else {
352 for (final uri in testUris) {
353 // Test files may have query strings to support name/line/col:
354 // flutter test test/foo.dart?name=a&line=1
355 String testPath = uri.replace(query: '').toFilePath();
356 testPath = globals.fs.path.absolute(testPath);
357 testPath = globals.fs.path.normalize(testPath);
358 if (globals.fs.isDirectorySync(testPath)) {
359 _testFileUris.addAll(_findTests(globals.fs.directory(testPath)).map(Uri.file));
360 } else {
361 _testFileUris.add(Uri.file(testPath).replace(query: uri.query));
362 }
363 }
364 }
365
366 // This needs to be set before [super.verifyThenRunCommand] so that the
367 // correct [requiredArtifacts] can be identified before [run] takes place.
368 final List<String> testFilePaths = _testFileUris
369 .map((Uri uri) => uri.replace(query: '').toFilePath())
370 .toList();
371 _isIntegrationTest = _shouldRunAsIntegrationTests(
372 globals.fs.currentDirectory.absolute.path,
373 testFilePaths,
374 );
375
376 globals.printTrace(
377 'Found ${_testFileUris.length} files which will be executed as '
378 '${_isIntegrationTest ? 'Integration' : 'Widget'} Tests.',
379 );
380 return super.verifyThenRunCommand(commandPath);
381 }
382
383 // Keep in sync with the [RunCommandBase.webRenderer] getter.
384 WebRendererMode get webRenderer {
385 final List<String> dartDefines = extractDartDefines(
386 defineConfigJsonMap: extractDartDefineConfigJsonMap(),
387 );
388 return WebRendererMode.fromDartDefines(dartDefines, useWasm: useWasm);
389 }
390
391 @override
392 Future<FlutterCommandResult> runCommand() async {
393 if (!globals.fs.isFileSync('pubspec.yaml')) {
394 throwToolExit(
395 'Error: No pubspec.yaml file found in the current working directory.\n'
396 'Run this command from the root of your project. Test files must be '
397 "called *_test.dart and must reside in the package's 'test' "
398 'directory (or one of its subdirectories).',
399 );
400 }
401 final FlutterProject flutterProject = FlutterProject.current();
402 final bool buildTestAssets = boolArg('test-assets');
403 final List<String> names = stringsArg('name');
404 final List<String> plainNames = stringsArg('plain-name');
405 final String? tags = stringArg('tags');
406 final String? excludeTags = stringArg('exclude-tags');
407 final BuildInfo buildInfo = await getBuildInfo(
408 forcedBuildMode: BuildMode.debug,
409 forcedUseLocalCanvasKit: true,
410 );
411
412 TestTimeRecorder? testTimeRecorder;
413 if (verbose) {
414 testTimeRecorder = TestTimeRecorder(globals.logger);
415 }
416
417 if (buildInfo.packageConfig['test_api'] == null) {
418 throwToolExit(
419 'Error: cannot run without a dependency on either "package:flutter_test" or "package:test". '
420 'Ensure the following lines are present in your pubspec.yaml:'
421 '\n\n'
422 'dev_dependencies:\n'
423 ' flutter_test:\n'
424 ' sdk: flutter\n',
425 );
426 }
427
428 bool experimentalFasterTesting = boolArg('experimental-faster-testing');
429 if (experimentalFasterTesting) {
430 if (_isIntegrationTest || isWeb) {
431 experimentalFasterTesting = false;
432 globals.printStatus(
433 '--experimental-faster-testing was parsed but will be ignored. This '
434 'option is not supported when running integration tests or web tests.',
435 );
436 } else if (_testFileUris.length == 1) {
437 experimentalFasterTesting = false;
438 globals.printStatus(
439 '--experimental-faster-testing was parsed but will be ignored. This '
440 'option should not be used when running a single test file.',
441 );
442 }
443 }
444
445 final bool startPaused = boolArg('start-paused');
446 if (startPaused && _testFileUris.length != 1) {
447 throwToolExit(
448 'When using --start-paused, you must specify a single test file to run.',
449 exitCode: 1,
450 );
451 }
452
453 final debuggingOptions = DebuggingOptions.enabled(
454 buildInfo,
455 startPaused: startPaused,
456 disableServiceAuthCodes: boolArg('disable-service-auth-codes'),
457 // On iOS >=14, keeping this enabled will leave a prompt on the screen.
458 disablePortPublication: true,
459 enableDds: enableDds,
460 usingCISystem: usingCISystem,
461 enableImpeller: ImpellerStatus.fromBool(argResults!['enable-impeller'] as bool?),
462 enableFlutterGpu: (argResults!['enable-flutter-gpu'] as bool?) ?? false,
463 debugLogsDirectoryPath: debugLogsDirectoryPath,
464 webRenderer: webRenderer,
465 printDtd: boolArg(FlutterGlobalOptions.kPrintDtd, global: true),
466 webUseWasm: useWasm,
467 );
468
469 final Uri? nativeAssetsJson = _isIntegrationTest
470 ? null // Don't build for host when running integration tests.
471 : await nativeAssetsBuilder?.build(buildInfo);
472 String? testAssetPath;
473 if (buildTestAssets) {
474 await _buildTestAsset(
475 flavor: buildInfo.flavor,
476 impellerStatus: debuggingOptions.enableImpeller,
477 buildMode: debuggingOptions.buildInfo.mode,
478 packageConfigPath: buildInfo.packageConfigPath,
479 );
480 }
481 if (buildTestAssets || nativeAssetsJson != null) {
482 testAssetPath = globals.fs.path.join(
483 flutterProject.directory.path,
484 'build',
485 'unit_test_assets',
486 );
487 }
488 if (nativeAssetsJson != null) {
489 final Directory testAssetDirectory = globals.fs.directory(testAssetPath);
490 if (!testAssetDirectory.existsSync()) {
491 await testAssetDirectory.create(recursive: true);
492 }
493 final File nativeAssetsManifest = testAssetDirectory.childFile('NativeAssetsManifest.json');
494 await globals.fs.file(nativeAssetsJson).copy(nativeAssetsManifest.path);
495 }
496
497 final String? concurrencyString = stringArg('concurrency');
498 int? jobs = concurrencyString == null ? null : int.tryParse(concurrencyString);
499 if (jobs != null && (jobs <= 0 || !jobs.isFinite)) {
500 throwToolExit(
501 'Could not parse -j/--concurrency argument. It must be an integer greater than zero.',
502 );
503 }
504
505 if (_isIntegrationTest || isWeb) {
506 if (argResults!.wasParsed('concurrency')) {
507 globals.printStatus(
508 '-j/--concurrency was parsed but will be ignored, this option is not '
509 'supported when running Integration Tests or web tests.',
510 );
511 }
512 // Running with concurrency will result in deploying multiple test apps
513 // on the connected device concurrently, which is not supported.
514 jobs = 1;
515 } else if (experimentalFasterTesting) {
516 if (argResults!.wasParsed('concurrency')) {
517 globals.printStatus(
518 '-j/--concurrency was parsed but will be ignored. This option is not '
519 'compatible with --experimental-faster-testing.',
520 );
521 }
522 }
523
524 final int? shardIndex = int.tryParse(stringArg('shard-index') ?? '');
525 if (shardIndex != null && (shardIndex < 0 || !shardIndex.isFinite)) {
526 throwToolExit(
527 'Could not parse --shard-index=$shardIndex argument. It must be an integer greater than -1.',
528 );
529 }
530
531 final int? totalShards = int.tryParse(stringArg('total-shards') ?? '');
532 if (totalShards != null && (totalShards <= 0 || !totalShards.isFinite)) {
533 throwToolExit(
534 'Could not parse --total-shards=$totalShards argument. It must be an integer greater than zero.',
535 );
536 }
537
538 if (totalShards != null && shardIndex == null) {
539 throwToolExit('If you set --total-shards you need to also set --shard-index.');
540 }
541 if (shardIndex != null && totalShards == null) {
542 throwToolExit('If you set --shard-index you need to also set --total-shards.');
543 }
544
545 final bool enableVmService = boolArg('enable-vmservice');
546 if (experimentalFasterTesting && enableVmService) {
547 globals.printStatus(
548 '--enable-vmservice was parsed but will be ignored. This option is not '
549 'compatible with --experimental-faster-testing.',
550 );
551 }
552
553 final bool ipv6 = boolArg('ipv6');
554 if (experimentalFasterTesting && ipv6) {
555 // [ipv6] is set when the user desires for the test harness server to use
556 // IPv6, but a test harness server will not be started at all when
557 // [experimentalFasterTesting] is set.
558 globals.printStatus(
559 '--ipv6 was parsed but will be ignored. This option is not compatible '
560 'with --experimental-faster-testing.',
561 );
562 }
563
564 CoverageCollector? collector;
565 if (boolArg('coverage') || boolArg('merge-coverage') || boolArg('branch-coverage')) {
566 final Set<String> packagesToInclude = _getCoveragePackages(
567 stringsArg('coverage-package'),
568 flutterProject,
569 buildInfo.packageConfig,
570 );
571 collector = CoverageCollector(
572 verbose: !outputMachineFormat,
573 libraryNames: packagesToInclude,
574 packagesPath: buildInfo.packageConfigPath,
575 resolver: await CoverageCollector.getResolver(buildInfo.packageConfigPath),
576 testTimeRecorder: testTimeRecorder,
577 branchCoverage: boolArg('branch-coverage'),
578 );
579 }
580
581 TestWatcher? watcher;
582 if (outputMachineFormat) {
583 watcher = EventPrinter(parent: collector, out: globals.stdio.stdout);
584 } else if (collector != null) {
585 watcher = collector;
586 }
587
588 if (!isWeb && useWasm) {
589 throwToolExit('--wasm is only supported on the web platform');
590 }
591
592 if (webRenderer == WebRendererMode.skwasm && !useWasm) {
593 throwToolExit('Skwasm renderer requires --wasm');
594 }
595
596 Device? integrationTestDevice;
597 if (_isIntegrationTest) {
598 integrationTestDevice = await findTargetDevice();
599
600 // Disable reporting of test results to native test frameworks. This isn't
601 // needed as the Flutter Tool will be responsible for reporting results.
602 buildInfo.dartDefines.add('INTEGRATION_TEST_SHOULD_REPORT_RESULTS_TO_NATIVE=false');
603
604 if (integrationTestDevice == null) {
605 throwToolExit(
606 'No devices are connected. '
607 'Ensure that `flutter doctor` shows at least one connected device',
608 );
609 }
610 if (integrationTestDevice.platformType == PlatformType.web) {
611 // TODO(jiahaog): Support web. https://github.com/flutter/flutter/issues/66264
612 throwToolExit('Web devices are not supported for integration tests yet.');
613 }
614
615 if (buildInfo.packageConfig['integration_test'] == null) {
616 throwToolExit(
617 'Error: cannot run without a dependency on "package:integration_test". '
618 'Ensure the following lines are present in your pubspec.yaml:'
619 '\n\n'
620 'dev_dependencies:\n'
621 ' integration_test:\n'
622 ' sdk: flutter\n',
623 );
624 }
625
626 if (stringArg('flavor') != null && !integrationTestDevice.supportsFlavors) {
627 throwToolExit('--flavor is only supported for Android, macOS, and iOS devices.');
628 }
629 }
630
631 final Stopwatch? testRunnerTimeRecorderStopwatch = testTimeRecorder?.start(
632 TestTimePhases.TestRunner,
633 );
634 final int result;
635 if (experimentalFasterTesting) {
636 assert(!isWeb && !_isIntegrationTest && _testFileUris.length > 1);
637 result = await testRunner.runTestsBySpawningLightweightEngines(
638 _testFileUris.toList(),
639 debuggingOptions: debuggingOptions,
640 names: names,
641 plainNames: plainNames,
642 tags: tags,
643 excludeTags: excludeTags,
644 machine: outputMachineFormat,
645 updateGoldens: boolArg('update-goldens'),
646 concurrency: jobs,
647 testAssetDirectory: testAssetPath,
648 flutterProject: flutterProject,
649 randomSeed: stringArg('test-randomize-ordering-seed'),
650 reporter: stringArg('reporter'),
651 fileReporter: stringArg('file-reporter'),
652 timeout: stringArg('timeout'),
653 ignoreTimeouts: boolArg('ignore-timeouts'),
654 failFast: boolArg('fail-fast'),
655 runSkipped: boolArg('run-skipped'),
656 shardIndex: shardIndex,
657 totalShards: totalShards,
658 testTimeRecorder: testTimeRecorder,
659 nativeAssetsBuilder: nativeAssetsBuilder,
660 );
661 } else {
662 result = await testRunner.runTests(
663 testWrapper,
664 _testFileUris.toList(),
665 debuggingOptions: debuggingOptions,
666 names: names,
667 plainNames: plainNames,
668 tags: tags,
669 excludeTags: excludeTags,
670 watcher: watcher,
671 enableVmService: collector != null || startPaused || enableVmService,
672 machine: outputMachineFormat,
673 updateGoldens: boolArg('update-goldens'),
674 concurrency: jobs,
675 testAssetDirectory: testAssetPath,
676 flutterProject: flutterProject,
677 web: isWeb,
678 randomSeed: stringArg('test-randomize-ordering-seed'),
679 reporter: stringArg('reporter'),
680 fileReporter: stringArg('file-reporter'),
681 timeout: stringArg('timeout'),
682 ignoreTimeouts: boolArg('ignore-timeouts'),
683 failFast: boolArg('fail-fast'),
684 runSkipped: boolArg('run-skipped'),
685 shardIndex: shardIndex,
686 totalShards: totalShards,
687 integrationTestDevice: integrationTestDevice,
688 integrationTestUserIdentifier: stringArg(FlutterOptions.kDeviceUser),
689 testTimeRecorder: testTimeRecorder,
690 nativeAssetsBuilder: nativeAssetsBuilder,
691 buildInfo: buildInfo,
692 );
693 }
694 testTimeRecorder?.stop(TestTimePhases.TestRunner, testRunnerTimeRecorderStopwatch!);
695
696 if (collector != null) {
697 final Stopwatch? collectTimeRecorderStopwatch = testTimeRecorder?.start(
698 TestTimePhases.CoverageDataCollect,
699 );
700 final bool collectionResult = await collector.collectCoverageData(
701 stringArg('coverage-path'),
702 mergeCoverageData: boolArg('merge-coverage'),
703 );
704 testTimeRecorder?.stop(TestTimePhases.CoverageDataCollect, collectTimeRecorderStopwatch!);
705 if (!collectionResult) {
706 testTimeRecorder?.print();
707 throwToolExit(null);
708 }
709 }
710
711 testTimeRecorder?.print();
712
713 if (result != 0) {
714 throwToolExit(null, exitCode: result);
715 }
716 return FlutterCommandResult.success();
717 }
718
719 Set<String> _getCoveragePackages(
720 List<String> packagesRegExps,
721 FlutterProject flutterProject,
722 PackageConfig packageConfig,
723 ) {
724 final packagesToInclude = <String>{};
725 if (packagesRegExps.isEmpty) {
726 void addProject(FlutterProject project) {
727 packagesToInclude.add(project.manifest.appName);
728 project.workspaceProjects.forEach(addProject);
729 }
730
731 addProject(flutterProject);
732 }
733 try {
734 for (final regExpStr in packagesRegExps) {
735 final regExp = RegExp(regExpStr);
736 packagesToInclude.addAll(
737 packageConfig.packages.map((Package e) => e.name).where((String e) => regExp.hasMatch(e)),
738 );
739 }
740 } on FormatException catch (e) {
741 throwToolExit('Regular expression syntax is invalid. $e');
742 }
743 return packagesToInclude;
744 }
745
746 /// Parses a test file/directory target passed as an argument and returns it
747 /// as an absolute `file:///` [Uri] with optional querystring for name/line/col.
748 Uri _parseTestArgument(String arg) {
749 // We can't parse Windows paths as URIs if they have query strings, so
750 // parse the file and query parts separately.
751 final int queryStart = arg.indexOf('?');
752 String filePart = queryStart == -1 ? arg : arg.substring(0, queryStart);
753 final String queryPart = queryStart == -1 ? '' : arg.substring(queryStart + 1);
754
755 filePart = globals.fs.path.absolute(filePart);
756 filePart = globals.fs.path.normalize(filePart);
757
758 return Uri.file(filePart).replace(query: queryPart.isEmpty ? null : queryPart);
759 }
760
761 Future<void> _buildTestAsset({
762 required String? flavor,
763 required ImpellerStatus impellerStatus,
764 required BuildMode buildMode,
765 required String packageConfigPath,
766 }) async {
767 final AssetBundle assetBundle = AssetBundleFactory.instance.createBundle();
768 final int build = await assetBundle.build(
769 packageConfigPath: packageConfigPath,
770 flavor: flavor,
771 includeAssetsFromDevDependencies: true,
772 );
773 if (build != 0) {
774 throwToolExit('Error: Failed to build asset bundle');
775 }
776 if (_needsRebuild(assetBundle.entries, flavor)) {
777 await writeBundle(
778 globals.fs.directory(globals.fs.path.join('build', 'unit_test_assets')),
779 assetBundle.entries,
780 targetPlatform: TargetPlatform.tester,
781 impellerStatus: impellerStatus,
782 processManager: globals.processManager,
783 fileSystem: globals.fs,
784 artifacts: globals.artifacts!,
785 logger: globals.logger,
786 projectDir: globals.fs.currentDirectory,
787 buildMode: buildMode,
788 );
789
790 final File cachedFlavorFile = globals.fs.file(
791 globals.fs.path.join('build', 'test_cache', 'flavor.txt'),
792 );
793 if (cachedFlavorFile.existsSync()) {
794 await cachedFlavorFile.delete();
795 }
796 if (flavor != null) {
797 cachedFlavorFile.createSync(recursive: true);
798 cachedFlavorFile.writeAsStringSync(flavor);
799 }
800 }
801 }
802
803 bool _needsRebuild(Map<String, AssetBundleEntry> entries, String? flavor) {
804 // TODO(andrewkolos): This logic might fail in the future if we change the
805 // schema of the contents of the asset manifest file and the user does not
806 // perform a `flutter clean` after upgrading.
807 // See https://github.com/flutter/flutter/issues/128563.
808 final File manifest = globals.fs.file(
809 globals.fs.path.join('build', 'unit_test_assets', 'AssetManifest.bin'),
810 );
811 if (!manifest.existsSync()) {
812 return true;
813 }
814 final DateTime lastModified = manifest.lastModifiedSync();
815 final File pub = globals.fs.file('pubspec.yaml');
816 if (pub.lastModifiedSync().isAfter(lastModified)) {
817 return true;
818 }
819
820 final Iterable<DevFSFileContent> files = entries.values
821 .map((AssetBundleEntry asset) => asset.content)
822 .whereType<DevFSFileContent>();
823 for (final entry in files) {
824 // Calling isModified to access file stats first in order for isModifiedAfter
825 // to work.
826 if (entry.isModified && entry.isModifiedAfter(lastModified)) {
827 return true;
828 }
829 }
830
831 final File cachedFlavorFile = globals.fs.file(
832 globals.fs.path.join('build', 'test_cache', 'flavor.txt'),
833 );
834 final String? cachedFlavor = cachedFlavorFile.existsSync()
835 ? cachedFlavorFile.readAsStringSync()
836 : null;
837 if (cachedFlavor != flavor) {
838 return true;
839 }
840
841 return false;
842 }
843}
844
845/// Searches [directory] and returns files that end with `_test.dart` as
846/// absolute paths.
847Iterable<String> _findTests(Directory directory) {
848 return directory
849 .listSync(recursive: true, followLinks: false)
850 .where(
851 (FileSystemEntity entity) =>
852 entity.path.endsWith('_test.dart') && globals.fs.isFileSync(entity.path),
853 )
854 .map((FileSystemEntity entity) => globals.fs.path.absolute(entity.path));
855}
856
857/// Returns true if there are files that are Integration Tests.
858///
859/// The [currentDirectory] and [testFiles] parameters here must be provided as
860/// absolute paths.
861///
862/// Throws an exception if there are both Integration Tests and Widget Tests
863/// found in [testFiles].
864bool _shouldRunAsIntegrationTests(String currentDirectory, List<String> testFiles) {
865 final String integrationTestDirectory = globals.fs.path.join(
866 currentDirectory,
867 _kIntegrationTestDirectory,
868 );
869
870 if (testFiles.every(
871 (String absolutePath) => !absolutePath.startsWith(integrationTestDirectory),
872 )) {
873 return false;
874 }
875
876 if (testFiles.every((String absolutePath) => absolutePath.startsWith(integrationTestDirectory))) {
877 return true;
878 }
879
880 throwToolExit(
881 'Integration tests and unit tests cannot be run in a single invocation.'
882 ' Use separate invocations of `flutter test` to run integration tests'
883 ' and unit tests.',
884 );
885}
886