1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5import 'dart:async';
6import 'dart:convert';
7import 'dart:io';
8
9import 'package:meta/meta.dart';
10import 'package:vm_service/vm_service.dart';
11import 'package:vm_service/vm_service_io.dart';
12
13import 'devices.dart';
14import 'metrics_result_writer.dart';
15import 'task_result.dart';
16import 'utils.dart';
17
18/// Run a list of tasks.
19///
20/// For each task, an auto rerun will be triggered when task fails.
21///
22/// If the task succeeds the first time, it will be recorded as successful.
23///
24/// If the task fails first, but gets passed in the end, the
25/// test will be recorded as successful but with a flake flag.
26///
27/// If the task fails all reruns, it will be recorded as failed.
28Future<void> runTasks(
29 List<String> taskNames, {
30 bool exitOnFirstTestFailure = false,
31 // terminateStrayDartProcesses defaults to false so that tests don't have to specify it.
32 // It is set based on the --terminate-stray-dart-processes command line argument in
33 // normal execution, and that flag defaults to true.
34 bool terminateStrayDartProcesses = false,
35 bool silent = false,
36 String? deviceId,
37 String? gitBranch,
38 String? localEngine,
39 String? localEngineHost,
40 String? localEngineSrcPath,
41 String? luciBuilder,
42 String? resultsPath,
43 List<String>? taskArgs,
44 bool useEmulator = false,
45 @visibleForTesting Map<String, String>? isolateParams,
46 @visibleForTesting void Function(String) print = print,
47 @visibleForTesting List<String>? logs,
48}) async {
49 for (final String taskName in taskNames) {
50 TaskResult result = TaskResult.success(null);
51 int failureCount = 0;
52 while (failureCount <= MetricsResultWriter.retryNumber) {
53 result = await rerunTask(
54 taskName,
55 deviceId: deviceId,
56 localEngine: localEngine,
57 localEngineHost: localEngineHost,
58 localEngineSrcPath: localEngineSrcPath,
59 terminateStrayDartProcesses: terminateStrayDartProcesses,
60 silent: silent,
61 taskArgs: taskArgs,
62 resultsPath: resultsPath,
63 gitBranch: gitBranch,
64 luciBuilder: luciBuilder,
65 isolateParams: isolateParams,
66 useEmulator: useEmulator,
67 );
68
69 if (!result.succeeded) {
70 failureCount += 1;
71 if (exitOnFirstTestFailure) {
72 break;
73 }
74 } else {
75 section('Flaky status for "$taskName"');
76 if (failureCount > 0) {
77 print(
78 'Total ${failureCount + 1} executions: $failureCount failures and 1 false positive.',
79 );
80 print('flaky: true');
81 // TODO(ianh): stop ignoring this failure. We should set exitCode=1, and quit
82 // if exitOnFirstTestFailure is true.
83 } else {
84 print('Test passed on first attempt.');
85 print('flaky: false');
86 }
87 break;
88 }
89 }
90
91 if (!result.succeeded) {
92 section('Flaky status for "$taskName"');
93 print('Consistently failed across all $failureCount executions.');
94 print('flaky: false');
95 exitCode = 1;
96 if (exitOnFirstTestFailure) {
97 return;
98 }
99 }
100 }
101}
102
103/// A rerun wrapper for `runTask`.
104///
105/// This separates reruns in separate sections.
106Future<TaskResult> rerunTask(
107 String taskName, {
108 String? deviceId,
109 String? localEngine,
110 String? localEngineHost,
111 String? localEngineSrcPath,
112 bool terminateStrayDartProcesses = false,
113 bool silent = false,
114 List<String>? taskArgs,
115 String? resultsPath,
116 String? gitBranch,
117 String? luciBuilder,
118 bool useEmulator = false,
119 @visibleForTesting Map<String, String>? isolateParams,
120}) async {
121 section('Running task "$taskName"');
122 final TaskResult result = await runTask(
123 taskName,
124 deviceId: deviceId,
125 localEngine: localEngine,
126 localEngineHost: localEngineHost,
127 localEngineSrcPath: localEngineSrcPath,
128 terminateStrayDartProcesses: terminateStrayDartProcesses,
129 silent: silent,
130 taskArgs: taskArgs,
131 isolateParams: isolateParams,
132 useEmulator: useEmulator,
133 );
134
135 print('Task result:');
136 print(const JsonEncoder.withIndent(' ').convert(result));
137 section('Finished task "$taskName"');
138
139 if (resultsPath != null) {
140 final MetricsResultWriter cocoon = MetricsResultWriter();
141 await cocoon.writeTaskResultToFile(
142 builderName: luciBuilder,
143 gitBranch: gitBranch,
144 result: result,
145 resultsPath: resultsPath,
146 );
147 }
148 return result;
149}
150
151/// Runs a task in a separate Dart VM and collects the result using the VM
152/// service protocol.
153///
154/// [taskName] is the name of the task. The corresponding task executable is
155/// expected to be found under `bin/tasks`.
156///
157/// Running the task in [silent] mode will suppress standard output from task
158/// processes and only print standard errors.
159///
160/// [taskArgs] are passed to the task executable for additional configuration.
161Future<TaskResult> runTask(
162 String taskName, {
163 bool terminateStrayDartProcesses = false,
164 bool silent = false,
165 String? localEngine,
166 String? localEngineHost,
167 String? localWebSdk,
168 String? localEngineSrcPath,
169 String? deviceId,
170 List<String>? taskArgs,
171 bool useEmulator = false,
172 @visibleForTesting Map<String, String>? isolateParams,
173}) async {
174 final String taskExecutable = 'bin/tasks/$taskName.dart';
175
176 if (!file(taskExecutable).existsSync()) {
177 print('Executable Dart file not found: $taskExecutable');
178 exit(1);
179 }
180
181 if (useEmulator) {
182 taskArgs ??= <String>[];
183 taskArgs
184 ..add('--android-emulator')
185 ..add('--browser-name=android-chrome');
186 }
187
188 stdout.writeln('Starting process for task: [$taskName]');
189
190 final Process runner = await startProcess(
191 dartBin,
192 <String>[
193 '--enable-vm-service=0', // zero causes the system to choose a free port
194 '--no-pause-isolates-on-exit',
195 if (localEngine != null) '-DlocalEngine=$localEngine',
196 if (localEngineHost != null) '-DlocalEngineHost=$localEngineHost',
197 if (localWebSdk != null) '-DlocalWebSdk=$localWebSdk',
198 if (localEngineSrcPath != null) '-DlocalEngineSrcPath=$localEngineSrcPath',
199 taskExecutable,
200 ...?taskArgs,
201 ],
202 environment: <String, String>{if (deviceId != null) DeviceIdEnvName: deviceId},
203 );
204
205 bool runnerFinished = false;
206
207 unawaited(
208 runner.exitCode.whenComplete(() {
209 runnerFinished = true;
210 }),
211 );
212
213 final Completer<Uri> uri = Completer<Uri>();
214
215 final StreamSubscription<String> stdoutSub = runner.stdout
216 .transform<String>(const Utf8Decoder())
217 .transform<String>(const LineSplitter())
218 .listen((String line) {
219 if (!uri.isCompleted) {
220 final Uri? serviceUri = parseServiceUri(
221 line,
222 prefix: RegExp('The Dart VM service is listening on '),
223 );
224 if (serviceUri != null) {
225 uri.complete(serviceUri);
226 }
227 }
228 if (!silent) {
229 stdout.writeln('[${DateTime.now()}] [STDOUT] $line');
230 }
231 });
232
233 final StreamSubscription<String> stderrSub = runner.stderr
234 .transform<String>(const Utf8Decoder())
235 .transform<String>(const LineSplitter())
236 .listen((String line) {
237 stderr.writeln('[${DateTime.now()}] [STDERR] $line');
238 });
239
240 try {
241 final ConnectionResult result = await _connectToRunnerIsolate(await uri.future);
242 print('[$taskName] Connected to VM server.');
243 isolateParams = isolateParams == null
244 ? <String, String>{}
245 : Map<String, String>.of(isolateParams);
246 isolateParams['runProcessCleanup'] = terminateStrayDartProcesses.toString();
247 final VmService service = result.vmService;
248 final String isolateId = result.isolate.id!;
249 final Map<String, dynamic> taskResultJson = (await service.callServiceExtension(
250 'ext.cocoonRunTask',
251 args: isolateParams,
252 isolateId: isolateId,
253 )).json!;
254 // Notify the task process that the task result has been received and it
255 // can proceed to shutdown.
256 await _acknowledgeTaskResultReceived(service: service, isolateId: isolateId);
257 final TaskResult taskResult = TaskResult.fromJson(taskResultJson);
258 final int exitCode = await runner.exitCode;
259 print('[$taskName] Process terminated with exit code $exitCode.');
260 return taskResult;
261 } catch (error, stack) {
262 print('[$taskName] Task runner system failed with exception!\n$error\n$stack');
263 rethrow;
264 } finally {
265 if (!runnerFinished) {
266 print('[$taskName] Terminating process...');
267 runner.kill(ProcessSignal.sigkill);
268 }
269 await stdoutSub.cancel();
270 await stderrSub.cancel();
271 }
272}
273
274Future<ConnectionResult> _connectToRunnerIsolate(Uri vmServiceUri) async {
275 final List<String> pathSegments = <String>[
276 // Add authentication code.
277 if (vmServiceUri.pathSegments.isNotEmpty) vmServiceUri.pathSegments[0],
278 'ws',
279 ];
280 final String url = vmServiceUri.replace(scheme: 'ws', pathSegments: pathSegments).toString();
281 final Stopwatch stopwatch = Stopwatch()..start();
282
283 while (true) {
284 try {
285 // Look up the isolate.
286 final VmService client = await vmServiceConnectUri(url);
287 VM vm = await client.getVM();
288 while (vm.isolates!.isEmpty) {
289 await Future<void>.delayed(const Duration(seconds: 1));
290 vm = await client.getVM();
291 }
292 final IsolateRef isolate = vm.isolates!.first;
293 // Sanity check to ensure we're talking with the main isolate.
294 final Response response = await client.callServiceExtension(
295 'ext.cocoonRunnerReady',
296 isolateId: isolate.id,
297 );
298 if (response.json!['result'] != 'success') {
299 throw 'not ready yet';
300 }
301 return ConnectionResult(client, isolate);
302 } catch (error) {
303 if (stopwatch.elapsed > const Duration(seconds: 10)) {
304 print(
305 'VM service still not ready. It is possible the target has failed.\n'
306 'Latest connection error:\n'
307 ' $error\n'
308 'Continuing to retry...\n',
309 );
310 stopwatch.reset();
311 }
312 await Future<void>.delayed(const Duration(milliseconds: 50));
313 }
314 }
315}
316
317Future<void> _acknowledgeTaskResultReceived({
318 required VmService service,
319 required String isolateId,
320}) async {
321 try {
322 await service.callServiceExtension('ext.cocoonTaskResultReceived', isolateId: isolateId);
323 } on RPCError {
324 // The target VM may shutdown before the response is received.
325 }
326}
327
328class ConnectionResult {
329 ConnectionResult(this.vmService, this.isolate);
330
331 final VmService vmService;
332 final IsolateRef isolate;
333}
334