1 | // Copyright 2014 The Flutter Authors. All rights reserved. |
2 | // Use of this source code is governed by a BSD-style license that can be |
3 | // found in the LICENSE file. |
4 | |
5 | import 'dart:async'; |
6 | import 'dart:convert'; |
7 | import 'dart:io'; |
8 | |
9 | import 'package:meta/meta.dart' ; |
10 | import 'package:vm_service/vm_service.dart' ; |
11 | import 'package:vm_service/vm_service_io.dart' ; |
12 | |
13 | import 'cocoon.dart'; |
14 | import 'devices.dart'; |
15 | import 'task_result.dart'; |
16 | import 'utils.dart'; |
17 | |
18 | /// Run a list of tasks. |
19 | /// |
20 | /// For each task, an auto rerun will be triggered when task fails. |
21 | /// |
22 | /// If the task succeeds the first time, it will be recorded as successful. |
23 | /// |
24 | /// If the task fails first, but gets passed in the end, the |
25 | /// test will be recorded as successful but with a flake flag. |
26 | /// |
27 | /// If the task fails all reruns, it will be recorded as failed. |
28 | Future<void> runTasks( |
29 | List<String> taskNames, { |
30 | bool exitOnFirstTestFailure = false, |
31 | // terminateStrayDartProcesses defaults to false so that tests don't have to specify it. |
32 | // It is set based on the --terminate-stray-dart-processes command line argument in |
33 | // normal execution, and that flag defaults to true. |
34 | bool terminateStrayDartProcesses = false, |
35 | bool silent = false, |
36 | String? deviceId, |
37 | String? gitBranch, |
38 | String? localEngine, |
39 | String? localEngineHost, |
40 | String? localEngineSrcPath, |
41 | String? luciBuilder, |
42 | String? resultsPath, |
43 | List<String>? taskArgs, |
44 | bool useEmulator = false, |
45 | @visibleForTesting Map<String, String>? isolateParams, |
46 | @visibleForTesting void Function(String) print = print, |
47 | @visibleForTesting List<String>? logs, |
48 | }) async { |
49 | for (final String taskName in taskNames) { |
50 | TaskResult result = TaskResult.success(null); |
51 | int failureCount = 0; |
52 | while (failureCount <= Cocoon.retryNumber) { |
53 | result = await rerunTask( |
54 | taskName, |
55 | deviceId: deviceId, |
56 | localEngine: localEngine, |
57 | localEngineHost: localEngineHost, |
58 | localEngineSrcPath: localEngineSrcPath, |
59 | terminateStrayDartProcesses: terminateStrayDartProcesses, |
60 | silent: silent, |
61 | taskArgs: taskArgs, |
62 | resultsPath: resultsPath, |
63 | gitBranch: gitBranch, |
64 | luciBuilder: luciBuilder, |
65 | isolateParams: isolateParams, |
66 | useEmulator: useEmulator, |
67 | ); |
68 | |
69 | if (!result.succeeded) { |
70 | failureCount += 1; |
71 | if (exitOnFirstTestFailure) { |
72 | break; |
73 | } |
74 | } else { |
75 | section('Flaky status for " $taskName"' ); |
76 | if (failureCount > 0) { |
77 | print('Total ${failureCount+1} executions: $failureCount failures and 1 false positive.' ); |
78 | print('flaky: true' ); |
79 | // TODO(ianh): stop ignoring this failure. We should set exitCode=1, and quit |
80 | // if exitOnFirstTestFailure is true. |
81 | } else { |
82 | print('Test passed on first attempt.' ); |
83 | print('flaky: false' ); |
84 | } |
85 | break; |
86 | } |
87 | } |
88 | |
89 | if (!result.succeeded) { |
90 | section('Flaky status for " $taskName"' ); |
91 | print('Consistently failed across all $failureCount executions.' ); |
92 | print('flaky: false' ); |
93 | exitCode = 1; |
94 | if (exitOnFirstTestFailure) { |
95 | return; |
96 | } |
97 | } |
98 | } |
99 | } |
100 | |
101 | /// A rerun wrapper for `runTask`. |
102 | /// |
103 | /// This separates reruns in separate sections. |
104 | Future<TaskResult> rerunTask( |
105 | String taskName, { |
106 | String? deviceId, |
107 | String? localEngine, |
108 | String? localEngineHost, |
109 | String? localEngineSrcPath, |
110 | bool terminateStrayDartProcesses = false, |
111 | bool silent = false, |
112 | List<String>? taskArgs, |
113 | String? resultsPath, |
114 | String? gitBranch, |
115 | String? luciBuilder, |
116 | bool useEmulator = false, |
117 | @visibleForTesting Map<String, String>? isolateParams, |
118 | }) async { |
119 | section('Running task " $taskName"' ); |
120 | final TaskResult result = await runTask( |
121 | taskName, |
122 | deviceId: deviceId, |
123 | localEngine: localEngine, |
124 | localEngineHost: localEngineHost, |
125 | localEngineSrcPath: localEngineSrcPath, |
126 | terminateStrayDartProcesses: terminateStrayDartProcesses, |
127 | silent: silent, |
128 | taskArgs: taskArgs, |
129 | isolateParams: isolateParams, |
130 | useEmulator: useEmulator, |
131 | ); |
132 | |
133 | print('Task result:' ); |
134 | print(const JsonEncoder.withIndent(' ' ).convert(result)); |
135 | section('Finished task " $taskName"' ); |
136 | |
137 | if (resultsPath != null) { |
138 | final Cocoon cocoon = Cocoon(); |
139 | await cocoon.writeTaskResultToFile( |
140 | builderName: luciBuilder, |
141 | gitBranch: gitBranch, |
142 | result: result, |
143 | resultsPath: resultsPath, |
144 | ); |
145 | } |
146 | return result; |
147 | } |
148 | |
149 | /// Runs a task in a separate Dart VM and collects the result using the VM |
150 | /// service protocol. |
151 | /// |
152 | /// [taskName] is the name of the task. The corresponding task executable is |
153 | /// expected to be found under `bin/tasks`. |
154 | /// |
155 | /// Running the task in [silent] mode will suppress standard output from task |
156 | /// processes and only print standard errors. |
157 | /// |
158 | /// [taskArgs] are passed to the task executable for additional configuration. |
159 | Future<TaskResult> runTask( |
160 | String taskName, { |
161 | bool terminateStrayDartProcesses = false, |
162 | bool silent = false, |
163 | String? localEngine, |
164 | String? localEngineHost, |
165 | String? localWebSdk, |
166 | String? localEngineSrcPath, |
167 | String? deviceId, |
168 | List<String>? taskArgs, |
169 | bool useEmulator = false, |
170 | @visibleForTesting Map<String, String>? isolateParams, |
171 | }) async { |
172 | final String taskExecutable = 'bin/tasks/ $taskName.dart' ; |
173 | |
174 | if (!file(taskExecutable).existsSync()) { |
175 | print('Executable Dart file not found: $taskExecutable' ); |
176 | exit(1); |
177 | } |
178 | |
179 | if (useEmulator) { |
180 | taskArgs ??= <String>[]; |
181 | taskArgs |
182 | ..add('--android-emulator' ) |
183 | ..add('--browser-name=android-chrome' ); |
184 | } |
185 | |
186 | stdout.writeln('Starting process for task: [ $taskName]' ); |
187 | |
188 | final Process runner = await startProcess( |
189 | dartBin, |
190 | <String>[ |
191 | '--enable-vm-service=0' , // zero causes the system to choose a free port |
192 | '--no-pause-isolates-on-exit' , |
193 | if (localEngine != null) '-DlocalEngine= $localEngine' , |
194 | if (localEngineHost != null) '-DlocalEngineHost= $localEngineHost' , |
195 | if (localWebSdk != null) '-DlocalWebSdk= $localWebSdk' , |
196 | if (localEngineSrcPath != null) '-DlocalEngineSrcPath= $localEngineSrcPath' , |
197 | taskExecutable, |
198 | ...?taskArgs, |
199 | ], |
200 | environment: <String, String>{ |
201 | if (deviceId != null) |
202 | DeviceIdEnvName: deviceId, |
203 | }, |
204 | ); |
205 | |
206 | bool runnerFinished = false; |
207 | |
208 | unawaited(runner.exitCode.whenComplete(() { |
209 | runnerFinished = true; |
210 | })); |
211 | |
212 | final Completer<Uri> uri = Completer<Uri>(); |
213 | |
214 | final StreamSubscription<String> stdoutSub = runner.stdout |
215 | .transform<String>(const Utf8Decoder()) |
216 | .transform<String>(const LineSplitter()) |
217 | .listen((String line) { |
218 | if (!uri.isCompleted) { |
219 | final Uri? serviceUri = parseServiceUri(line, prefix: RegExp('The Dart VM service is listening on ' )); |
220 | if (serviceUri != null) { |
221 | uri.complete(serviceUri); |
222 | } |
223 | } |
224 | if (!silent) { |
225 | stdout.writeln('[ ${DateTime.now()}] [STDOUT] $line' ); |
226 | } |
227 | }); |
228 | |
229 | final StreamSubscription<String> stderrSub = runner.stderr |
230 | .transform<String>(const Utf8Decoder()) |
231 | .transform<String>(const LineSplitter()) |
232 | .listen((String line) { |
233 | stderr.writeln('[ ${DateTime.now()}] [STDERR] $line' ); |
234 | }); |
235 | |
236 | try { |
237 | final ConnectionResult result = await _connectToRunnerIsolate(await uri.future); |
238 | print('[ $taskName] Connected to VM server.' ); |
239 | isolateParams = isolateParams == null ? <String, String>{} : Map<String, String>.of(isolateParams); |
240 | isolateParams['runProcessCleanup' ] = terminateStrayDartProcesses.toString(); |
241 | final VmService service = result.vmService; |
242 | final String isolateId = result.isolate.id!; |
243 | final Map<String, dynamic> taskResultJson = (await service.callServiceExtension( |
244 | 'ext.cocoonRunTask' , |
245 | args: isolateParams, |
246 | isolateId: isolateId, |
247 | )).json!; |
248 | // Notify the task process that the task result has been received and it |
249 | // can proceed to shutdown. |
250 | await _acknowledgeTaskResultReceived( |
251 | service: service, |
252 | isolateId: isolateId, |
253 | ); |
254 | final TaskResult taskResult = TaskResult.fromJson(taskResultJson); |
255 | final int exitCode = await runner.exitCode; |
256 | print('[ $taskName] Process terminated with exit code $exitCode.' ); |
257 | return taskResult; |
258 | } catch (error, stack) { |
259 | print('[ $taskName] Task runner system failed with exception!\n $error\n $stack' ); |
260 | rethrow; |
261 | } finally { |
262 | if (!runnerFinished) { |
263 | print('[ $taskName] Terminating process...' ); |
264 | runner.kill(ProcessSignal.sigkill); |
265 | } |
266 | await stdoutSub.cancel(); |
267 | await stderrSub.cancel(); |
268 | } |
269 | } |
270 | |
271 | Future<ConnectionResult> _connectToRunnerIsolate(Uri vmServiceUri) async { |
272 | final List<String> pathSegments = <String>[ |
273 | // Add authentication code. |
274 | if (vmServiceUri.pathSegments.isNotEmpty) vmServiceUri.pathSegments[0], |
275 | 'ws' , |
276 | ]; |
277 | final String url = vmServiceUri.replace(scheme: 'ws' , pathSegments: pathSegments).toString(); |
278 | final Stopwatch stopwatch = Stopwatch()..start(); |
279 | |
280 | while (true) { |
281 | try { |
282 | // Look up the isolate. |
283 | final VmService client = await vmServiceConnectUri(url); |
284 | VM vm = await client.getVM(); |
285 | while (vm.isolates!.isEmpty) { |
286 | await Future<void>.delayed(const Duration(seconds: 1)); |
287 | vm = await client.getVM(); |
288 | } |
289 | final IsolateRef isolate = vm.isolates!.first; |
290 | // Sanity check to ensure we're talking with the main isolate. |
291 | final Response response = await client.callServiceExtension('ext.cocoonRunnerReady' , isolateId: isolate.id); |
292 | if (response.json!['result' ] != 'success' ) { |
293 | throw 'not ready yet' ; |
294 | } |
295 | return ConnectionResult(client, isolate); |
296 | } catch (error) { |
297 | if (stopwatch.elapsed > const Duration(seconds: 10)) { |
298 | print( |
299 | 'VM service still not ready. It is possible the target has failed.\n' |
300 | 'Latest connection error:\n' |
301 | ' $error\n' |
302 | 'Continuing to retry...\n' , |
303 | ); |
304 | stopwatch.reset(); |
305 | } |
306 | await Future<void>.delayed(const Duration(milliseconds: 50)); |
307 | } |
308 | } |
309 | } |
310 | |
311 | Future<void> _acknowledgeTaskResultReceived({ |
312 | required VmService service, |
313 | required String isolateId, |
314 | }) async { |
315 | try { |
316 | await service.callServiceExtension( |
317 | 'ext.cocoonTaskResultReceived' , |
318 | isolateId: isolateId, |
319 | ); |
320 | } on RPCError { |
321 | // The target VM may shutdown before the response is received. |
322 | } |
323 | } |
324 | |
325 | class ConnectionResult { |
326 | ConnectionResult(this.vmService, this.isolate); |
327 | |
328 | final VmService vmService; |
329 | final IsolateRef isolate; |
330 | } |
331 | |