1 | // Copyright 2014 The Flutter Authors. All rights reserved. |
2 | // Use of this source code is governed by a BSD-style license that can be |
3 | // found in the LICENSE file. |
4 | |
5 | import 'dart:async'; |
6 | import 'dart:convert'; |
7 | import 'dart:developer'; |
8 | import 'dart:io'; |
9 | import 'dart:isolate'; |
10 | |
11 | import 'package:logging/logging.dart' ; |
12 | import 'package:path/path.dart' as path; |
13 | import 'package:process/process.dart' ; |
14 | import 'package:stack_trace/stack_trace.dart' ; |
15 | |
16 | import 'devices.dart'; |
17 | import 'host_agent.dart'; |
18 | import 'running_processes.dart'; |
19 | import 'task_result.dart'; |
20 | import 'utils.dart'; |
21 | |
22 | /// Identifiers for devices that should never be rebooted. |
23 | final Set<String> noRebootForbidList = <String>{ |
24 | '822ef7958bba573829d85eef4df6cbdd86593730' , // 32bit iPhone requires manual intervention on reboot. |
25 | }; |
26 | |
27 | /// The maximum number of test runs before a device must be rebooted. |
28 | /// |
29 | /// This number was chosen arbitrarily. |
30 | const int maximumRuns = 30; |
31 | |
32 | /// Represents a unit of work performed in the CI environment that can |
33 | /// succeed, fail and be retried independently of others. |
34 | typedef TaskFunction = Future<TaskResult> Function(); |
35 | |
36 | bool _isTaskRegistered = false; |
37 | |
38 | /// Registers a [task] to run, returns the result when it is complete. |
39 | /// |
40 | /// The task does not run immediately but waits for the request via the |
41 | /// VM service protocol to run it. |
42 | /// |
43 | /// It is OK for a [task] to perform many things. However, only one task can be |
44 | /// registered per Dart VM. |
45 | /// |
46 | /// If no `processManager` is provided, a default [LocalProcessManager] is created |
47 | /// for the task. |
48 | Future<TaskResult> task(TaskFunction task, {ProcessManager? processManager}) async { |
49 | if (_isTaskRegistered) { |
50 | throw StateError('A task is already registered' ); |
51 | } |
52 | _isTaskRegistered = true; |
53 | |
54 | processManager ??= const LocalProcessManager(); |
55 | |
56 | // TODO(ianh): allow overriding logging. |
57 | Logger.root.level = Level.ALL; |
58 | Logger.root.onRecord.listen((LogRecord rec) { |
59 | print(' ${rec.level.name}: ${rec.time}: ${rec.message}' ); |
60 | }); |
61 | |
62 | final _TaskRunner runner = _TaskRunner(task, processManager); |
63 | runner.keepVmAliveUntilTaskRunRequested(); |
64 | return runner.whenDone; |
65 | } |
66 | |
67 | class _TaskRunner { |
68 | _TaskRunner(this.task, this.processManager) { |
69 | final String successResponse = json.encode(const <String, String>{'result' : 'success' }); |
70 | |
71 | registerExtension('ext.cocoonRunTask' , (String method, Map<String, String> parameters) async { |
72 | final Duration? taskTimeout = |
73 | parameters.containsKey('timeoutInMinutes' ) |
74 | ? Duration(minutes: int.parse(parameters['timeoutInMinutes' ]!)) |
75 | : null; |
76 | final bool runFlutterConfig = |
77 | parameters['runFlutterConfig' ] != |
78 | 'false' ; // used by tests to avoid changing the configuration |
79 | final bool runProcessCleanup = parameters['runProcessCleanup' ] != 'false' ; |
80 | final String? localEngine = parameters['localEngine' ]; |
81 | final String? localEngineHost = parameters['localEngineHost' ]; |
82 | final TaskResult result = await run( |
83 | taskTimeout, |
84 | runProcessCleanup: runProcessCleanup, |
85 | runFlutterConfig: runFlutterConfig, |
86 | localEngine: localEngine, |
87 | localEngineHost: localEngineHost, |
88 | ); |
89 | const Duration taskResultReceivedTimeout = Duration(seconds: 30); |
90 | _taskResultReceivedTimeout = Timer(taskResultReceivedTimeout, () { |
91 | logger.severe( |
92 | 'Task runner did not acknowledge task results in $taskResultReceivedTimeout.' , |
93 | ); |
94 | _closeKeepAlivePort(); |
95 | exitCode = 1; |
96 | }); |
97 | return ServiceExtensionResponse.result(json.encode(result.toJson())); |
98 | }); |
99 | registerExtension('ext.cocoonRunnerReady' , ( |
100 | String method, |
101 | Map<String, String> parameters, |
102 | ) async { |
103 | return ServiceExtensionResponse.result(successResponse); |
104 | }); |
105 | registerExtension('ext.cocoonTaskResultReceived' , ( |
106 | String method, |
107 | Map<String, String> parameters, |
108 | ) async { |
109 | _closeKeepAlivePort(); |
110 | return ServiceExtensionResponse.result(successResponse); |
111 | }); |
112 | } |
113 | |
114 | final TaskFunction task; |
115 | final ProcessManager processManager; |
116 | |
117 | Future<Device?> _getWorkingDeviceIfAvailable() async { |
118 | try { |
119 | return await devices.workingDevice; |
120 | } on DeviceException { |
121 | return null; |
122 | } |
123 | } |
124 | |
125 | // TODO(ianh): workaround for https://github.com/dart-lang/sdk/issues/23797 |
126 | RawReceivePort? _keepAlivePort; |
127 | Timer? _startTaskTimeout; |
128 | Timer? _taskResultReceivedTimeout; |
129 | bool _taskStarted = false; |
130 | |
131 | final Completer<TaskResult> _completer = Completer<TaskResult>(); |
132 | |
133 | static final Logger logger = Logger('TaskRunner' ); |
134 | |
135 | /// Signals that this task runner finished running the task. |
136 | Future<TaskResult> get whenDone => _completer.future; |
137 | |
138 | Future<TaskResult> run( |
139 | Duration? taskTimeout, { |
140 | bool runFlutterConfig = true, |
141 | bool runProcessCleanup = true, |
142 | required String? localEngine, |
143 | required String? localEngineHost, |
144 | }) async { |
145 | try { |
146 | _taskStarted = true; |
147 | print('Running task with a timeout of $taskTimeout.' ); |
148 | final String exe = Platform.isWindows ? '.exe' : '' ; |
149 | late Set<RunningProcessInfo> beforeRunningDartInstances; |
150 | if (runProcessCleanup) { |
151 | section('Checking running Dart $exe processes' ); |
152 | beforeRunningDartInstances = await getRunningProcesses( |
153 | processName: 'dart $exe' , |
154 | processManager: processManager, |
155 | ); |
156 | final Set<RunningProcessInfo> allProcesses = await getRunningProcesses( |
157 | processManager: processManager, |
158 | ); |
159 | beforeRunningDartInstances.forEach(print); |
160 | for (final RunningProcessInfo info in allProcesses) { |
161 | if (info.commandLine.contains('iproxy' )) { |
162 | print('[LEAK]: ${info.commandLine} ${info.creationDate} ${info.pid} ' ); |
163 | } |
164 | } |
165 | } |
166 | |
167 | if (runFlutterConfig) { |
168 | print('Enabling configs for macOS and Linux...' ); |
169 | final int configResult = await exec( |
170 | path.join(flutterDirectory.path, 'bin' , 'flutter' ), |
171 | <String>[ |
172 | 'config' , |
173 | '-v' , |
174 | '--enable-macos-desktop' , |
175 | '--enable-linux-desktop' , |
176 | if (localEngine != null) ...<String>['--local-engine' , localEngine], |
177 | if (localEngineHost != null) ...<String>['--local-engine-host' , localEngineHost], |
178 | ], |
179 | canFail: true, |
180 | ); |
181 | if (configResult != 0) { |
182 | print('Failed to enable configuration, tasks may not run.' ); |
183 | } |
184 | } |
185 | |
186 | final Device? device = await _getWorkingDeviceIfAvailable(); |
187 | |
188 | // Some tests assume the phone is in home |
189 | await device?.home(); |
190 | |
191 | late TaskResult result; |
192 | IOSink? sink; |
193 | try { |
194 | if (device != null && device.canStreamLogs && hostAgent.dumpDirectory != null) { |
195 | sink = |
196 | File(path.join(hostAgent.dumpDirectory!.path, ' ${device.deviceId}.log' )).openWrite(); |
197 | await device.startLoggingToSink(sink); |
198 | } |
199 | |
200 | Future<TaskResult> futureResult = _performTask(); |
201 | if (taskTimeout != null) { |
202 | futureResult = futureResult.timeout(taskTimeout); |
203 | } |
204 | |
205 | result = await futureResult; |
206 | } finally { |
207 | if (device != null && device.canStreamLogs) { |
208 | await device.stopLoggingToSink(); |
209 | await sink?.close(); |
210 | } |
211 | } |
212 | |
213 | if (runProcessCleanup) { |
214 | section('Terminating lingering Dart $exe processes after task...' ); |
215 | final Set<RunningProcessInfo> afterRunningDartInstances = await getRunningProcesses( |
216 | processName: 'dart $exe' , |
217 | processManager: processManager, |
218 | ); |
219 | for (final RunningProcessInfo info in afterRunningDartInstances) { |
220 | if (!beforeRunningDartInstances.contains(info)) { |
221 | print(' $info was leaked by this test.' ); |
222 | if (result is TaskResultCheckProcesses) { |
223 | result = TaskResult.failure('This test leaked dart processes' ); |
224 | } |
225 | if (await info.terminate(processManager: processManager)) { |
226 | print('Killed process id ${info.pid}.' ); |
227 | } else { |
228 | print('Failed to kill process ${info.pid}.' ); |
229 | } |
230 | } |
231 | } |
232 | } |
233 | _completer.complete(result); |
234 | return result; |
235 | } on TimeoutException catch (err, stackTrace) { |
236 | print('Task timed out in framework.dart after $taskTimeout.' ); |
237 | print(err); |
238 | print(stackTrace); |
239 | return TaskResult.failure('Task timed out after $taskTimeout' ); |
240 | } finally { |
241 | await checkForRebootRequired(); |
242 | await forceQuitRunningProcesses(); |
243 | } |
244 | } |
245 | |
246 | Future<void> checkForRebootRequired() async { |
247 | print('Checking for reboot' ); |
248 | try { |
249 | final Device device = await devices.workingDevice; |
250 | if (noRebootForbidList.contains(device.deviceId)) { |
251 | return; |
252 | } |
253 | final File rebootFile = _rebootFile(); |
254 | int runCount; |
255 | if (rebootFile.existsSync()) { |
256 | runCount = int.tryParse(rebootFile.readAsStringSync().trim()) ?? 0; |
257 | } else { |
258 | runCount = 0; |
259 | } |
260 | if (runCount < maximumRuns) { |
261 | rebootFile |
262 | ..createSync() |
263 | ..writeAsStringSync((runCount + 1).toString()); |
264 | return; |
265 | } |
266 | rebootFile.deleteSync(); |
267 | print('rebooting' ); |
268 | await device.reboot(); |
269 | } on TimeoutException { |
270 | // Could not find device in order to reboot. |
271 | } on DeviceException { |
272 | // No attached device needed to reboot. |
273 | } |
274 | } |
275 | |
276 | /// Causes the Dart VM to stay alive until a request to run the task is |
277 | /// received via the VM service protocol. |
278 | void keepVmAliveUntilTaskRunRequested() { |
279 | if (_taskStarted) { |
280 | throw StateError('Task already started.' ); |
281 | } |
282 | |
283 | // Merely creating this port object will cause the VM to stay alive and keep |
284 | // the VM service server running until the port is disposed of. |
285 | _keepAlivePort = RawReceivePort(); |
286 | |
287 | // Timeout if nothing bothers to connect and ask us to run the task. |
288 | const Duration taskStartTimeout = Duration(seconds: 60); |
289 | _startTaskTimeout = Timer(taskStartTimeout, () { |
290 | if (!_taskStarted) { |
291 | logger.severe('Task did not start in $taskStartTimeout.' ); |
292 | _closeKeepAlivePort(); |
293 | exitCode = 1; |
294 | } |
295 | }); |
296 | } |
297 | |
298 | /// Disables the keepalive port, allowing the VM to exit. |
299 | void _closeKeepAlivePort() { |
300 | _startTaskTimeout?.cancel(); |
301 | _taskResultReceivedTimeout?.cancel(); |
302 | _keepAlivePort?.close(); |
303 | } |
304 | |
305 | Future<TaskResult> _performTask() { |
306 | final Completer<TaskResult> completer = Completer<TaskResult>(); |
307 | Chain.capture( |
308 | () async { |
309 | completer.complete(await task()); |
310 | }, |
311 | onError: (dynamic taskError, Chain taskErrorStack) { |
312 | final String message = 'Task failed: $taskError' ; |
313 | stderr |
314 | ..writeln(message) |
315 | ..writeln('\nStack trace:' ) |
316 | ..writeln(taskErrorStack.terse); |
317 | // IMPORTANT: We're completing the future _successfully_ but with a value |
318 | // that indicates a task failure. This is intentional. At this point we |
319 | // are catching errors coming from arbitrary (and untrustworthy) task |
320 | // code. Our goal is to convert the failure into a readable message. |
321 | // Propagating it further is not useful. |
322 | if (!completer.isCompleted) { |
323 | completer.complete(TaskResult.failure(message)); |
324 | } |
325 | }, |
326 | ); |
327 | return completer.future; |
328 | } |
329 | } |
330 | |
331 | File _rebootFile() { |
332 | if (Platform.isLinux || Platform.isMacOS) { |
333 | return File(path.join(Platform.environment['HOME' ]!, '.reboot-count' )); |
334 | } |
335 | if (!Platform.isWindows) { |
336 | throw StateError('Unexpected platform ${Platform.operatingSystem}' ); |
337 | } |
338 | return File(path.join(Platform.environment['USERPROFILE' ]!, '.reboot-count' )); |
339 | } |
340 | |