1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5/// @docImport 'flutter_goldens.dart';
6library;
7
8import 'dart:convert';
9import 'dart:io' as io;
10
11import 'package:crypto/crypto.dart';
12import 'package:file/file.dart';
13import 'package:path/path.dart' as path;
14import 'package:platform/platform.dart';
15import 'package:process/process.dart';
16
17// If you are here trying to figure out how to use golden files in the Flutter
18// repo itself, consider reading this wiki page:
19// https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md
20
21const String _kFlutterRootKey = 'FLUTTER_ROOT';
22const String _kGoldctlKey = 'GOLDCTL';
23const String _kTestBrowserKey = 'FLUTTER_TEST_BROWSER';
24const String _kWebRendererKey = 'FLUTTER_WEB_RENDERER';
25const String _kImpellerKey = 'FLUTTER_TEST_IMPELLER';
26
27/// Signature of callbacks used to inject [print] replacements.
28typedef LogCallback = void Function(String);
29
30/// Exception thrown when an error is returned from the [SkiaGoldClient].
31class SkiaException implements Exception {
32 /// Creates a new `SkiaException` with a required error [message].
33 const SkiaException(this.message);
34
35 /// A message describing the error.
36 final String message;
37
38 /// Returns a description of the Skia exception.
39 ///
40 /// The description always contains the [message].
41 @override
42 String toString() => 'SkiaException: $message';
43}
44
45/// A client for uploading image tests and making baseline requests to the
46/// Flutter Gold Dashboard.
47class SkiaGoldClient {
48 /// Creates a [SkiaGoldClient] with the given [workDirectory] and [Platform].
49 ///
50 /// All other parameters are optional. They may be provided in tests to
51 /// override the defaults for [fs], [process], and [httpClient].
52 SkiaGoldClient(
53 this.workDirectory, {
54 required this.fs,
55 required this.process,
56 required this.platform,
57 required this.httpClient,
58 required this.log,
59 });
60
61 /// The file system to use for storing the local clone of the repository.
62 ///
63 /// This is useful in tests, where a local file system (the default) can be
64 /// replaced by a memory file system.
65 final FileSystem fs;
66
67 /// The environment (current working directory, identity of the OS,
68 /// environment variables, etc).
69 final Platform platform;
70
71 /// A controller for launching sub-processes.
72 ///
73 /// This is useful in tests, where the real process manager (the default) can
74 /// be replaced by a mock process manager that doesn't really create
75 /// sub-processes.
76 final ProcessManager process;
77
78 /// A client for making Http requests to the Flutter Gold dashboard.
79 final io.HttpClient httpClient;
80
81 /// The local [Directory] within the comparison root for the current test
82 /// context. In this directory, the client will create image and JSON files
83 /// for the goldctl tool to use.
84 ///
85 /// This is informed by [FlutterGoldenFileComparator.basedir]. It cannot be
86 /// null.
87 final Directory workDirectory;
88
89 /// The logging function to use when reporting messages to the console.
90 final LogCallback log;
91
92 /// The local [Directory] where the Flutter repository is hosted.
93 ///
94 /// Uses the [fs] file system.
95 Directory get _flutterRoot => fs.directory(platform.environment[_kFlutterRootKey]);
96
97 /// The path to the local [Directory] where the goldctl tool is hosted.
98 ///
99 /// Uses the [platform] environment in this implementation.
100 String get _goldctl => platform.environment[_kGoldctlKey]!;
101
102 /// Prepares the local work space for golden file testing and calls the
103 /// goldctl `auth` command.
104 ///
105 /// This ensures that the goldctl tool is authorized and ready for testing.
106 /// Used by the [FlutterPostSubmitFileComparator] and the
107 /// [FlutterPreSubmitFileComparator].
108 Future<void> auth() async {
109 if (await clientIsAuthorized()) {
110 return;
111 }
112 final List<String> authCommand = <String>[
113 _goldctl,
114 'auth',
115 '--work-dir',
116 workDirectory.childDirectory('temp').path,
117 '--luci',
118 ];
119
120 final io.ProcessResult result = await process.run(authCommand);
121
122 if (result.exitCode != 0) {
123 final StringBuffer buf =
124 StringBuffer()
125 ..writeln('Skia Gold authorization failed.')
126 ..writeln(
127 'Luci environments authenticate using the file provided '
128 'by LUCI_CONTEXT. There may be an error with this file or Gold '
129 'authentication.',
130 )
131 ..writeln('Debug information for Gold --------------------------------')
132 ..writeln('stdout: ${result.stdout}')
133 ..writeln('stderr: ${result.stderr}');
134 throw SkiaException(buf.toString());
135 }
136 }
137
138 /// Signals if this client is initialized for uploading images to the Gold
139 /// service.
140 ///
141 /// Since Flutter framework tests are executed in parallel, and in random
142 /// order, this will signal is this instance of the Gold client has been
143 /// initialized.
144 bool _initialized = false;
145
146 /// Executes the `imgtest init` command in the goldctl tool.
147 ///
148 /// The `imgtest` command collects and uploads test results to the Skia Gold
149 /// backend, the `init` argument initializes the current test. Used by the
150 /// [FlutterPostSubmitFileComparator].
151 Future<void> imgtestInit() async {
152 // This client has already been initialized
153 if (_initialized) {
154 return;
155 }
156
157 final File keys = workDirectory.childFile('keys.json');
158 final File failures = workDirectory.childFile('failures.json');
159
160 await keys.writeAsString(_getKeysJSON());
161 await failures.create();
162 final String commitHash = await _getCurrentCommit();
163
164 final List<String> imgtestInitCommand = <String>[
165 _goldctl,
166 'imgtest',
167 'init',
168 '--instance',
169 'flutter',
170 '--work-dir',
171 workDirectory.childDirectory('temp').path,
172 '--commit',
173 commitHash,
174 '--keys-file',
175 keys.path,
176 '--failure-file',
177 failures.path,
178 '--passfail',
179 ];
180
181 if (imgtestInitCommand.contains(null)) {
182 final StringBuffer buf =
183 StringBuffer()
184 ..writeln('A null argument was provided for Skia Gold imgtest init.')
185 ..writeln('Please confirm the settings of your golden file test.')
186 ..writeln('Arguments provided:');
187 imgtestInitCommand.forEach(buf.writeln);
188 throw SkiaException(buf.toString());
189 }
190
191 final io.ProcessResult result = await process.run(imgtestInitCommand);
192
193 if (result.exitCode != 0) {
194 _initialized = false;
195 final StringBuffer buf =
196 StringBuffer()
197 ..writeln('Skia Gold imgtest init failed.')
198 ..writeln('An error occurred when initializing golden file test with ')
199 ..writeln('goldctl.')
200 ..writeln()
201 ..writeln('Debug information for Gold --------------------------------')
202 ..writeln('stdout: ${result.stdout}')
203 ..writeln('stderr: ${result.stderr}');
204 throw SkiaException(buf.toString());
205 }
206 _initialized = true;
207 }
208
209 /// Executes the `imgtest add` command in the goldctl tool.
210 ///
211 /// The `imgtest` command collects and uploads test results to the Skia Gold
212 /// backend, the `add` argument uploads the current image test. A response is
213 /// returned from the invocation of this command that indicates a pass or fail
214 /// result.
215 ///
216 /// The [testName] and [goldenFile] parameters reference the current
217 /// comparison being evaluated by the [FlutterPostSubmitFileComparator].
218 Future<bool> imgtestAdd(String testName, File goldenFile) async {
219 final List<String> imgtestCommand = <String>[
220 _goldctl,
221 'imgtest',
222 'add',
223 '--work-dir',
224 workDirectory.childDirectory('temp').path,
225 '--test-name',
226 cleanTestName(testName),
227 '--png-file',
228 goldenFile.path,
229 '--passfail',
230 ];
231
232 final io.ProcessResult result = await process.run(imgtestCommand);
233
234 if (result.exitCode != 0) {
235 // If an unapproved image has made it to post-submit, throw to close the
236 // tree.
237 String? resultContents;
238 final File resultFile = workDirectory.childFile(fs.path.join('result-state.json'));
239 if (await resultFile.exists()) {
240 resultContents = await resultFile.readAsString();
241 }
242
243 final StringBuffer buf =
244 StringBuffer()
245 ..writeln('Skia Gold received an unapproved image in post-submit ')
246 ..writeln('testing. Golden file images in flutter/flutter are triaged ')
247 ..writeln('in pre-submit during code review for the given PR.')
248 ..writeln()
249 ..writeln('Visit https://flutter-gold.skia.org/ to view and approve ')
250 ..writeln('the image(s), or revert the associated change. For more ')
251 ..writeln('information, visit the wiki: ')
252 ..writeln(
253 'https://github.com/flutter/flutter/blob/main/docs/contributing/testing/Writing-a-golden-file-test-for-package-flutter.md',
254 )
255 ..writeln()
256 ..writeln('Debug information for Gold --------------------------------')
257 ..writeln('stdout: ${result.stdout}')
258 ..writeln('stderr: ${result.stderr}')
259 ..writeln()
260 ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}');
261 throw SkiaException(buf.toString());
262 }
263
264 return true;
265 }
266
267 /// Signals if this client is initialized for uploading tryjobs to the Gold
268 /// service.
269 ///
270 /// Since Flutter framework tests are executed in parallel, and in random
271 /// order, this will signal is this instance of the Gold client has been
272 /// initialized for tryjobs.
273 bool _tryjobInitialized = false;
274
275 /// Executes the `imgtest init` command in the goldctl tool for tryjobs.
276 ///
277 /// The `imgtest` command collects and uploads test results to the Skia Gold
278 /// backend, the `init` argument initializes the current tryjob. Used by the
279 /// [FlutterPreSubmitFileComparator].
280 Future<void> tryjobInit() async {
281 // This client has already been initialized
282 if (_tryjobInitialized) {
283 return;
284 }
285
286 final File keys = workDirectory.childFile('keys.json');
287 final File failures = workDirectory.childFile('failures.json');
288
289 await keys.writeAsString(_getKeysJSON());
290 await failures.create();
291 final String commitHash = await _getCurrentCommit();
292
293 final List<String> imgtestInitCommand = <String>[
294 _goldctl,
295 'imgtest',
296 'init',
297 '--instance',
298 'flutter',
299 '--work-dir',
300 workDirectory.childDirectory('temp').path,
301 '--commit',
302 commitHash,
303 '--keys-file',
304 keys.path,
305 '--failure-file',
306 failures.path,
307 '--passfail',
308 '--crs',
309 'github',
310 '--patchset_id',
311 commitHash,
312 ...getCIArguments(),
313 ];
314
315 if (imgtestInitCommand.contains(null)) {
316 final StringBuffer buf =
317 StringBuffer()
318 ..writeln('A null argument was provided for Skia Gold tryjob init.')
319 ..writeln('Please confirm the settings of your golden file test.')
320 ..writeln('Arguments provided:');
321 imgtestInitCommand.forEach(buf.writeln);
322 throw SkiaException(buf.toString());
323 }
324
325 final io.ProcessResult result = await process.run(imgtestInitCommand);
326
327 if (result.exitCode != 0) {
328 _tryjobInitialized = false;
329 final StringBuffer buf =
330 StringBuffer()
331 ..writeln('Skia Gold tryjobInit failure.')
332 ..writeln('An error occurred when initializing golden file tryjob with ')
333 ..writeln('goldctl.')
334 ..writeln()
335 ..writeln('Debug information for Gold --------------------------------')
336 ..writeln('stdout: ${result.stdout}')
337 ..writeln('stderr: ${result.stderr}');
338 throw SkiaException(buf.toString());
339 }
340 _tryjobInitialized = true;
341 }
342
343 /// Executes the `imgtest add` command in the goldctl tool for tryjobs.
344 ///
345 /// The `imgtest` command collects and uploads test results to the Skia Gold
346 /// backend, the `add` argument uploads the current image test. A response is
347 /// returned from the invocation of this command that indicates a pass or fail
348 /// result for the tryjob.
349 ///
350 /// The [testName] and [goldenFile] parameters reference the current
351 /// comparison being evaluated by the [FlutterPreSubmitFileComparator].
352 ///
353 /// If the tryjob fails due to pixel differences, the method will succeed
354 /// as the failure will be triaged in the 'Flutter Gold' dashboard, and the
355 /// `stdout` will contain the failure message; otherwise will return `null`.
356 Future<String?> tryjobAdd(String testName, File goldenFile) async {
357 final List<String> imgtestCommand = <String>[
358 _goldctl,
359 'imgtest',
360 'add',
361 '--work-dir',
362 workDirectory.childDirectory('temp').path,
363 '--test-name',
364 cleanTestName(testName),
365 '--png-file',
366 goldenFile.path,
367 ];
368
369 final io.ProcessResult result = await process.run(imgtestCommand);
370
371 final String resultStdout = result.stdout.toString();
372 if (result.exitCode != 0 &&
373 !(resultStdout.contains('Untriaged') || resultStdout.contains('negative image'))) {
374 String? resultContents;
375 final File resultFile = workDirectory.childFile(fs.path.join('result-state.json'));
376 if (await resultFile.exists()) {
377 resultContents = await resultFile.readAsString();
378 }
379 final StringBuffer buf =
380 StringBuffer()
381 ..writeln('Unexpected Gold tryjobAdd failure.')
382 ..writeln('Tryjob execution for golden file test $testName failed for')
383 ..writeln('a reason unrelated to pixel comparison.')
384 ..writeln()
385 ..writeln('Debug information for Gold --------------------------------')
386 ..writeln('stdout: ${result.stdout}')
387 ..writeln('stderr: ${result.stderr}')
388 ..writeln()
389 ..writeln()
390 ..writeln('result-state.json: ${resultContents ?? 'No result file found.'}');
391 throw SkiaException(buf.toString());
392 }
393 return result.exitCode == 0 ? null : resultStdout;
394 }
395
396 /// Returns the latest positive digest for the given test known to Flutter
397 /// Gold at head.
398 Future<String?> getExpectationForTest(String testName) async {
399 late String? expectation;
400 final String traceID = getTraceID(testName);
401 final Uri requestForExpectations = Uri.parse(
402 'https://flutter-gold.skia.org/json/v2/latestpositivedigest/$traceID',
403 );
404 late String rawResponse;
405 try {
406 final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations);
407 final io.HttpClientResponse response = await request.close();
408 rawResponse = await utf8.decodeStream(response);
409 final dynamic jsonResponse = json.decode(rawResponse);
410 if (jsonResponse is! Map<String, dynamic>) {
411 throw const FormatException('Skia gold expectations do not match expected format.');
412 }
413 expectation = jsonResponse['digest'] as String?;
414 } on FormatException catch (error) {
415 log(
416 'Formatting error detected requesting expectations from Flutter Gold.\n'
417 'error: $error\n'
418 'url: $requestForExpectations\n'
419 'response: $rawResponse',
420 );
421 rethrow;
422 }
423 return expectation;
424 }
425
426 /// Returns a list of bytes representing the golden image retrieved from the
427 /// Flutter Gold dashboard.
428 ///
429 /// The provided image hash represents an expectation from Flutter Gold.
430 Future<List<int>> getImageBytes(String imageHash) async {
431 final List<int> imageBytes = <int>[];
432 final Uri requestForImage = Uri.parse(
433 'https://flutter-gold.skia.org/img/images/$imageHash.png',
434 );
435 final io.HttpClientRequest request = await httpClient.getUrl(requestForImage);
436 final io.HttpClientResponse response = await request.close();
437 await response.forEach((List<int> bytes) => imageBytes.addAll(bytes));
438 return imageBytes;
439 }
440
441 /// Returns the current commit hash of the Flutter repository.
442 Future<String> _getCurrentCommit() async {
443 if (!_flutterRoot.existsSync()) {
444 throw SkiaException('Flutter root could not be found: $_flutterRoot\n');
445 } else {
446 final io.ProcessResult revParse = await process.run(<String>[
447 'git',
448 'rev-parse',
449 'HEAD',
450 ], workingDirectory: _flutterRoot.path);
451 if (revParse.exitCode != 0) {
452 throw const SkiaException('Current commit of Flutter can not be found.');
453 }
454 return (revParse.stdout as String).trim();
455 }
456 }
457
458 /// Returns a JSON String with keys value pairs used to uniquely identify the
459 /// configuration that generated the given golden file.
460 ///
461 /// Currently, the only key value pairs being tracked is the platform the
462 /// image was rendered on, and for web tests, the browser the image was
463 /// rendered on.
464 String _getKeysJSON() {
465 final String? webRenderer = _webRendererValue;
466 final Map<String, dynamic> keys = <String, dynamic>{
467 'Platform': platform.operatingSystem,
468 'CI': 'luci',
469 if (_isImpeller) 'impeller': 'swiftshader',
470 };
471 if (_isBrowserTest) {
472 keys['Browser'] = _browserKey;
473 keys['Platform'] = '${keys['Platform']}-browser';
474 if (webRenderer != null) {
475 keys['WebRenderer'] = webRenderer;
476 }
477 }
478 return json.encode(keys);
479 }
480
481 /// Removes the file extension from the [fileName] to represent the test name
482 /// properly.
483 String cleanTestName(String fileName) {
484 return fileName.split(path.extension(fileName))[0];
485 }
486
487 /// Returns a boolean value to prevent the client from re-authorizing itself
488 /// for multiple tests.
489 Future<bool> clientIsAuthorized() async {
490 final File authFile = workDirectory.childFile(fs.path.join('temp', 'auth_opt.json'));
491
492 if (await authFile.exists()) {
493 final String contents = await authFile.readAsString();
494 final Map<String, dynamic> decoded = json.decode(contents) as Map<String, dynamic>;
495 return !(decoded['GSUtil'] as bool);
496 }
497 return false;
498 }
499
500 /// Returns a list of arguments for initializing a tryjob based on the testing
501 /// environment.
502 List<String> getCIArguments() {
503 final String jobId = platform.environment['LOGDOG_STREAM_PREFIX']!.split('/').last;
504 final List<String> refs = platform.environment['GOLD_TRYJOB']!.split('/');
505 final String pullRequest = refs[refs.length - 2];
506
507 return <String>['--changelist', pullRequest, '--cis', 'buildbucket', '--jobid', jobId];
508 }
509
510 bool get _isBrowserTest {
511 return platform.environment[_kTestBrowserKey] != null;
512 }
513
514 bool get _isBrowserSkiaTest {
515 return _isBrowserTest &&
516 switch (platform.environment[_kWebRendererKey]) {
517 'canvaskit' || 'skwasm' => true,
518 _ => false,
519 };
520 }
521
522 String? get _webRendererValue {
523 return _isBrowserSkiaTest ? platform.environment[_kWebRendererKey] : null;
524 }
525
526 bool get _isImpeller {
527 return (platform.environment[_kImpellerKey] != null);
528 }
529
530 String get _browserKey {
531 assert(_isBrowserTest);
532 return platform.environment[_kTestBrowserKey]!;
533 }
534
535 /// Returns a trace id based on the current testing environment to lookup
536 /// the latest positive digest on Flutter Gold with a hex-encoded md5 hash of
537 /// the image keys.
538 String getTraceID(String testName) {
539 final String? webRenderer = _webRendererValue;
540 final Map<String, Object?> parameters = <String, Object?>{
541 if (_isBrowserTest) 'Browser': _browserKey,
542 'CI': 'luci',
543 'Platform': platform.operatingSystem,
544 if (webRenderer != null) 'WebRenderer': webRenderer,
545 if (_isImpeller) 'impeller': 'swiftshader',
546 'name': testName,
547 'source_type': 'flutter',
548 };
549 final Map<String, Object?> sorted = <String, Object?>{};
550 for (final String key in parameters.keys.toList()..sort()) {
551 sorted[key] = parameters[key];
552 }
553 final String jsonTrace = json.encode(sorted);
554 final String md5Sum = md5.convert(utf8.encode(jsonTrace)).toString();
555 return md5Sum;
556 }
557}
558

Provided by KDAB

Privacy Policy
Learn more about Flutter for embedded and desktop on industrialflutter.com