1 | /**************************************************************************** |
2 | ** |
3 | ** Copyright (C) 2016 The Qt Company Ltd. |
4 | ** Contact: https://www.qt.io/licensing/ |
5 | ** |
6 | ** This file is part of the V4VM module of the Qt Toolkit. |
7 | ** |
8 | ** $QT_BEGIN_LICENSE:GPL-EXCEPT$ |
9 | ** Commercial License Usage |
10 | ** Licensees holding valid commercial Qt licenses may use this file in |
11 | ** accordance with the commercial license agreement provided with the |
12 | ** Software or, alternatively, in accordance with the terms contained in |
13 | ** a written agreement between you and The Qt Company. For licensing terms |
14 | ** and conditions see https://www.qt.io/terms-conditions. For further |
15 | ** information use the contact form at https://www.qt.io/contact-us. |
16 | ** |
17 | ** GNU General Public License Usage |
18 | ** Alternatively, this file may be used under the terms of the GNU |
19 | ** General Public License version 3 as published by the Free Software |
20 | ** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT |
21 | ** included in the packaging of this file. Please review the following |
22 | ** information to ensure the GNU General Public License requirements will |
23 | ** be met: https://www.gnu.org/licenses/gpl-3.0.html. |
24 | ** |
25 | ** $QT_END_LICENSE$ |
26 | ** |
27 | ****************************************************************************/ |
28 | #include "test262runner.h" |
29 | |
30 | #include <qfile.h> |
31 | #include <qdir.h> |
32 | #include <qdiriterator.h> |
33 | #include <qdebug.h> |
34 | #include <qprocess.h> |
35 | #include <qtemporaryfile.h> |
36 | |
37 | #include <private/qv4script_p.h> |
38 | #include "private/qv4globalobject_p.h" |
39 | #include "private/qqmlbuiltinfunctions_p.h" |
40 | #include "private/qv4arraybuffer_p.h" |
41 | |
42 | #include "qrunnable.h" |
43 | |
44 | static const char *excludedFeatures[] = { |
45 | "BigInt" , |
46 | "class-fields-public" , |
47 | "class-fields-private" , |
48 | "Promise.prototype.finally" , |
49 | "async-iteration" , |
50 | "Symbol.asyncIterator" , |
51 | "object-rest" , |
52 | "object-spread" , |
53 | "optional-catch-binding" , |
54 | "regexp-dotall" , |
55 | "regexp-lookbehind" , |
56 | "regexp-named-groups" , |
57 | "regexp-unicode-property-escapes" , |
58 | "Atomics" , |
59 | "SharedArrayBuffer" , |
60 | "Array.prototype.flatten" , |
61 | "Array.prototype.flatMap" , |
62 | "string-trimming" , |
63 | "String.prototype.trimEnd" , |
64 | "String.prototype.trimStart" , |
65 | "numeric-separator-literal" , |
66 | |
67 | // optional features, not supported by us |
68 | "caller" , |
69 | nullptr |
70 | }; |
71 | |
72 | static const char *excludedFilePatterns[] = { |
73 | "realm" , |
74 | nullptr |
75 | }; |
76 | |
77 | QT_BEGIN_NAMESPACE |
78 | |
79 | namespace QV4 { |
80 | |
81 | static ReturnedValue method_detachArrayBuffer(const FunctionObject *f, const Value *, const Value *argv, int argc) |
82 | { |
83 | Scope scope(f); |
84 | if (!argc) |
85 | return scope.engine->throwTypeError(); |
86 | Scoped<ArrayBuffer> a(scope, argv[0]); |
87 | if (!a) |
88 | return scope.engine->throwTypeError(); |
89 | |
90 | if (a->isShared()) |
91 | return scope.engine->throwTypeError(); |
92 | |
93 | a->d()->detachArrayBuffer(); |
94 | |
95 | return Encode::null(); |
96 | } |
97 | |
98 | static void initD262(ExecutionEngine *e) |
99 | { |
100 | Scope scope(e); |
101 | ScopedObject d262(scope, e->newObject()); |
102 | |
103 | d262->defineDefaultProperty(QStringLiteral("detachArrayBuffer" ), code: method_detachArrayBuffer, argumentCount: 1); |
104 | |
105 | ScopedString s(scope, e->newString(QStringLiteral("$262" ))); |
106 | e->globalObject->put(name: s, v: d262); |
107 | } |
108 | |
109 | } |
110 | |
111 | QT_END_NAMESPACE |
112 | |
113 | Test262Runner::Test262Runner(const QString &command, const QString &dir) |
114 | : command(command), testDir(dir) |
115 | { |
116 | if (testDir.endsWith(c: QLatin1Char('/'))) |
117 | testDir = testDir.chopped(n: 1); |
118 | } |
119 | |
120 | Test262Runner::~Test262Runner() |
121 | { |
122 | delete threadPool; |
123 | } |
124 | |
125 | void Test262Runner::cat() |
126 | { |
127 | if (!loadTests()) |
128 | return; |
129 | |
130 | if (testCases.size() != 1) |
131 | qWarning() << "test262 --cat: Ambiguous test case, using" << testCases.begin().key(); |
132 | TestData data = getTestData(testCase: testCases.begin().value()); |
133 | printf(format: "%s" , data.content.constData()); |
134 | } |
135 | |
136 | bool Test262Runner::run() |
137 | { |
138 | if (!loadTests()) |
139 | return false; |
140 | |
141 | if (flags & Parallel) { |
142 | threadPool = new QThreadPool; |
143 | threadPool->setStackSize(16*1024*1024); |
144 | if (flags & Verbose) |
145 | qDebug() << "Running in parallel with" << QThread::idealThreadCount() << "threads." ; |
146 | } |
147 | |
148 | if (flags & ForceJIT) |
149 | qputenv(varName: "QV4_JIT_CALL_THRESHOLD" , value: QByteArray("0" )); |
150 | else if (flags & ForceBytecode) |
151 | qputenv(varName: "QV4_FORCE_INTERPRETER" , value: QByteArray("1" )); |
152 | |
153 | if (flags & WithTestExpectations) |
154 | loadTestExpectations(); |
155 | |
156 | for (auto it = testCases.constBegin(); it != testCases.constEnd(); ++it) { |
157 | auto c = it.value(); |
158 | if (!c.skipTestCase) { |
159 | int result = runSingleTest(testCase: c); |
160 | if (result == -2) |
161 | return false; |
162 | } |
163 | } |
164 | |
165 | if (threadPool) |
166 | threadPool->waitForDone(); |
167 | |
168 | const bool testsOk = report(); |
169 | |
170 | if (flags & WriteTestExpectations) |
171 | writeTestExpectations(); |
172 | else if (flags & UpdateTestExpectations) |
173 | updateTestExpectations(); |
174 | |
175 | return testsOk; |
176 | } |
177 | |
178 | bool Test262Runner::report() |
179 | { |
180 | qDebug() << "Test execution summary:" ; |
181 | qDebug() << " Executed" << testCases.size() << "test cases." ; |
182 | QStringList crashes; |
183 | QStringList unexpectedFailures; |
184 | QStringList unexpectedPasses; |
185 | for (auto it = testCases.constBegin(); it != testCases.constEnd(); ++it) { |
186 | const auto c = it.value(); |
187 | if (c.strictResult == c.strictExpectation && c.sloppyResult == c.sloppyExpectation) |
188 | continue; |
189 | auto report = [&] (TestCase::Result expected, TestCase::Result result, const char *s) { |
190 | if (result == TestCase::Crashes) |
191 | crashes << (it.key() + " crashed in " + s + " mode" ); |
192 | if (result == TestCase::Fails && expected == TestCase::Passes) |
193 | unexpectedFailures << (it.key() + " failed in " + s + " mode" ); |
194 | if (result == TestCase::Passes && expected == TestCase::Fails) |
195 | unexpectedPasses << (it.key() + " unexpectedly passed in " + s + " mode" ); |
196 | }; |
197 | report(c.strictExpectation, c.strictResult, "strict" ); |
198 | report(c.sloppyExpectation, c.sloppyResult, "sloppy" ); |
199 | } |
200 | if (!crashes.isEmpty()) { |
201 | qDebug() << " Encountered" << crashes.size() << "crashes in the following files:" ; |
202 | for (const QString &f : qAsConst(t&: crashes)) |
203 | qDebug() << " " << f; |
204 | } |
205 | if (!unexpectedFailures.isEmpty()) { |
206 | qDebug() << " Encountered" << unexpectedFailures.size() << "unexpected failures in the following files:" ; |
207 | for (const QString &f : qAsConst(t&: unexpectedFailures)) |
208 | qDebug() << " " << f; |
209 | } |
210 | if (!unexpectedPasses.isEmpty()) { |
211 | qDebug() << " Encountered" << unexpectedPasses.size() << "unexpected passes in the following files:" ; |
212 | for (const QString &f : qAsConst(t&: unexpectedPasses)) |
213 | qDebug() << " " << f; |
214 | } |
215 | return crashes.isEmpty() && unexpectedFailures.isEmpty() && unexpectedPasses.isEmpty(); |
216 | } |
217 | |
218 | bool Test262Runner::loadTests() |
219 | { |
220 | QDir dir(testDir + "/test" ); |
221 | if (!dir.exists()) { |
222 | qWarning() << "Could not load tests," << dir.path() << "does not exist." ; |
223 | return false; |
224 | } |
225 | |
226 | QString annexB = "annexB" ; |
227 | QString harness = "harness" ; |
228 | QString intl402 = "intl402" ; |
229 | |
230 | int pathlen = dir.path().length() + 1; |
231 | QDirIterator it(dir, QDirIterator::Subdirectories); |
232 | while (it.hasNext()) { |
233 | QString file = it.next().mid(position: pathlen); |
234 | if (!file.endsWith(s: ".js" )) |
235 | continue; |
236 | if (file.endsWith(s: "_FIXTURE.js" )) |
237 | continue; |
238 | if (!filter.isEmpty() && !file.contains(s: filter)) |
239 | continue; |
240 | if (file.startsWith(s: annexB) || file.startsWith(s: harness) || file.startsWith(s: intl402)) |
241 | continue; |
242 | const char **excluded = excludedFilePatterns; |
243 | bool skip = false; |
244 | while (*excluded) { |
245 | if (file.contains(s: QLatin1String(*excluded))) |
246 | skip = true; |
247 | ++excluded; |
248 | } |
249 | if (skip) |
250 | continue; |
251 | |
252 | testCases.insert(akey: file, avalue: TestCase{ file }); |
253 | } |
254 | if (testCases.isEmpty()) { |
255 | qWarning() << "No tests to run." ; |
256 | return false; |
257 | } |
258 | |
259 | return true; |
260 | } |
261 | |
262 | |
263 | struct TestExpectationLine { |
264 | TestExpectationLine(const QByteArray &line); |
265 | enum State { |
266 | Fails, |
267 | SloppyFails, |
268 | StrictFails, |
269 | Skip, |
270 | Passes |
271 | } state; |
272 | QString testCase; |
273 | |
274 | QByteArray toLine() const; |
275 | void update(const TestCase &testCase); |
276 | |
277 | static TestExpectationLine fromTestCase(const TestCase &testCase); |
278 | private: |
279 | TestExpectationLine() = default; |
280 | static State stateFromTestCase(const TestCase &testCase); |
281 | }; |
282 | |
283 | TestExpectationLine::TestExpectationLine(const QByteArray &line) |
284 | { |
285 | int space = line.indexOf(c: ' '); |
286 | |
287 | testCase = QString::fromUtf8(str: space > 0 ? line.left(len: space) : line); |
288 | if (!testCase.endsWith(s: ".js" )) |
289 | testCase += ".js" ; |
290 | |
291 | state = Fails; |
292 | if (space < 0) |
293 | return; |
294 | QByteArray qualifier = line.mid(index: space + 1); |
295 | if (qualifier == "skip" ) |
296 | state = Skip; |
297 | else if (qualifier == "strictFails" ) |
298 | state = StrictFails; |
299 | else if (qualifier == "sloppyFails" ) |
300 | state = SloppyFails; |
301 | else if (qualifier == "fails" ) |
302 | state = Fails; |
303 | else |
304 | qWarning() << "illegal format in TestExpectations, line" << line; |
305 | } |
306 | |
307 | QByteArray TestExpectationLine::toLine() const { |
308 | const char *res = nullptr; |
309 | switch (state) { |
310 | case Fails: |
311 | res = " fails\n" ; |
312 | break; |
313 | case SloppyFails: |
314 | res = " sloppyFails\n" ; |
315 | break; |
316 | case StrictFails: |
317 | res = " strictFails\n" ; |
318 | break; |
319 | case Skip: |
320 | res = " skip\n" ; |
321 | break; |
322 | case Passes: |
323 | // no need for an entry |
324 | return QByteArray(); |
325 | } |
326 | QByteArray result = testCase.toUtf8() + res; |
327 | return result; |
328 | } |
329 | |
330 | void TestExpectationLine::update(const TestCase &testCase) |
331 | { |
332 | Q_ASSERT(testCase.test == this->testCase); |
333 | |
334 | State resultState = stateFromTestCase(testCase); |
335 | switch (resultState) { |
336 | case Fails: |
337 | // no improvement, don't update |
338 | break; |
339 | case SloppyFails: |
340 | if (state == Fails) |
341 | state = SloppyFails; |
342 | else if (state == StrictFails) |
343 | // we have a regression in sloppy mode, but strict now passes |
344 | state = Passes; |
345 | break; |
346 | case StrictFails: |
347 | if (state == Fails) |
348 | state = StrictFails; |
349 | else if (state == SloppyFails) |
350 | // we have a regression in strict mode, but sloppy now passes |
351 | state = Passes; |
352 | break; |
353 | case Skip: |
354 | Q_ASSERT(state == Skip); |
355 | // nothing to do |
356 | break; |
357 | case Passes: |
358 | state = Passes; |
359 | } |
360 | } |
361 | |
362 | TestExpectationLine TestExpectationLine::fromTestCase(const TestCase &testCase) |
363 | { |
364 | TestExpectationLine l; |
365 | l.testCase = testCase.test; |
366 | l.state = stateFromTestCase(testCase); |
367 | return l; |
368 | } |
369 | |
370 | TestExpectationLine::State TestExpectationLine::stateFromTestCase(const TestCase &testCase) |
371 | { |
372 | // keep skipped tests |
373 | if (testCase.skipTestCase) |
374 | return Skip; |
375 | |
376 | bool strictFails = (testCase.strictResult == TestCase::Crashes || testCase.strictResult == TestCase::Fails); |
377 | bool sloppyFails = (testCase.sloppyResult == TestCase::Crashes || testCase.sloppyResult == TestCase::Fails); |
378 | if (strictFails && sloppyFails) |
379 | return Fails; |
380 | if (strictFails) |
381 | return StrictFails; |
382 | if (sloppyFails) |
383 | return SloppyFails; |
384 | return Passes; |
385 | } |
386 | |
387 | |
388 | void Test262Runner::loadTestExpectations() |
389 | { |
390 | QFile file("TestExpectations" ); |
391 | if (!file.open(flags: QFile::ReadOnly)) { |
392 | qWarning() << "Could not open TestExpectations file." ; |
393 | return; |
394 | } |
395 | |
396 | int line = 0; |
397 | while (!file.atEnd()) { |
398 | ++line; |
399 | QByteArray line = file.readLine().trimmed(); |
400 | if (line.startsWith(c: '#') || line.isEmpty()) |
401 | continue; |
402 | TestExpectationLine expectation(line); |
403 | if (!filter.isEmpty() && !expectation.testCase.contains(s: filter)) |
404 | continue; |
405 | |
406 | if (!testCases.contains(akey: expectation.testCase)) |
407 | qWarning() << "Unknown test case" << expectation.testCase << "in TestExpectations file." ; |
408 | //qDebug() << "TestExpectations:" << expectation.testCase << expectation.state; |
409 | TestCase &s = testCases[expectation.testCase]; |
410 | switch (expectation.state) { |
411 | case TestExpectationLine::Fails: |
412 | s.strictExpectation = TestCase::Fails; |
413 | s.sloppyExpectation = TestCase::Fails; |
414 | break; |
415 | case TestExpectationLine::SloppyFails: |
416 | s.strictExpectation = TestCase::Passes; |
417 | s.sloppyExpectation = TestCase::Fails; |
418 | break; |
419 | case TestExpectationLine::StrictFails: |
420 | s.strictExpectation = TestCase::Fails; |
421 | s.sloppyExpectation = TestCase::Passes; |
422 | break; |
423 | case TestExpectationLine::Skip: |
424 | s.skipTestCase = true; |
425 | break; |
426 | case TestExpectationLine::Passes: |
427 | Q_UNREACHABLE(); |
428 | } |
429 | } |
430 | } |
431 | |
432 | void Test262Runner::updateTestExpectations() |
433 | { |
434 | QFile file("TestExpectations" ); |
435 | if (!file.open(flags: QFile::ReadOnly)) { |
436 | qWarning() << "Could not open TestExpectations file." ; |
437 | return; |
438 | } |
439 | |
440 | QTemporaryFile updatedExpectations; |
441 | updatedExpectations.open(); |
442 | |
443 | int line = 0; |
444 | while (!file.atEnd()) { |
445 | ++line; |
446 | QByteArray originalLine = file.readLine(); |
447 | QByteArray line = originalLine.trimmed(); |
448 | if (line.startsWith(c: '#') || line.isEmpty()) { |
449 | updatedExpectations.write(data: originalLine); |
450 | continue; |
451 | } |
452 | |
453 | TestExpectationLine expectation(line); |
454 | // qDebug() << "checking: " << expectation.testCase; |
455 | if (!testCases.contains(akey: expectation.testCase)) { |
456 | updatedExpectations.write(data: originalLine); |
457 | continue; |
458 | } |
459 | const TestCase &testcase = testCases.value(akey: expectation.testCase); |
460 | expectation.update(testCase: testcase); |
461 | |
462 | line = expectation.toLine(); |
463 | // qDebug() << "updated line:" << line; |
464 | updatedExpectations.write(data: line); |
465 | } |
466 | file.close(); |
467 | updatedExpectations.close(); |
468 | file.remove(); |
469 | qDebug() << updatedExpectations.fileName() << file.fileName(); |
470 | updatedExpectations.copy(newName: file.fileName()); |
471 | qDebug() << "Updated TestExpectations file written!" ; |
472 | } |
473 | |
474 | void Test262Runner::writeTestExpectations() |
475 | { |
476 | QFile file("TestExpectations" ); |
477 | |
478 | QTemporaryFile expectations; |
479 | expectations.open(); |
480 | |
481 | for (auto c : qAsConst(t&: testCases)) { |
482 | TestExpectationLine line = TestExpectationLine::fromTestCase(testCase: c); |
483 | expectations.write(data: line.toLine()); |
484 | } |
485 | |
486 | expectations.close(); |
487 | if (file.exists()) |
488 | file.remove(); |
489 | qDebug() << expectations.fileName() << file.fileName(); |
490 | expectations.copy(newName: file.fileName()); |
491 | qDebug() << "new TestExpectations file written!" ; |
492 | |
493 | } |
494 | |
495 | static bool executeTest(const QByteArray &data, bool runAsModule = false, const QString &testCasePath = QString(), const QByteArray &harnessForModules = QByteArray()) |
496 | { |
497 | QString testData = QString::fromUtf8(str: data.constData(), size: data.size()); |
498 | |
499 | QV4::ExecutionEngine vm; |
500 | |
501 | QV4::Scope scope(&vm); |
502 | |
503 | QV4::GlobalExtensions::init(globalObject: vm.globalObject, extensions: QJSEngine::ConsoleExtension | QJSEngine::GarbageCollectionExtension); |
504 | QV4::initD262(e: &vm); |
505 | |
506 | if (runAsModule) { |
507 | const QUrl rootModuleUrl = QUrl::fromLocalFile(localfile: testCasePath); |
508 | // inject all modules with the harness |
509 | QVector<QUrl> modulesToLoad = { rootModuleUrl }; |
510 | while (!modulesToLoad.isEmpty()) { |
511 | QUrl url = modulesToLoad.takeFirst(); |
512 | QQmlRefPointer<QV4::ExecutableCompilationUnit> module; |
513 | |
514 | QFile f(url.toLocalFile()); |
515 | if (f.open(flags: QIODevice::ReadOnly)) { |
516 | QByteArray content = harnessForModules + f.readAll(); |
517 | module = vm.compileModule(url: url.toString(), sourceCode: QString::fromUtf8(str: content.constData(), size: content.length()), sourceTimeStamp: QFileInfo(f).lastModified()); |
518 | if (vm.hasException) |
519 | break; |
520 | vm.injectModule(moduleUnit: module); |
521 | } else { |
522 | vm.throwError(QStringLiteral("Could not load module" )); |
523 | break; |
524 | } |
525 | |
526 | for (const QString &request: module->moduleRequests()) { |
527 | const QUrl absoluteRequest = module->finalUrl().resolved(relative: QUrl(request)); |
528 | if (!vm.modules.contains(akey: absoluteRequest)) |
529 | modulesToLoad << absoluteRequest; |
530 | } |
531 | } |
532 | |
533 | if (!vm.hasException) { |
534 | if (auto rootModuleUnit = vm.loadModule(url: rootModuleUrl)) { |
535 | if (rootModuleUnit->instantiate(engine: &vm)) |
536 | rootModuleUnit->evaluate(); |
537 | } |
538 | } |
539 | } else { |
540 | QV4::ScopedContext ctx(scope, vm.rootContext()); |
541 | |
542 | QV4::Script script(ctx, QV4::Compiler::ContextType::Global, testData); |
543 | script.parse(); |
544 | |
545 | if (!vm.hasException) |
546 | script.run(); |
547 | } |
548 | return !vm.hasException; |
549 | } |
550 | |
551 | class SingleTest : public QRunnable |
552 | { |
553 | public: |
554 | SingleTest(Test262Runner *runner, const TestData &data) |
555 | : runner(runner), data(data) |
556 | { |
557 | command = runner->command; |
558 | } |
559 | void run(); |
560 | |
561 | void runExternalTest(); |
562 | |
563 | QString command; |
564 | Test262Runner *runner; |
565 | TestData data; |
566 | }; |
567 | |
568 | void SingleTest::run() |
569 | { |
570 | if (!command.isEmpty()) { |
571 | runExternalTest(); |
572 | return; |
573 | } |
574 | |
575 | if (data.runInSloppyMode) { |
576 | bool ok = ::executeTest(data: data.content); |
577 | if (data.negative) |
578 | ok = !ok; |
579 | |
580 | data.sloppyResult = ok ? TestCase::Passes : TestCase::Fails; |
581 | } else { |
582 | data.sloppyResult = TestCase::Skipped; |
583 | } |
584 | if (data.runInStrictMode) { |
585 | const QString testCasePath = QFileInfo(runner->testDir + "/test/" + data.test).absoluteFilePath(); |
586 | QByteArray c = "'use strict';\n" + data.content; |
587 | bool ok = ::executeTest(data: c, runAsModule: data.runAsModuleCode, testCasePath, harnessForModules: data.harness); |
588 | if (data.negative) |
589 | ok = !ok; |
590 | |
591 | data.strictResult = ok ? TestCase::Passes : TestCase::Fails; |
592 | } else { |
593 | data.strictResult = TestCase::Skipped; |
594 | } |
595 | runner->addResult(result: data); |
596 | } |
597 | |
598 | void SingleTest::runExternalTest() |
599 | { |
600 | auto runTest = [this] (const char *, TestCase::Result *result) { |
601 | QTemporaryFile tempFile; |
602 | tempFile.open(); |
603 | tempFile.write(data: header); |
604 | tempFile.write(data: data.content); |
605 | tempFile.close(); |
606 | |
607 | QProcess process; |
608 | // if (flags & Verbose) |
609 | // process.setReadChannelMode(QProcess::ForwardedChannels); |
610 | |
611 | process.start(program: command, arguments: QStringList(tempFile.fileName())); |
612 | if (!process.waitForFinished(msecs: -1) || process.error() == QProcess::FailedToStart) { |
613 | qWarning() << "Could not execute" << command; |
614 | *result = TestCase::Crashes; |
615 | } |
616 | if (process.exitStatus() != QProcess::NormalExit) { |
617 | *result = TestCase::Crashes; |
618 | } |
619 | bool ok = (process.exitCode() == EXIT_SUCCESS); |
620 | if (data.negative) |
621 | ok = !ok; |
622 | *result = ok ? TestCase::Passes : TestCase::Fails; |
623 | }; |
624 | |
625 | if (data.runInSloppyMode) |
626 | runTest("" , &data.sloppyResult); |
627 | if (data.runInStrictMode) |
628 | runTest("'use strict';\n" , &data.strictResult); |
629 | |
630 | runner->addResult(result: data); |
631 | } |
632 | |
633 | int Test262Runner::runSingleTest(TestCase testCase) |
634 | { |
635 | TestData data = getTestData(testCase); |
636 | // qDebug() << "starting test" << data.test; |
637 | |
638 | if (data.isExcluded || data.async) |
639 | return 0; |
640 | |
641 | if (threadPool) { |
642 | SingleTest *test = new SingleTest(this, data); |
643 | threadPool->start(runnable: test); |
644 | return 0; |
645 | } |
646 | SingleTest test(this, data); |
647 | test.run(); |
648 | return 0; |
649 | } |
650 | |
651 | void Test262Runner::addResult(TestCase result) |
652 | { |
653 | { |
654 | QMutexLocker locker(&mutex); |
655 | Q_ASSERT(result.strictExpectation == testCases[result.test].strictExpectation); |
656 | Q_ASSERT(result.sloppyExpectation == testCases[result.test].sloppyExpectation); |
657 | testCases[result.test] = result; |
658 | } |
659 | |
660 | if (!(flags & Verbose)) |
661 | return; |
662 | |
663 | QString test = result.test; |
664 | if (result.strictResult == TestCase::Skipped) { |
665 | ; |
666 | } else if (result.strictResult == TestCase::Crashes) { |
667 | qDebug() << "FAIL:" << test << "crashed in strict mode!" ; |
668 | } else if ((result.strictResult == TestCase::Fails) && (result.strictExpectation == TestCase::Fails)) { |
669 | qDebug() << "PASS:" << test << "failed in strict mode as expected" ; |
670 | } else if ((result.strictResult == TestCase::Passes) == (result.strictExpectation == TestCase::Passes)) { |
671 | qDebug() << "PASS:" << test << "passed in strict mode" ; |
672 | } else if (!(result.strictExpectation == TestCase::Fails)) { |
673 | qDebug() << "FAIL:" << test << "failed in strict mode" ; |
674 | } else { |
675 | qDebug() << "XPASS:" << test << "unexpectedly passed in strict mode" ; |
676 | } |
677 | |
678 | if (result.sloppyResult == TestCase::Skipped) { |
679 | ; |
680 | } else if (result.sloppyResult == TestCase::Crashes) { |
681 | qDebug() << "FAIL:" << test << "crashed in sloppy mode!" ; |
682 | } else if ((result.sloppyResult == TestCase::Fails) && (result.sloppyExpectation == TestCase::Fails)) { |
683 | qDebug() << "PASS:" << test << "failed in sloppy mode as expected" ; |
684 | } else if ((result.sloppyResult == TestCase::Passes) == (result.sloppyExpectation == TestCase::Passes)) { |
685 | qDebug() << "PASS:" << test << "passed in sloppy mode" ; |
686 | } else if (!(result.sloppyExpectation == TestCase::Fails)) { |
687 | qDebug() << "FAIL:" << test << "failed in sloppy mode" ; |
688 | } else { |
689 | qDebug() << "XPASS:" << test << "unexpectedly passed in sloppy mode" ; |
690 | } |
691 | } |
692 | |
693 | TestData Test262Runner::getTestData(const TestCase &testCase) |
694 | { |
695 | QFile testFile(testDir + "/test/" + testCase.test); |
696 | if (!testFile.open(flags: QFile::ReadOnly)) { |
697 | qWarning() << "wrong test file" << testCase.test; |
698 | exit(status: 1); |
699 | } |
700 | QByteArray content = testFile.readAll(); |
701 | content.replace(QByteArrayLiteral("\r\n" ), c: "\n" ); |
702 | |
703 | // qDebug() << "parsing test file" << test; |
704 | |
705 | TestData data(testCase); |
706 | parseYaml(content, data: &data); |
707 | |
708 | data.harness += harness(name: "assert.js" ); |
709 | data.harness += harness(name: "sta.js" ); |
710 | |
711 | for (QByteArray inc : qAsConst(t&: data.includes)) { |
712 | inc = inc.trimmed(); |
713 | data.harness += harness(name: inc); |
714 | } |
715 | |
716 | if (data.async) |
717 | data.harness += harness(name: "doneprintHandle.js" ); |
718 | |
719 | data.content = data.harness + content; |
720 | |
721 | return data; |
722 | } |
723 | |
724 | struct YamlSection { |
725 | YamlSection(const QByteArray &yaml, const char *sectionName); |
726 | |
727 | bool contains(const char *keyword) const; |
728 | QList<QByteArray> keywords() const; |
729 | |
730 | QByteArray yaml; |
731 | int start = -1; |
732 | int length = 0; |
733 | bool shortSection = false; |
734 | }; |
735 | |
736 | YamlSection::YamlSection(const QByteArray &yaml, const char *sectionName) |
737 | : yaml(yaml) |
738 | { |
739 | start = yaml.indexOf(c: sectionName); |
740 | if (start < 0) |
741 | return; |
742 | start += static_cast<int>(strlen(s: sectionName)); |
743 | int end = yaml.indexOf(c: '\n', from: start + 1); |
744 | if (end < 0) |
745 | end = yaml.length(); |
746 | |
747 | int s = yaml.indexOf(c: '[', from: start); |
748 | if (s > 0 && s < end) { |
749 | shortSection = true; |
750 | start = s + 1; |
751 | end = yaml.indexOf(c: ']', from: s); |
752 | } else { |
753 | while (end < yaml.size() - 1 && yaml.at(i: end + 1) == ' ') |
754 | end = yaml.indexOf(c: '\n', from: end + 1); |
755 | } |
756 | length = end - start; |
757 | } |
758 | |
759 | bool YamlSection::contains(const char *keyword) const |
760 | { |
761 | if (start < 0) |
762 | return false; |
763 | int idx = yaml.indexOf(c: keyword, from: start); |
764 | if (idx >= start && idx < start + length) |
765 | return true; |
766 | return false; |
767 | } |
768 | |
769 | QList<QByteArray> YamlSection::keywords() const |
770 | { |
771 | if (start < 0) |
772 | return QList<QByteArray>(); |
773 | |
774 | QByteArray content = yaml.mid(index: start, len: length); |
775 | QList<QByteArray> keywords; |
776 | if (shortSection) { |
777 | keywords = content.split(sep: ','); |
778 | } else { |
779 | const QList<QByteArray> list = content.split(sep: '\n'); |
780 | for (const QByteArray &l : list) { |
781 | int i = 0; |
782 | while (i < l.size() && (l.at(i) == ' ' || l.at(i) == '-')) |
783 | ++i; |
784 | QByteArray entry = l.mid(index: i); |
785 | if (!entry.isEmpty()) |
786 | keywords.append(t: entry); |
787 | } |
788 | } |
789 | // qDebug() << "keywords:" << keywords; |
790 | return keywords; |
791 | } |
792 | |
793 | |
794 | void Test262Runner::parseYaml(const QByteArray &content, TestData *data) |
795 | { |
796 | int start = content.indexOf(c: "/*---" ); |
797 | if (start < 0) |
798 | return; |
799 | start += sizeof("/*---" ); |
800 | |
801 | int end = content.indexOf(c: "---*/" ); |
802 | if (end < 0) |
803 | return; |
804 | |
805 | QByteArray yaml = content.mid(index: start, len: end - start); |
806 | |
807 | if (yaml.contains(c: "negative:" )) |
808 | data->negative = true; |
809 | |
810 | YamlSection flags(yaml, "flags:" ); |
811 | data->runInSloppyMode = !flags.contains(keyword: "onlyStrict" ); |
812 | data->runInStrictMode = !flags.contains(keyword: "noStrict" ) && !flags.contains(keyword: "raw" ); |
813 | data->runAsModuleCode = flags.contains(keyword: "module" ); |
814 | data->async = flags.contains(keyword: "async" ); |
815 | |
816 | if (data->runAsModuleCode) { |
817 | data->runInStrictMode = true; |
818 | data->runInSloppyMode = false; |
819 | } |
820 | |
821 | YamlSection includes(yaml, "includes:" ); |
822 | data->includes = includes.keywords(); |
823 | |
824 | YamlSection features = YamlSection(yaml, "features:" ); |
825 | |
826 | const char **f = excludedFeatures; |
827 | while (*f) { |
828 | if (features.contains(keyword: *f)) { |
829 | data->isExcluded = true; |
830 | break; |
831 | } |
832 | ++f; |
833 | } |
834 | |
835 | // qDebug() << "Yaml:\n" << yaml; |
836 | } |
837 | |
838 | QByteArray Test262Runner::harness(const QByteArray &name) |
839 | { |
840 | if (harnessFiles.contains(akey: name)) |
841 | return harnessFiles.value(akey: name); |
842 | |
843 | QFile h(testDir + QLatin1String("/harness/" ) + name); |
844 | if (!h.open(flags: QFile::ReadOnly)) { |
845 | qWarning() << "Illegal test harness file" << name; |
846 | exit(status: 1); |
847 | } |
848 | |
849 | QByteArray content = h.readAll(); |
850 | harnessFiles.insert(akey: name, avalue: content); |
851 | return content; |
852 | } |
853 | |