1/*
2 * Copyright (C) 2009, 2010, 2012 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#ifndef LinkBuffer_h
27#define LinkBuffer_h
28
29#include <Platform.h>
30
31#if ENABLE(ASSEMBLER)
32
33#define DUMP_LINK_STATISTICS 0
34#define DUMP_CODE 0
35
36#define GLOBAL_THUNK_ID reinterpret_cast<void*>(static_cast<intptr_t>(-1))
37#define REGEXP_CODE_ID reinterpret_cast<void*>(static_cast<intptr_t>(-2))
38
39#include "JITCompilationEffort.h"
40#include "MacroAssembler.h"
41#include "Options.h"
42#include <wtf/DataLog.h>
43#include <wtf/Noncopyable.h>
44
45namespace JSC {
46
47class JSGlobalData;
48
49template <typename T>
50struct DefaultExecutableOffsetCalculator {
51 template <typename Assembler>
52 static T applyOffset(Assembler *, T src) { return src; }
53};
54
55// LinkBuffer:
56//
57// This class assists in linking code generated by the macro assembler, once code generation
58// has been completed, and the code has been copied to is final location in memory. At this
59// time pointers to labels within the code may be resolved, and relative offsets to external
60// addresses may be fixed.
61//
62// Specifically:
63// * Jump objects may be linked to external targets,
64// * The address of Jump objects may taken, such that it can later be relinked.
65// * The return address of a Call may be acquired.
66// * The address of a Label pointing into the code may be resolved.
67// * The value referenced by a DataLabel may be set.
68//
69template <typename MacroAssembler, template <typename T> class ExecutableOffsetCalculator>
70class LinkBufferBase {
71 WTF_MAKE_NONCOPYABLE(LinkBufferBase)
72 typedef MacroAssemblerCodeRef CodeRef;
73 typedef MacroAssemblerCodePtr CodePtr;
74 typedef typename MacroAssembler::Label Label;
75 typedef typename MacroAssembler::Jump Jump;
76 typedef typename MacroAssembler::PatchableJump PatchableJump;
77 typedef typename MacroAssembler::JumpList JumpList;
78 typedef typename MacroAssembler::Call Call;
79 typedef typename MacroAssembler::DataLabelCompact DataLabelCompact;
80 typedef typename MacroAssembler::DataLabel32 DataLabel32;
81 typedef typename MacroAssembler::DataLabelPtr DataLabelPtr;
82 typedef typename MacroAssembler::ConvertibleLoadLabel ConvertibleLoadLabel;
83
84public:
85 LinkBufferBase(JSGlobalData& globalData, MacroAssembler* masm, JITCompilationEffort effort = JITCompilationMustSucceed)
86 : m_size(0)
87 , m_code(0)
88 , m_assembler(masm)
89 , m_globalData(&globalData)
90#ifndef NDEBUG
91 , m_completed(false)
92 , m_effort(effort)
93#endif
94 {
95#ifdef NDEBUG
96 UNUSED_PARAM(effort)
97#endif
98 // Simon: Moved this to the sub-classes linkCode(ownerUID, effort);
99 }
100
101 ~LinkBufferBase()
102 {
103 ASSERT(m_completed || (!m_executableMemory && m_effort == JITCompilationCanFail));
104 }
105
106 bool didFailToAllocate() const
107 {
108 return !m_executableMemory;
109 }
110
111 bool isValid() const
112 {
113 return !didFailToAllocate();
114 }
115
116 // These methods are used to link or set values at code generation time.
117
118 void link(Call call, FunctionPtr function)
119 {
120 ASSERT(call.isFlagSet(Call::Linkable));
121 call.m_label = applyOffset(call.m_label);
122 MacroAssembler::linkCall(code(), call, function);
123 }
124
125 void link(Jump jump, CodeLocationLabel label)
126 {
127 jump.m_label = applyOffset(jump.m_label);
128 MacroAssembler::linkJump(code(), jump, label);
129 }
130
131 void link(JumpList list, CodeLocationLabel label)
132 {
133 for (unsigned i = 0; i < list.m_jumps.size(); ++i)
134 link(list.m_jumps[i], label);
135 }
136
137 void patch(DataLabelPtr label, void* value)
138 {
139 AssemblerLabel target = applyOffset(label.m_label);
140 MacroAssembler::linkPointer(code(), target, value);
141 }
142
143 void patch(DataLabelPtr label, CodeLocationLabel value)
144 {
145 AssemblerLabel target = applyOffset(label.m_label);
146 MacroAssembler::linkPointer(code(), target, value.executableAddress());
147 }
148
149 // These methods are used to obtain handles to allow the code to be relinked / repatched later.
150
151 CodeLocationCall locationOf(Call call)
152 {
153 ASSERT(call.isFlagSet(Call::Linkable));
154 ASSERT(!call.isFlagSet(Call::Near));
155 return CodeLocationCall(MacroAssembler::getLinkerAddress(code(), applyOffset(call.m_label)));
156 }
157
158 CodeLocationNearCall locationOfNearCall(Call call)
159 {
160 ASSERT(call.isFlagSet(Call::Linkable));
161 ASSERT(call.isFlagSet(Call::Near));
162 return CodeLocationNearCall(MacroAssembler::getLinkerAddress(code(), applyOffset(call.m_label)));
163 }
164
165 CodeLocationLabel locationOf(PatchableJump jump)
166 {
167 return CodeLocationLabel(MacroAssembler::getLinkerAddress(code(), applyOffset(jump.m_jump.m_label)));
168 }
169
170 CodeLocationLabel locationOf(Label label)
171 {
172 return CodeLocationLabel(MacroAssembler::getLinkerAddress(code(), applyOffset(label.m_label)));
173 }
174
175 CodeLocationDataLabelPtr locationOf(DataLabelPtr label)
176 {
177 return CodeLocationDataLabelPtr(MacroAssembler::getLinkerAddress(code(), applyOffset(label.m_label)));
178 }
179
180 CodeLocationDataLabel32 locationOf(DataLabel32 label)
181 {
182 return CodeLocationDataLabel32(MacroAssembler::getLinkerAddress(code(), applyOffset(label.m_label)));
183 }
184
185 CodeLocationDataLabelCompact locationOf(DataLabelCompact label)
186 {
187 return CodeLocationDataLabelCompact(MacroAssembler::getLinkerAddress(code(), applyOffset(label.m_label)));
188 }
189
190 CodeLocationConvertibleLoad locationOf(ConvertibleLoadLabel label)
191 {
192 return CodeLocationConvertibleLoad(MacroAssembler::getLinkerAddress(code(), applyOffset(label.m_label)));
193 }
194
195 // This method obtains the return address of the call, given as an offset from
196 // the start of the code.
197 unsigned returnAddressOffset(Call call)
198 {
199 call.m_label = applyOffset(call.m_label);
200 return MacroAssembler::getLinkerCallReturnOffset(call);
201 }
202
203 uint32_t offsetOf(Label label)
204 {
205 return applyOffset(label.m_label).m_offset;
206 }
207
208 // Upon completion of all patching 'FINALIZE_CODE()' should be called once to
209 // complete generation of the code. Alternatively, call
210 // finalizeCodeWithoutDisassembly() directly if you have your own way of
211 // displaying disassembly.
212
213 inline CodeRef finalizeCodeWithoutDisassembly();
214 inline CodeRef finalizeCodeWithDisassembly(const char *jitKind, const char* func);
215
216 CodePtr trampolineAt(Label label)
217 {
218 return CodePtr(MacroAssembler::AssemblerType_T::getRelocatedAddress(code(), applyOffset(label.m_label)));
219 }
220
221 void* debugAddress()
222 {
223 return m_code;
224 }
225
226 size_t debugSize()
227 {
228 return m_size;
229 }
230
231 inline bool makeExecutable();
232
233private:
234 template <typename T> T applyOffset(T src)
235 {
236 return ExecutableOffsetCalculator<T>::applyOffset(m_assembler, src);
237 }
238
239protected:
240 // Keep this private! - the underlying code should only be obtained externally via finalizeCode().
241 void* code()
242 {
243 return m_code;
244 }
245
246 inline void linkCode(void* ownerUID, JITCompilationEffort);
247
248 virtual void performFinalization();
249
250#if DUMP_LINK_STATISTICS
251 static void dumpLinkStatistics(void* code, size_t initialSize, size_t finalSize);
252#endif
253
254#if DUMP_CODE
255 static void dumpCode(void* code, size_t);
256#endif
257
258 RefPtr<ExecutableMemoryHandle> m_executableMemory;
259 size_t m_size;
260 void* m_code;
261 MacroAssembler* m_assembler;
262 JSGlobalData* m_globalData;
263protected:
264#ifndef NDEBUG
265 bool m_completed;
266 JITCompilationEffort m_effort;
267#endif
268};
269
270#define FINALIZE_CODE_IF(condition, linkBufferReference, jitKind, func) \
271 (UNLIKELY((condition)) \
272 ? ((linkBufferReference).finalizeCodeWithDisassembly (jitKind, func)) \
273 : (linkBufferReference).finalizeCodeWithoutDisassembly())
274
275// Use this to finalize code, like so:
276//
277// CodeRef code = FINALIZE_CODE(linkBuffer, ("my super thingy number %d", number));
278//
279// Which, in disassembly mode, will print:
280//
281// Generated JIT code for my super thingy number 42:
282// Code at [0x123456, 0x234567]:
283// 0x123456: mov $0, 0
284// 0x12345a: ret
285//
286// ... and so on.
287//
288// Note that the dataLogFArgumentsForHeading are only evaluated when showDisassembly
289// is true, so you can hide expensive disassembly-only computations inside there.
290
291#define FINALIZE_CODE(linkBufferReference, jitKind, func) \
292 FINALIZE_CODE_IF(Options::showDisassembly(), linkBufferReference, jitKind, func)
293
294#define FINALIZE_DFG_CODE(linkBufferReference, jitKind, func) \
295 FINALIZE_CODE_IF((Options::showDisassembly() || Options::showDFGDisassembly()), linkBufferReference, jitKind, func)
296
297
298template <typename MacroAssembler, template <typename T> class ExecutableOffsetCalculator>
299inline typename LinkBufferBase<MacroAssembler, ExecutableOffsetCalculator>::CodeRef LinkBufferBase<MacroAssembler, ExecutableOffsetCalculator>::finalizeCodeWithoutDisassembly()
300{
301 performFinalization();
302
303 return CodeRef(m_executableMemory);
304}
305
306template <typename MacroAssembler, template <typename T> class ExecutableOffsetCalculator>
307inline typename LinkBufferBase<MacroAssembler, ExecutableOffsetCalculator>::CodeRef LinkBufferBase<MacroAssembler, ExecutableOffsetCalculator>::finalizeCodeWithDisassembly(const char *jitKind, const char* func)
308{
309 ASSERT(Options::showDisassembly() || Options::showDFGDisassembly());
310
311 CodeRef result = finalizeCodeWithoutDisassembly();
312
313 dataLogF(format: "Generated %s code for function %s:", jitKind, func);
314
315 dataLogF(
316#if OS(WINDOWS)
317 " Code at [0x%p, 0x%p):",
318#else
319 format: " Code at [%p, %p):",
320#endif
321 result.code().executableAddress(), static_cast<char*>(result.code().executableAddress()) + result.size());
322 disassemble(result.code(), m_size, prefix: " ", out&: WTF::dataFile());
323
324 return result;
325}
326
327template <typename MacroAssembler, template <typename T> class ExecutableOffsetCalculator>
328inline void LinkBufferBase<MacroAssembler, ExecutableOffsetCalculator>::linkCode(void* ownerUID, JITCompilationEffort effort)
329{
330 UNUSED_PARAM(ownerUID)
331 UNUSED_PARAM(effort)
332 ASSERT(!m_code);
333 m_executableMemory = m_assembler->m_assembler.executableCopy(*m_globalData, ownerUID, effort);
334 if (!m_executableMemory)
335 return;
336 m_code = m_executableMemory->codeStart();
337 m_size = m_assembler->m_assembler.codeSize();
338 ASSERT(m_code);
339}
340
341template <typename MacroAssembler, template <typename T> class ExecutableOffsetCalculator>
342void LinkBufferBase<MacroAssembler, ExecutableOffsetCalculator>::performFinalization()
343{
344 // NOTE: This function is specialized in LinkBuffer<MacroAssemblerARMv7>
345#ifndef NDEBUG
346 ASSERT(!m_completed);
347 ASSERT(isValid());
348 m_completed = true;
349#endif
350
351 ASSERT(m_size <= INT_MAX);
352 MacroAssembler::cacheFlush(code(), m_size);
353}
354
355template <typename MacroAssembler, template <typename T> class ExecutableOffsetCalculator>
356inline bool LinkBufferBase<MacroAssembler, ExecutableOffsetCalculator>::makeExecutable()
357{
358 return ExecutableAllocator::makeExecutable(addr: m_executableMemory->memoryStart(),
359 size: m_executableMemory->memorySize());
360}
361
362template <typename MacroAssembler>
363class LinkBuffer : public LinkBufferBase<MacroAssembler, DefaultExecutableOffsetCalculator>
364{
365public:
366 LinkBuffer(JSGlobalData& globalData, MacroAssembler* masm, void* ownerUID, JITCompilationEffort effort = JITCompilationMustSucceed)
367 : LinkBufferBase<MacroAssembler, DefaultExecutableOffsetCalculator>(globalData, masm, effort)
368 {
369 this->linkCode(ownerUID, effort);
370 }
371};
372
373#if CPU(ARM_THUMB2) || CPU(ARM64)
374
375template <typename T>
376struct BranchCompactingExecutableOffsetCalculator {
377 template <typename Assembler>
378 static T applyOffset(Assembler *as, T src) {
379 src.m_offset -= as->executableOffsetFor(src.m_offset);
380 return src;
381 }
382};
383
384template <typename MacroAssembler>
385class BranchCompactingLinkBuffer : public LinkBufferBase<MacroAssembler, BranchCompactingExecutableOffsetCalculator>
386{
387public:
388 BranchCompactingLinkBuffer(JSGlobalData& globalData, MacroAssembler* masm, void* ownerUID, JITCompilationEffort effort = JITCompilationMustSucceed)
389 : LinkBufferBase<MacroAssembler, BranchCompactingExecutableOffsetCalculator>(globalData, masm, effort)
390 {
391 linkCode(ownerUID, effort);
392 }
393
394 virtual void performFinalization() override final;
395 inline bool makeExecutable();
396
397 inline void linkCode(void* ownerUID, JITCompilationEffort);
398
399private:
400 using Base = LinkBufferBase<MacroAssembler, BranchCompactingExecutableOffsetCalculator>;
401#ifndef NDEBUG
402 using Base::m_completed;
403#endif
404 using Base::isValid;
405 using Base::code;
406 using Base::m_code;
407 using Base::m_size;
408 using Base::m_assembler;
409 using Base::m_executableMemory;
410 using Base::m_globalData;
411
412 using LinkRecord = typename MacroAssembler::LinkRecord;
413 using JumpLinkType = typename MacroAssembler::JumpLinkType;
414
415 size_t m_initialSize = 0;
416};
417
418template <typename MacroAssembler>
419void BranchCompactingLinkBuffer<MacroAssembler>::performFinalization()
420{
421#ifndef NDEBUG
422 ASSERT(!m_completed);
423 ASSERT(isValid());
424 this->m_completed = true;
425#endif
426
427 MacroAssembler::cacheFlush(code(), m_size);
428}
429
430template <typename MacroAssembler>
431inline bool BranchCompactingLinkBuffer<MacroAssembler>::makeExecutable()
432{
433 return ExecutableAllocator::makeExecutable(code(), m_initialSize);
434}
435
436template <typename MacroAssembler>
437inline void BranchCompactingLinkBuffer<MacroAssembler>::linkCode(void* ownerUID, JITCompilationEffort effort)
438{
439 UNUSED_PARAM(ownerUID)
440 UNUSED_PARAM(effort)
441 ASSERT(!m_code);
442 m_initialSize = m_assembler->m_assembler.codeSize();
443 m_executableMemory = m_globalData->executableAllocator.allocate(*m_globalData, m_initialSize, ownerUID, effort);
444 if (!m_executableMemory)
445 return;
446 if (Q_UNLIKELY(!ExecutableAllocator::makeWritable(m_executableMemory->memoryStart(), m_executableMemory->memorySize()))) {
447 m_executableMemory = {};
448 return;
449 }
450 m_code = (uint8_t*)m_executableMemory->codeStart();
451 ASSERT(m_code);
452 uint8_t* inData = (uint8_t*)m_assembler->unlinkedCode();
453 uint8_t* outData = reinterpret_cast<uint8_t*>(m_code);
454 int readPtr = 0;
455 int writePtr = 0;
456 Vector<LinkRecord, 0, UnsafeVectorOverflow>& jumpsToLink = m_assembler->jumpsToLink();
457 unsigned jumpCount = unsigned(jumpsToLink.size());
458 for (unsigned i = 0; i < jumpCount; ++i) {
459 int offset = readPtr - writePtr;
460 ASSERT(!(offset & 1));
461
462 // Copy the instructions from the last jump to the current one.
463 unsigned regionSize = unsigned(jumpsToLink[i].from() - readPtr);
464 uint16_t* copySource = reinterpret_cast_ptr<uint16_t*>(inData + readPtr);
465 uint16_t* copyEnd = reinterpret_cast_ptr<uint16_t*>(inData + readPtr + regionSize);
466 uint16_t* copyDst = reinterpret_cast_ptr<uint16_t*>(outData + writePtr);
467 ASSERT(!(regionSize % 2));
468 ASSERT(!(readPtr % 2));
469 ASSERT(!(writePtr % 2));
470 while (copySource != copyEnd)
471 *copyDst++ = *copySource++;
472 m_assembler->recordLinkOffsets(readPtr, jumpsToLink[i].from(), offset);
473 readPtr += regionSize;
474 writePtr += regionSize;
475
476 // Calculate absolute address of the jump target, in the case of backwards
477 // branches we need to be precise, forward branches we are pessimistic
478 const uint8_t* target;
479 if (jumpsToLink[i].to() >= jumpsToLink[i].from())
480 target = outData + jumpsToLink[i].to() - offset; // Compensate for what we have collapsed so far
481 else
482 target = outData + jumpsToLink[i].to() - m_assembler->executableOffsetFor(jumpsToLink[i].to());
483
484 JumpLinkType jumpLinkType = m_assembler->computeJumpType(jumpsToLink[i], outData + writePtr, target);
485 // Compact branch if we can...
486 if (m_assembler->canCompact(jumpsToLink[i].type())) {
487 // Step back in the write stream
488 int32_t delta = m_assembler->jumpSizeDelta(jumpsToLink[i].type(), jumpLinkType);
489 if (delta) {
490 writePtr -= delta;
491 m_assembler->recordLinkOffsets(jumpsToLink[i].from() - delta, readPtr, readPtr - writePtr);
492 }
493 }
494 jumpsToLink[i].setFrom(writePtr);
495 }
496 // Copy everything after the last jump
497 memcpy(outData + writePtr, inData + readPtr, m_initialSize - readPtr);
498 m_assembler->recordLinkOffsets(readPtr, unsigned(m_initialSize), readPtr - writePtr);
499
500 for (unsigned i = 0; i < jumpCount; ++i) {
501 uint8_t* location = outData + jumpsToLink[i].from();
502 uint8_t* target = outData + jumpsToLink[i].to() - m_assembler->executableOffsetFor(jumpsToLink[i].to());
503 m_assembler->link(jumpsToLink[i], location, target);
504 }
505
506 jumpsToLink.clear();
507 m_size = writePtr + m_initialSize - readPtr;
508 m_executableMemory->shrink(m_size);
509}
510
511#if CPU(ARM_THUMB2)
512template <>
513class LinkBuffer<JSC::MacroAssembler<MacroAssemblerARMv7>> : public BranchCompactingLinkBuffer<JSC::MacroAssembler<MacroAssemblerARMv7>>
514{
515public:
516 LinkBuffer(JSGlobalData& globalData, JSC::MacroAssembler<MacroAssemblerARMv7>* masm, void* ownerUID, JITCompilationEffort effort = JITCompilationMustSucceed)
517 : BranchCompactingLinkBuffer<JSC::MacroAssembler<MacroAssemblerARMv7>>(globalData, masm, ownerUID, effort)
518 {}
519};
520#endif
521
522#if CPU(ARM64)
523template <>
524class LinkBuffer<JSC::MacroAssembler<MacroAssemblerARM64>> : public BranchCompactingLinkBuffer<JSC::MacroAssembler<MacroAssemblerARM64>>
525{
526public:
527 LinkBuffer(JSGlobalData& globalData, JSC::MacroAssembler<MacroAssemblerARM64>* masm, void* ownerUID, JITCompilationEffort effort = JITCompilationMustSucceed)
528 : BranchCompactingLinkBuffer<JSC::MacroAssembler<JSC::MacroAssemblerARM64>>(globalData, masm, ownerUID, effort)
529 {}
530};
531#endif
532
533#endif
534
535#if CPU(ARM_THUMB2)
536typedef LinkBuffer<MacroAssembler<MacroAssemblerARMv7>> DefaultLinkBuffer;
537#elif CPU(ARM64)
538typedef LinkBuffer<MacroAssembler<MacroAssemblerARM64>> DefaultLinkBuffer;
539#elif CPU(MIPS)
540typedef LinkBuffer<MacroAssembler<MacroAssemblerMIPS>> DefaultLinkBuffer;
541#elif CPU(X86)
542typedef LinkBuffer<MacroAssembler<MacroAssemblerX86>> DefaultLinkBuffer;
543#elif CPU(X86_64)
544typedef LinkBuffer<MacroAssembler<MacroAssemblerX86_64>> DefaultLinkBuffer;
545#endif
546
547} // namespace JSC
548
549#endif // ENABLE(ASSEMBLER)
550
551#endif // LinkBuffer_h
552

source code of qtdeclarative/src/3rdparty/masm/assembler/LinkBuffer.h