1 : /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : * vim: set ts=4 sw=4 et tw=99:
3 : *
4 : * ***** BEGIN LICENSE BLOCK *****
5 : * Version: MPL 1.1/GPL 2.0/LGPL 2.1
6 : *
7 : * The contents of this file are subject to the Mozilla Public License Version
8 : * 1.1 (the "License"); you may not use this file except in compliance with
9 : * the License. You may obtain a copy of the License at
10 : * http://www.mozilla.org/MPL/
11 : *
12 : * Software distributed under the License is distributed on an "AS IS" basis,
13 : * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
14 : * for the specific language governing rights and limitations under the
15 : * License.
16 : *
17 : * The Original Code is Mozilla Jaegermonkey.
18 : *
19 : * The Initial Developer of the Original Code is the Mozilla Foundation.
20 : *
21 : * Portions created by the Initial Developer are Copyright (C) 2010
22 : * the Initial Developer. All Rights Reserved.
23 : *
24 : * Contributor(s):
25 : * Andrew Drake <drakedevel@gmail.com>
26 : *
27 : * Alternatively, the contents of this file may be used under the terms of
28 : * either the GNU General Public License Version 2 or later (the "GPL"), or
29 : * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
30 : * in which case the provisions of the GPL or the LGPL are applicable instead
31 : * of those above. If you wish to allow use of your version of this file only
32 : * under the terms of either the GPL or the LGPL, and not to allow others to
33 : * use your version of this file under the terms of the MPL, indicate your
34 : * decision by deleting the provisions above and replace them with the notice
35 : * and other provisions required by the GPL or the LGPL. If you do not delete
36 : * the provisions above, a recipient may use your version of this file under
37 : * the terms of any one of the MPL, the GPL or the LGPL.
38 : *
39 : * ***** END LICENSE BLOCK ***** */
40 :
41 : #ifdef JS_METHODJIT
42 :
43 : #include "Retcon.h"
44 : #include "MethodJIT.h"
45 : #include "Compiler.h"
46 : #include "StubCalls.h"
47 : #include "jsdbgapi.h"
48 : #include "jsnum.h"
49 : #include "assembler/assembler/LinkBuffer.h"
50 : #include "assembler/assembler/RepatchBuffer.h"
51 :
52 : #include "jscntxtinlines.h"
53 : #include "jsinterpinlines.h"
54 :
55 : using namespace js;
56 : using namespace js::mjit;
57 :
58 : namespace js {
59 : namespace mjit {
60 :
61 : static inline void
62 109298 : SetRejoinState(StackFrame *fp, const CallSite &site, void **location)
63 : {
64 109298 : if (site.rejoin == REJOIN_SCRIPTED) {
65 70765 : fp->setRejoin(ScriptedRejoin(site.pcOffset));
66 70765 : *location = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolineScripted);
67 : } else {
68 38533 : fp->setRejoin(StubRejoin(site.rejoin));
69 38533 : *location = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpoline);
70 : }
71 109298 : }
72 :
73 : static inline bool
74 1212659 : CallsiteMatches(uint8_t *codeStart, const CallSite &site, void *location)
75 : {
76 1212659 : if (codeStart + site.codeOffset == location)
77 109298 : return true;
78 :
79 : #ifdef JS_CPU_ARM
80 : if (codeStart + site.codeOffset + 4 == location)
81 : return true;
82 : #endif
83 :
84 1103361 : return false;
85 : }
86 :
87 : void
88 109226 : Recompiler::patchCall(JITChunk *chunk, StackFrame *fp, void **location)
89 : {
90 109226 : uint8_t* codeStart = (uint8_t *)chunk->code.m_code.executableAddress();
91 :
92 109226 : CallSite *callSites_ = chunk->callSites();
93 1212530 : for (uint32_t i = 0; i < chunk->nCallSites; i++) {
94 1212530 : if (CallsiteMatches(codeStart, callSites_[i], *location)) {
95 109226 : JS_ASSERT(callSites_[i].inlineIndex == analyze::CrossScriptSSA::OUTER_FRAME);
96 109226 : SetRejoinState(fp, callSites_[i], location);
97 : return;
98 : }
99 : }
100 :
101 0 : JS_NOT_REACHED("failed to find call site");
102 : }
103 :
104 : void
105 884 : Recompiler::patchNative(JSCompartment *compartment, JITChunk *chunk, StackFrame *fp,
106 : jsbytecode *pc, RejoinState rejoin)
107 : {
108 : /*
109 : * There is a native call or getter IC at pc which triggered recompilation.
110 : * The recompilation could have been triggered either by the native call
111 : * itself, or by a SplatApplyArgs preparing for the native call. Either
112 : * way, we don't want to patch up the call, but will instead steal the pool
113 : * for the IC so it doesn't get freed with the JITChunk, and patch up the
114 : * jump at the end to go to the interpoline.
115 : *
116 : * When doing this, we do not reset the the IC itself; the JITChunk must
117 : * be dead and about to be released due to the recompilation (or a GC).
118 : */
119 884 : fp->setRejoin(StubRejoin(rejoin));
120 :
121 : /* :XXX: We might crash later if this fails. */
122 884 : compartment->jaegerCompartment()->orphanedNativeFrames.append(fp);
123 :
124 1768 : DebugOnly<bool> found = false;
125 :
126 : /*
127 : * Find and patch all native call stubs attached to the given PC. There may
128 : * be multiple ones for getter stubs attached to e.g. a GETELEM.
129 : */
130 7336 : for (unsigned i = 0; i < chunk->nativeCallStubs.length(); i++) {
131 6452 : NativeCallStub &stub = chunk->nativeCallStubs[i];
132 6452 : if (stub.pc != pc)
133 5568 : continue;
134 :
135 884 : found = true;
136 :
137 : /* Check for pools that were already patched. */
138 884 : if (!stub.pool)
139 590 : continue;
140 :
141 : /* Patch the native fallthrough to go to the interpoline. */
142 : {
143 : #if (defined(JS_NO_FASTCALL) && defined(JS_CPU_X86)) || defined(_WIN64)
144 : /* Win64 needs stack adjustment */
145 : void *interpoline = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolinePatched);
146 : #else
147 294 : void *interpoline = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpoline);
148 : #endif
149 294 : uint8_t *start = (uint8_t *)stub.jump.executableAddress();
150 588 : JSC::RepatchBuffer repatch(JSC::JITCode(start - 32, 64));
151 : #ifdef JS_CPU_X64
152 : repatch.repatch(stub.jump, interpoline);
153 : #else
154 294 : repatch.relink(stub.jump, JSC::CodeLocationLabel(interpoline));
155 : #endif
156 : }
157 :
158 : /* :XXX: We leak the pool if this fails. Oh well. */
159 294 : compartment->jaegerCompartment()->orphanedNativePools.append(stub.pool);
160 :
161 : /* Mark as stolen in case there are multiple calls on the stack. */
162 294 : stub.pool = NULL;
163 : }
164 :
165 884 : JS_ASSERT(found);
166 884 : }
167 :
168 : void
169 174366 : Recompiler::patchFrame(JSCompartment *compartment, VMFrame *f, JSScript *script)
170 : {
171 : /*
172 : * Check if the VMFrame returns directly into the script's jitcode. This
173 : * depends on the invariant that f->fp() reflects the frame at the point
174 : * where the call occurred, irregardless of any frames which were pushed
175 : * inside the call.
176 : */
177 174366 : StackFrame *fp = f->fp();
178 174366 : void **addr = f->returnAddressLocation();
179 174366 : RejoinState rejoin = (RejoinState) f->stubRejoin;
180 174366 : if (rejoin == REJOIN_NATIVE ||
181 : rejoin == REJOIN_NATIVE_LOWERED ||
182 : rejoin == REJOIN_NATIVE_GETTER) {
183 : /* Native call. */
184 1788 : if (fp->script() == script) {
185 884 : patchNative(compartment, fp->jit()->chunk(f->regs.pc), fp, f->regs.pc, rejoin);
186 884 : f->stubRejoin = REJOIN_NATIVE_PATCHED;
187 : }
188 173472 : } else if (rejoin == REJOIN_NATIVE_PATCHED) {
189 : /* Already patched, don't do anything. */
190 172881 : } else if (rejoin) {
191 : /* Recompilation triggered by CompileFunction. */
192 20 : if (fp->script() == script) {
193 1 : fp->setRejoin(StubRejoin(rejoin));
194 1 : *addr = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpoline);
195 1 : f->stubRejoin = 0;
196 : }
197 : } else {
198 172861 : if (script->jitHandleCtor.isValid()) {
199 333 : JITChunk *chunk = script->jitHandleCtor.getValid()->findCodeChunk(*addr);
200 333 : if (chunk)
201 318 : patchCall(chunk, fp, addr);
202 : }
203 172861 : if (script->jitHandleNormal.isValid()) {
204 43890 : JITChunk *chunk = script->jitHandleNormal.getValid()->findCodeChunk(*addr);
205 43890 : if (chunk)
206 38143 : patchCall(chunk, fp, addr);
207 : }
208 : }
209 174366 : }
210 :
211 : StackFrame *
212 140 : Recompiler::expandInlineFrameChain(StackFrame *outer, InlineFrame *inner)
213 : {
214 : StackFrame *parent;
215 140 : if (inner->parent)
216 11 : parent = expandInlineFrameChain(outer, inner->parent);
217 : else
218 129 : parent = outer;
219 :
220 140 : JaegerSpew(JSpew_Recompile, "Expanding inline frame\n");
221 :
222 140 : StackFrame *fp = (StackFrame *) ((uint8_t *)outer + sizeof(Value) * inner->depth);
223 140 : fp->initInlineFrame(inner->fun, parent, inner->parentpc);
224 140 : uint32_t pcOffset = inner->parentpc - parent->script()->code;
225 :
226 140 : void **location = fp->addressOfNativeReturnAddress();
227 140 : *location = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolineScripted);
228 140 : parent->setRejoin(ScriptedRejoin(pcOffset));
229 :
230 140 : return fp;
231 : }
232 :
233 : /*
234 : * Whether a given return address for a frame indicates it returns directly
235 : * into JIT code.
236 : */
237 : static inline bool
238 70937 : JITCodeReturnAddress(void *data)
239 : {
240 : return data != NULL /* frame is interpreted */
241 : && data != JS_FUNC_TO_DATA_PTR(void *, JaegerTrampolineReturn)
242 : && data != JS_FUNC_TO_DATA_PTR(void *, JaegerInterpoline)
243 : #if (defined(JS_NO_FASTCALL) && defined(JS_CPU_X86)) || defined(_WIN64)
244 : && data != JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolinePatched)
245 : #endif
246 70937 : && data != JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolineScripted);
247 : }
248 :
249 : /*
250 : * Expand all inlined frames within fp per 'inlined' and update next and regs
251 : * to refer to the new innermost frame.
252 : */
253 : void
254 129 : Recompiler::expandInlineFrames(JSCompartment *compartment,
255 : StackFrame *fp, mjit::CallSite *inlined,
256 : StackFrame *next, VMFrame *f)
257 : {
258 129 : JS_ASSERT_IF(next, next->prev() == fp && next->prevInline() == inlined);
259 :
260 : /*
261 : * Treat any frame expansion as a recompilation event, so that f.jit() is
262 : * stable if no recompilations have occurred.
263 : */
264 129 : compartment->types.frameExpansions++;
265 :
266 129 : jsbytecode *pc = next ? next->prevpc(NULL) : f->regs.pc;
267 129 : JITChunk *chunk = fp->jit()->chunk(pc);
268 :
269 : /*
270 : * Patch the VMFrame's return address if it is returning at the given inline site.
271 : * Note there is no worry about handling a native or CompileFunction call here,
272 : * as such IC stubs are not generated within inline frames.
273 : */
274 129 : void **frameAddr = f->returnAddressLocation();
275 129 : uint8_t* codeStart = (uint8_t *)chunk->code.m_code.executableAddress();
276 :
277 129 : InlineFrame *inner = &chunk->inlineFrames()[inlined->inlineIndex];
278 129 : jsbytecode *innerpc = inner->fun->script()->code + inlined->pcOffset;
279 :
280 129 : StackFrame *innerfp = expandInlineFrameChain(fp, inner);
281 :
282 : /* Check if the VMFrame returns into the inlined frame. */
283 129 : if (f->stubRejoin && f->fp() == fp) {
284 : /* The VMFrame is calling CompileFunction. */
285 0 : JS_ASSERT(f->stubRejoin != REJOIN_NATIVE &&
286 : f->stubRejoin != REJOIN_NATIVE_LOWERED &&
287 : f->stubRejoin != REJOIN_NATIVE_GETTER &&
288 0 : f->stubRejoin != REJOIN_NATIVE_PATCHED);
289 0 : innerfp->setRejoin(StubRejoin((RejoinState) f->stubRejoin));
290 0 : *frameAddr = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpoline);
291 0 : f->stubRejoin = 0;
292 : }
293 129 : if (CallsiteMatches(codeStart, *inlined, *frameAddr)) {
294 : /* The VMFrame returns directly into the expanded frame. */
295 72 : SetRejoinState(innerfp, *inlined, frameAddr);
296 : }
297 :
298 129 : if (f->fp() == fp) {
299 74 : JS_ASSERT(f->regs.inlined() == inlined);
300 74 : f->regs.expandInline(innerfp, innerpc);
301 : }
302 :
303 : /*
304 : * Note: unlike the case for recompilation, during frame expansion we don't
305 : * need to worry about the next VMFrame holding a reference to the inlined
306 : * frame in its entryncode. entryncode is non-NULL only if the next frame's
307 : * code was discarded and has executed via the Interpoline, which can only
308 : * happen after all inline frames have been expanded.
309 : */
310 :
311 129 : if (next) {
312 55 : next->resetInlinePrev(innerfp, innerpc);
313 55 : void **addr = next->addressOfNativeReturnAddress();
314 55 : if (JITCodeReturnAddress(*addr)) {
315 55 : innerfp->setRejoin(ScriptedRejoin(inlined->pcOffset));
316 55 : *addr = JS_FUNC_TO_DATA_PTR(void *, JaegerInterpolineScripted);
317 : }
318 : }
319 129 : }
320 :
321 : void
322 701752 : ExpandInlineFrames(JSCompartment *compartment)
323 : {
324 701752 : if (!compartment || !compartment->hasJaegerCompartment())
325 190264 : return;
326 :
327 44652336 : for (VMFrame *f = compartment->jaegerCompartment()->activeFrame();
328 : f != NULL;
329 : f = f->previous) {
330 :
331 44140848 : if (f->regs.inlined())
332 74 : mjit::Recompiler::expandInlineFrames(compartment, f->fp(), f->regs.inlined(), NULL, f);
333 :
334 44140848 : StackFrame *end = f->entryfp->prev();
335 44140848 : StackFrame *next = NULL;
336 90332965 : for (StackFrame *fp = f->fp(); fp != end; fp = fp->prev()) {
337 79941291 : if (!next) {
338 44140903 : next = fp;
339 44140903 : continue;
340 : }
341 : mjit::CallSite *inlined;
342 35800388 : next->prevpc(&inlined);
343 35800388 : if (inlined) {
344 55 : mjit::Recompiler::expandInlineFrames(compartment, fp, inlined, next, f);
345 55 : fp = next;
346 55 : next = NULL;
347 : } else {
348 35800333 : if (fp->downFramesExpanded())
349 33749174 : break;
350 2051159 : next = fp;
351 : }
352 2051214 : fp->setDownFramesExpanded();
353 : }
354 : }
355 : }
356 :
357 : void
358 94341 : ClearAllFrames(JSCompartment *compartment)
359 : {
360 94341 : if (!compartment || !compartment->hasJaegerCompartment())
361 70170 : return;
362 :
363 24171 : ExpandInlineFrames(compartment);
364 :
365 158202 : for (VMFrame *f = compartment->jaegerCompartment()->activeFrame();
366 : f != NULL;
367 : f = f->previous) {
368 :
369 134031 : Recompiler::patchFrame(compartment, f, f->fp()->script());
370 :
371 : // Clear ncode values from all frames associated with the VMFrame.
372 : // Patching the VMFrame's return address will cause all its frames to
373 : // finish in the interpreter, unless the interpreter enters one of the
374 : // intermediate frames at a loop boundary (where EnterMethodJIT will
375 : // overwrite ncode). However, leaving stale values for ncode in stack
376 : // frames can confuse the recompiler, which may see the VMFrame before
377 : // it has resumed execution.
378 :
379 188974 : for (StackFrame *fp = f->fp(); fp != f->entryfp; fp = fp->prev())
380 54943 : fp->setNativeReturnAddress(NULL);
381 : }
382 : }
383 :
384 : /*
385 : * Recompilation can be triggered either by the debugger (turning debug mode on for
386 : * a script or setting/clearing a trap), or by dynamic changes in type information
387 : * from type inference. When recompiling we don't immediately recompile the JIT
388 : * code, but destroy the old code and remove all references to the code, including
389 : * those from active stack frames. Things to do:
390 : *
391 : * - Purge scripted call inline caches calling into the script.
392 : *
393 : * - For frames with an ncode return address in the original script, redirect
394 : * to the interpoline.
395 : *
396 : * - For VMFrames with a stub call return address in the original script,
397 : * redirect to the interpoline.
398 : *
399 : * - For VMFrames whose entryncode address (the value of entryfp->ncode before
400 : * being clobbered with JaegerTrampolineReturn) is in the original script,
401 : * redirect that entryncode to the interpoline.
402 : */
403 : void
404 34485 : Recompiler::clearStackReferences(FreeOp *fop, JSScript *script)
405 : {
406 34485 : JS_ASSERT(script->hasJITCode());
407 :
408 : JaegerSpew(JSpew_Recompile, "recompiling script (file \"%s\") (line \"%d\") (length \"%d\")\n",
409 34485 : script->filename, script->lineno, script->length);
410 :
411 34485 : JSCompartment *comp = script->compartment();
412 68970 : types::AutoEnterTypeInference enter(fop, comp);
413 :
414 : /*
415 : * The strategy for this goes as follows:
416 : *
417 : * 1) Scan the stack, looking at all return addresses that could go into JIT
418 : * code.
419 : * 2) If an address corresponds to a call site registered by |callSite| during
420 : * the last compilation, patch it to go to the interpoline.
421 : * 3) Purge the old compiled state.
422 : */
423 :
424 : // Find all JIT'd stack frames to account for return addresses that will
425 : // need to be patched after recompilation.
426 74820 : for (VMFrame *f = comp->jaegerCompartment()->activeFrame();
427 : f != NULL;
428 : f = f->previous) {
429 :
430 : // Scan all frames owned by this VMFrame.
431 40335 : StackFrame *end = f->entryfp->prev();
432 40335 : StackFrame *next = NULL;
433 189748 : for (StackFrame *fp = f->fp(); fp != end; fp = fp->prev()) {
434 149413 : if (fp->script() != script) {
435 43829 : next = fp;
436 43829 : continue;
437 : }
438 :
439 105584 : if (next) {
440 : // check for a scripted call returning into the recompiled script.
441 : // this misses scanning the entry fp, which cannot return directly
442 : // into JIT code.
443 70882 : void **addr = next->addressOfNativeReturnAddress();
444 :
445 70882 : if (JITCodeReturnAddress(*addr)) {
446 70765 : JITChunk *chunk = fp->jit()->findCodeChunk(*addr);
447 70765 : patchCall(chunk, fp, addr);
448 : }
449 : }
450 :
451 105584 : next = fp;
452 : }
453 :
454 40335 : patchFrame(comp, f, script);
455 : }
456 :
457 34485 : comp->types.recompilations++;
458 34485 : }
459 :
460 : void
461 33843 : Recompiler::clearStackReferencesAndChunk(FreeOp *fop, JSScript *script,
462 : JITScript *jit, size_t chunkIndex,
463 : bool resetUses)
464 : {
465 33843 : Recompiler::clearStackReferences(fop, script);
466 :
467 33843 : bool releaseChunk = true;
468 33843 : if (jit->nchunks > 1) {
469 : // If we are in the middle of a native call from a native or getter IC,
470 : // we need to make sure all JIT code for the script is purged, as
471 : // otherwise we will have orphaned the native stub but pointers to it
472 : // still exist in the containing chunk.
473 1428 : for (VMFrame *f = script->compartment()->jaegerCompartment()->activeFrame();
474 : f != NULL;
475 : f = f->previous) {
476 681 : if (f->fp()->script() == script) {
477 0 : JS_ASSERT(f->stubRejoin != REJOIN_NATIVE &&
478 : f->stubRejoin != REJOIN_NATIVE_LOWERED &&
479 678 : f->stubRejoin != REJOIN_NATIVE_GETTER);
480 678 : if (f->stubRejoin == REJOIN_NATIVE_PATCHED) {
481 95 : mjit::ReleaseScriptCode(fop, script);
482 95 : releaseChunk = false;
483 95 : break;
484 : }
485 : }
486 : }
487 : }
488 :
489 33843 : if (releaseChunk)
490 33748 : jit->destroyChunk(fop, chunkIndex, resetUses);
491 33843 : }
492 :
493 : } /* namespace mjit */
494 : } /* namespace js */
495 :
496 : #endif /* JS_METHODJIT */
497 :
|