?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'master' into v2.1
[luajit-2.0.git] / src / lj_record.c
blob6543f2745c0f1b4f6c9db5da969179c36b4c5060
1 /*
2 ** Trace recorder (bytecode -> SSA IR).
3 ** Copyright (C) 2005-2025 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 #define lj_record_c
7 #define LUA_CORE
8
9 #include "lj_obj.h"
10
11 #if LJ_HASJIT
12
13 #include "lj_err.h"
14 #include "lj_str.h"
15 #include "lj_tab.h"
16 #include "lj_meta.h"
17 #include "lj_frame.h"
18 #if LJ_HASFFI
19 #include "lj_ctype.h"
20 #endif
21 #include "lj_bc.h"
22 #include "lj_ff.h"
23 #if LJ_HASPROFILE
24 #include "lj_debug.h"
25 #endif
26 #include "lj_ir.h"
27 #include "lj_jit.h"
28 #include "lj_ircall.h"
29 #include "lj_iropt.h"
30 #include "lj_trace.h"
31 #include "lj_record.h"
32 #include "lj_ffrecord.h"
33 #include "lj_snap.h"
34 #include "lj_dispatch.h"
35 #include "lj_vm.h"
36 #include "lj_prng.h"
37
38 /* Some local macros to save typing. Undef'd at the end. */
39 #define IR(ref) (&J->cur.ir[(ref)])
40
41 /* Pass IR on to next optimization in chain (FOLD). */
42 #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
43
44 /* Emit raw IR without passing through optimizations. */
45 #define emitir_raw(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J))
46
47 /* -- Sanity checks ------------------------------------------------------- */
48
49 #ifdef LUA_USE_ASSERT
50 /* Sanity check the whole IR -- sloooow. */
51 static void rec_check_ir(jit_State *J)
52 {
53 IRRef i, nins = J->cur.nins, nk = J->cur.nk;
54 lj_assertJ(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536,
55 "inconsistent IR layout");
56 for (i = nk; i < nins; i++) {
57 IRIns *ir = IR(i);
58 uint32_t mode = lj_ir_mode[ir->o];
59 IRRef op1 = ir->op1;
60 IRRef op2 = ir->op2;
61 const char *err = NULL;
62 switch (irm_op1(mode)) {
63 case IRMnone:
64 if (op1 != 0) err = "IRMnone op1 used";
65 break;
66 case IRMref:
67 if (op1 < nk || (i >= REF_BIAS ? op1 >= i : op1 <= i))
68 err = "IRMref op1 out of range";
69 break;
70 case IRMlit: break;
71 case IRMcst:
72 if (i >= REF_BIAS) { err = "constant in IR range"; break; }
73 if (irt_is64(ir->t) && ir->o != IR_KNULL)
74 i++;
75 continue;
76 }
77 switch (irm_op2(mode)) {
78 case IRMnone:
79 if (op2) err = "IRMnone op2 used";
80 break;
81 case IRMref:
82 if (op2 < nk || (i >= REF_BIAS ? op2 >= i : op2 <= i))
83 err = "IRMref op2 out of range";
84 break;
85 case IRMlit: break;
86 case IRMcst: err = "IRMcst op2"; break;
87 }
88 if (!err && ir->prev) {
89 if (ir->prev < nk || (i >= REF_BIAS ? ir->prev >= i : ir->prev <= i))
90 err = "chain out of range";
91 else if (ir->o != IR_NOP && IR(ir->prev)->o != ir->o)
92 err = "chain to different op";
93 }
94 lj_assertJ(!err, "bad IR %04d op %d(%04d,%04d): %s",
95 i-REF_BIAS,
96 ir->o,
97 irm_op1(mode) == IRMref ? op1-REF_BIAS : op1,
98 irm_op2(mode) == IRMref ? op2-REF_BIAS : op2,
99 err);
100 }
101 }
102
103 /* Compare stack slots and frames of the recorder and the VM. */
104 static void rec_check_slots(jit_State *J)
105 {
106 BCReg s, nslots = J->baseslot + J->maxslot;
107 int32_t depth = 0;
108 cTValue *base = J->L->base - J->baseslot;
109 lj_assertJ(J->baseslot >= 1+LJ_FR2, "bad baseslot");
110 lj_assertJ(J->baseslot == 1+LJ_FR2 || (J->slot[J->baseslot-1] & TREF_FRAME),
111 "baseslot does not point to frame");
112 lj_assertJ(nslots <= LJ_MAX_JSLOTS, "slot overflow");
113 for (s = 0; s < nslots; s++) {
114 TRef tr = J->slot[s];
115 if (tr) {
116 cTValue *tv = &base[s];
117 IRRef ref = tref_ref(tr);
118 IRIns *ir = NULL; /* Silence compiler. */
119 lj_assertJ(tv < J->L->top, "slot %d above top of Lua stack", s);
120 if (!LJ_FR2 || ref || !(tr & (TREF_FRAME | TREF_CONT))) {
121 lj_assertJ(ref >= J->cur.nk && ref < J->cur.nins,
122 "slot %d ref %04d out of range", s, ref - REF_BIAS);
123 ir = IR(ref);
124 lj_assertJ(irt_t(ir->t) == tref_t(tr), "slot %d IR type mismatch", s);
125 }
126 if (s == 0) {
127 lj_assertJ(tref_isfunc(tr), "frame slot 0 is not a function");
128 #if LJ_FR2
129 } else if (s == 1) {
130 lj_assertJ((tr & ~TREF_FRAME) == 0, "bad frame slot 1");
131 #endif
132 } else if ((tr & TREF_FRAME)) {
133 GCfunc *fn = gco2func(frame_gc(tv));
134 BCReg delta = (BCReg)(tv - frame_prev(tv));
135 #if LJ_FR2
136 lj_assertJ(!ref || ir_knum(ir)->u64 == tv->u64,
137 "frame slot %d PC mismatch", s);
138 tr = J->slot[s-1];
139 ir = IR(tref_ref(tr));
140 #endif
141 lj_assertJ(tref_isfunc(tr),
142 "frame slot %d is not a function", s-LJ_FR2);
143 lj_assertJ(!tref_isk(tr) || fn == ir_kfunc(ir),
144 "frame slot %d function mismatch", s-LJ_FR2);
145 lj_assertJ(s > delta + LJ_FR2 ? (J->slot[s-delta] & TREF_FRAME)
146 : (s == delta + LJ_FR2),
147 "frame slot %d broken chain", s-LJ_FR2);
148 depth++;
149 } else if ((tr & TREF_CONT)) {
150 #if LJ_FR2
151 lj_assertJ(!ref || ir_knum(ir)->u64 == tv->u64,
152 "cont slot %d continuation mismatch", s);
153 #else
154 lj_assertJ(ir_kptr(ir) == gcrefp(tv->gcr, void),
155 "cont slot %d continuation mismatch", s);
156 #endif
157 lj_assertJ((J->slot[s+1+LJ_FR2] & TREF_FRAME),
158 "cont slot %d not followed by frame", s);
159 depth++;
160 } else if ((tr & TREF_KEYINDEX)) {
161 lj_assertJ(tref_isint(tr), "keyindex slot %d bad type %d",
162 s, tref_type(tr));
163 } else {
164 /* Number repr. may differ, but other types must be the same. */
165 lj_assertJ(tvisnumber(tv) ? tref_isnumber(tr) :
166 itype2irt(tv) == tref_type(tr),
167 "slot %d type mismatch: stack type %d vs IR type %d",
168 s, itypemap(tv), tref_type(tr));
169 if (tref_isk(tr)) { /* Compare constants. */
170 TValue tvk;
171 lj_ir_kvalue(J->L, &tvk, ir);
172 lj_assertJ((tvisnum(&tvk) && tvisnan(&tvk)) ?
173 (tvisnum(tv) && tvisnan(tv)) :
174 lj_obj_equal(tv, &tvk),
175 "slot %d const mismatch: stack %016llx vs IR %016llx",
176 s, tv->u64, tvk.u64);
177 }
178 }
179 }
180 }
181 lj_assertJ(J->framedepth == depth,
182 "frame depth mismatch %d vs %d", J->framedepth, depth);
183 }
184 #endif
185
186 /* -- Type handling and specialization ------------------------------------ */
187
188 /* Note: these functions return tagged references (TRef). */
189
190 /* Specialize a slot to a specific type. Note: slot can be negative! */
191 static TRef sloadt(jit_State *J, int32_t slot, IRType t, int mode)
192 {
193 /* Caller may set IRT_GUARD in t. */
194 TRef ref = emitir_raw(IRT(IR_SLOAD, t), (int32_t)J->baseslot+slot, mode);
195 J->base[slot] = ref;
196 return ref;
197 }
198
199 /* Specialize a slot to the runtime type. Note: slot can be negative! */
200 static TRef sload(jit_State *J, int32_t slot)
201 {
202 IRType t = itype2irt(&J->L->base[slot]);
203 TRef ref = emitir_raw(IRTG(IR_SLOAD, t), (int32_t)J->baseslot+slot,
204 IRSLOAD_TYPECHECK);
205 if (irtype_ispri(t)) ref = TREF_PRI(t); /* Canonicalize primitive refs. */
206 J->base[slot] = ref;
207 return ref;
208 }
209
210 /* Get TRef from slot. Load slot and specialize if not done already. */
211 #define getslot(J, s) (J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s)))
212
213 /* Get TRef for current function. */
214 static TRef getcurrf(jit_State *J)
215 {
216 if (J->base[-1-LJ_FR2])
217 return J->base[-1-LJ_FR2];
218 /* Non-base frame functions ought to be loaded already. */
219 lj_assertJ(J->baseslot == 1+LJ_FR2, "bad baseslot");
220 return sloadt(J, -1-LJ_FR2, IRT_FUNC, IRSLOAD_READONLY);
221 }
222
223 /* Compare for raw object equality.
224 ** Returns 0 if the objects are the same.
225 ** Returns 1 if they are different, but the same type.
226 ** Returns 2 for two different types.
227 ** Comparisons between primitives always return 1 -- no caller cares about it.
228 */
229 int lj_record_objcmp(jit_State *J, TRef a, TRef b, cTValue *av, cTValue *bv)
230 {
231 int diff = !lj_obj_equal(av, bv);
232 if (!tref_isk2(a, b)) { /* Shortcut, also handles primitives. */
233 IRType ta = tref_isinteger(a) ? IRT_INT : tref_type(a);
234 IRType tb = tref_isinteger(b) ? IRT_INT : tref_type(b);
235 if (ta != tb) {
236 /* Widen mixed number/int comparisons to number/number comparison. */
237 if (ta == IRT_INT && tb == IRT_NUM) {
238 a = emitir(IRTN(IR_CONV), a, IRCONV_NUM_INT);
239 ta = IRT_NUM;
240 } else if (ta == IRT_NUM && tb == IRT_INT) {
241 b = emitir(IRTN(IR_CONV), b, IRCONV_NUM_INT);
242 } else {
243 return 2; /* Two different types are never equal. */
244 }
245 }
246 emitir(IRTG(diff ? IR_NE : IR_EQ, ta), a, b);
247 }
248 return diff;
249 }
250
251 /* Constify a value. Returns 0 for non-representable object types. */
252 TRef lj_record_constify(jit_State *J, cTValue *o)
253 {
254 if (tvisgcv(o))
255 return lj_ir_kgc(J, gcV(o), itype2irt(o));
256 else if (tvisint(o))
257 return lj_ir_kint(J, intV(o));
258 else if (tvisnum(o))
259 return lj_ir_knumint(J, numV(o));
260 else if (tvisbool(o))
261 return TREF_PRI(itype2irt(o));
262 else
263 return 0; /* Can't represent lightuserdata (pointless). */
264 }
265
266 /* Emit a VLOAD with the correct type. */
267 TRef lj_record_vload(jit_State *J, TRef ref, MSize idx, IRType t)
268 {
269 TRef tr = emitir(IRTG(IR_VLOAD, t), ref, idx);
270 if (irtype_ispri(t)) tr = TREF_PRI(t); /* Canonicalize primitives. */
271 return tr;
272 }
273
274 /* -- Record loop ops ----------------------------------------------------- */
275
276 /* Loop event. */
277 typedef enum {
278 LOOPEV_LEAVE, /* Loop is left or not entered. */
279 LOOPEV_ENTERLO, /* Loop is entered with a low iteration count left. */
280 LOOPEV_ENTER /* Loop is entered. */
281 } LoopEvent;
282
283 /* Canonicalize slots: convert integers to numbers. */
284 static void canonicalize_slots(jit_State *J)
285 {
286 BCReg s;
287 if (LJ_DUALNUM) return;
288 for (s = J->baseslot+J->maxslot-1; s >= 1; s--) {
289 TRef tr = J->slot[s];
290 if (tref_isinteger(tr) && !(tr & TREF_KEYINDEX)) {
291 IRIns *ir = IR(tref_ref(tr));
292 if (!(ir->o == IR_SLOAD && (ir->op2 & (IRSLOAD_READONLY))))
293 J->slot[s] = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
294 }
295 }
296 }
297
298 /* Stop recording. */
299 void lj_record_stop(jit_State *J, TraceLink linktype, TraceNo lnk)
300 {
301 #ifdef LUAJIT_ENABLE_TABLE_BUMP
302 if (J->retryrec)
303 lj_trace_err(J, LJ_TRERR_RETRY);
304 #endif
305 lj_trace_end(J);
306 J->cur.linktype = (uint8_t)linktype;
307 J->cur.link = (uint16_t)lnk;
308 /* Looping back at the same stack level? */
309 if (lnk == J->cur.traceno && J->framedepth + J->retdepth == 0) {
310 if ((J->flags & JIT_F_OPT_LOOP)) /* Shall we try to create a loop? */
311 goto nocanon; /* Do not canonicalize or we lose the narrowing. */
312 if (J->cur.root) /* Otherwise ensure we always link to the root trace. */
313 J->cur.link = J->cur.root;
314 }
315 canonicalize_slots(J);
316 nocanon:
317 /* Note: all loop ops must set J->pc to the following instruction! */
318 lj_snap_add(J); /* Add loop snapshot. */
319 J->needsnap = 0;
320 J->mergesnap = 1; /* In case recording continues. */
321 }
322
323 /* Search bytecode backwards for a int/num constant slot initializer. */
324 static TRef find_kinit(jit_State *J, const BCIns *endpc, BCReg slot, IRType t)
325 {
326 /* This algorithm is rather simplistic and assumes quite a bit about
327 ** how the bytecode is generated. It works fine for FORI initializers,
328 ** but it won't necessarily work in other cases (e.g. iterator arguments).
329 ** It doesn't do anything fancy, either (like backpropagating MOVs).
330 */
331 const BCIns *pc, *startpc = proto_bc(J->pt);
332 for (pc = endpc-1; pc > startpc; pc--) {
333 BCIns ins = *pc;
334 BCOp op = bc_op(ins);
335 /* First try to find the last instruction that stores to this slot. */
336 if (bcmode_a(op) == BCMbase && bc_a(ins) <= slot) {
337 return 0; /* Multiple results, e.g. from a CALL or KNIL. */
338 } else if (bcmode_a(op) == BCMdst && bc_a(ins) == slot) {
339 if (op == BC_KSHORT || op == BC_KNUM) { /* Found const. initializer. */
340 /* Now try to verify there's no forward jump across it. */
341 const BCIns *kpc = pc;
342 for (; pc > startpc; pc--)
343 if (bc_op(*pc) == BC_JMP) {
344 const BCIns *target = pc+bc_j(*pc)+1;
345 if (target > kpc && target <= endpc)
346 return 0; /* Conditional assignment. */
347 }
348 if (op == BC_KSHORT) {
349 int32_t k = (int32_t)(int16_t)bc_d(ins);
350 return t == IRT_INT ? lj_ir_kint(J, k) : lj_ir_knum(J, (lua_Number)k);
351 } else {
352 cTValue *tv = proto_knumtv(J->pt, bc_d(ins));
353 if (t == IRT_INT) {
354 int32_t k = numberVint(tv);
355 if (tvisint(tv) || numV(tv) == (lua_Number)k) /* -0 is ok here. */
356 return lj_ir_kint(J, k);
357 return 0; /* Type mismatch. */
358 } else {
359 return lj_ir_knum(J, numberVnum(tv));
360 }
361 }
362 }
363 return 0; /* Non-constant initializer. */
364 }
365 }
366 return 0; /* No assignment to this slot found? */
367 }
368
369 /* Load and optionally convert a FORI argument from a slot. */
370 static TRef fori_load(jit_State *J, BCReg slot, IRType t, int mode)
371 {
372 int conv = (tvisint(&J->L->base[slot]) != (t==IRT_INT)) ? IRSLOAD_CONVERT : 0;
373 return sloadt(J, (int32_t)slot,
374 t + (((mode & IRSLOAD_TYPECHECK) ||
375 (conv && t == IRT_INT && !(mode >> 16))) ?
376 IRT_GUARD : 0),
377 mode + conv);
378 }
379
380 /* Peek before FORI to find a const initializer. Otherwise load from slot. */
381 static TRef fori_arg(jit_State *J, const BCIns *fori, BCReg slot,
382 IRType t, int mode)
383 {
384 TRef tr = J->base[slot];
385 if (!tr) {
386 tr = find_kinit(J, fori, slot, t);
387 if (!tr)
388 tr = fori_load(J, slot, t, mode);
389 }
390 return tr;
391 }
392
393 /* Return the direction of the FOR loop iterator.
394 ** It's important to exactly reproduce the semantics of the interpreter.
395 */
396 static int rec_for_direction(cTValue *o)
397 {
398 return (tvisint(o) ? intV(o) : (int32_t)o->u32.hi) >= 0;
399 }
400
401 /* Simulate the runtime behavior of the FOR loop iterator. */
402 static LoopEvent rec_for_iter(IROp *op, cTValue *o, int isforl)
403 {
404 lua_Number stopv = numberVnum(&o[FORL_STOP]);
405 lua_Number idxv = numberVnum(&o[FORL_IDX]);
406 lua_Number stepv = numberVnum(&o[FORL_STEP]);
407 if (isforl)
408 idxv += stepv;
409 if (rec_for_direction(&o[FORL_STEP])) {
410 if (idxv <= stopv) {
411 *op = IR_LE;
412 return idxv + 2*stepv > stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER;
413 }
414 *op = IR_GT; return LOOPEV_LEAVE;
415 } else {
416 if (stopv <= idxv) {
417 *op = IR_GE;
418 return idxv + 2*stepv < stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER;
419 }
420 *op = IR_LT; return LOOPEV_LEAVE;
421 }
422 }
423
424 /* Record checks for FOR loop overflow and step direction. */
425 static void rec_for_check(jit_State *J, IRType t, int dir,
426 TRef stop, TRef step, int init)
427 {
428 if (!tref_isk(step)) {
429 /* Non-constant step: need a guard for the direction. */
430 TRef zero = (t == IRT_INT) ? lj_ir_kint(J, 0) : lj_ir_knum_zero(J);
431 emitir(IRTG(dir ? IR_GE : IR_LT, t), step, zero);
432 /* Add hoistable overflow checks for a narrowed FORL index. */
433 if (init && t == IRT_INT) {
434 if (tref_isk(stop)) {
435 /* Constant stop: optimize check away or to a range check for step. */
436 int32_t k = IR(tref_ref(stop))->i;
437 if (dir) {
438 if (k > 0)
439 emitir(IRTGI(IR_LE), step, lj_ir_kint(J, (int32_t)0x7fffffff-k));
440 } else {
441 if (k < 0)
442 emitir(IRTGI(IR_GE), step, lj_ir_kint(J, (int32_t)0x80000000-k));
443 }
444 } else {
445 /* Stop+step variable: need full overflow check. */
446 TRef tr = emitir(IRTGI(IR_ADDOV), step, stop);
447 emitir(IRTI(IR_USE), tr, 0); /* ADDOV is weak. Avoid dead result. */
448 }
449 }
450 } else if (init && t == IRT_INT && !tref_isk(stop)) {
451 /* Constant step: optimize overflow check to a range check for stop. */
452 int32_t k = IR(tref_ref(step))->i;
453 k = (int32_t)(dir ? 0x7fffffff : 0x80000000) - k;
454 emitir(IRTGI(dir ? IR_LE : IR_GE), stop, lj_ir_kint(J, k));
455 }
456 }
457
458 /* Record a FORL instruction. */
459 static void rec_for_loop(jit_State *J, const BCIns *fori, ScEvEntry *scev,
460 int init)
461 {
462 BCReg ra = bc_a(*fori);
463 cTValue *tv = &J->L->base[ra];
464 TRef idx = J->base[ra+FORL_IDX];
465 IRType t = idx ? tref_type(idx) :
466 (init || LJ_DUALNUM) ? lj_opt_narrow_forl(J, tv) : IRT_NUM;
467 int mode = IRSLOAD_INHERIT +
468 ((!LJ_DUALNUM || tvisint(tv) == (t == IRT_INT)) ? IRSLOAD_READONLY : 0);
469 TRef stop = fori_arg(J, fori, ra+FORL_STOP, t, mode);
470 TRef step = fori_arg(J, fori, ra+FORL_STEP, t, mode);
471 int tc, dir = rec_for_direction(&tv[FORL_STEP]);
472 lj_assertJ(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI,
473 "bad bytecode %d instead of FORI/JFORI", bc_op(*fori));
474 scev->t.irt = t;
475 scev->dir = dir;
476 scev->stop = tref_ref(stop);
477 scev->step = tref_ref(step);
478 rec_for_check(J, t, dir, stop, step, init);
479 scev->start = tref_ref(find_kinit(J, fori, ra+FORL_IDX, IRT_INT));
480 tc = (LJ_DUALNUM &&
481 !(scev->start && irref_isk(scev->stop) && irref_isk(scev->step) &&
482 tvisint(&tv[FORL_IDX]) == (t == IRT_INT))) ?
483 IRSLOAD_TYPECHECK : 0;
484 if (tc) {
485 J->base[ra+FORL_STOP] = stop;
486 J->base[ra+FORL_STEP] = step;
487 }
488 if (!idx)
489 idx = fori_load(J, ra+FORL_IDX, t,
490 IRSLOAD_INHERIT + tc + (J->scev.start << 16));
491 if (!init)
492 J->base[ra+FORL_IDX] = idx = emitir(IRT(IR_ADD, t), idx, step);
493 J->base[ra+FORL_EXT] = idx;
494 scev->idx = tref_ref(idx);
495 setmref(scev->pc, fori);
496 J->maxslot = ra+FORL_EXT+1;
497 }
498
499 /* Record FORL/JFORL or FORI/JFORI. */
500 static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl)
501 {
502 BCReg ra = bc_a(*fori);
503 TValue *tv = &J->L->base[ra];
504 TRef *tr = &J->base[ra];
505 IROp op;
506 LoopEvent ev;
507 TRef stop;
508 IRType t;
509 if (isforl) { /* Handle FORL/JFORL opcodes. */
510 TRef idx = tr[FORL_IDX];
511 if (mref(J->scev.pc, const BCIns) == fori && tref_ref(idx) == J->scev.idx) {
512 t = J->scev.t.irt;
513 stop = J->scev.stop;
514 idx = emitir(IRT(IR_ADD, t), idx, J->scev.step);
515 tr[FORL_EXT] = tr[FORL_IDX] = idx;
516 } else {
517 ScEvEntry scev;
518 rec_for_loop(J, fori, &scev, 0);
519 t = scev.t.irt;
520 stop = scev.stop;
521 }
522 } else { /* Handle FORI/JFORI opcodes. */
523 BCReg i;
524 lj_meta_for(J->L, tv);
525 t = (LJ_DUALNUM || tref_isint(tr[FORL_IDX])) ? lj_opt_narrow_forl(J, tv) :
526 IRT_NUM;
527 for (i = FORL_IDX; i <= FORL_STEP; i++) {
528 if (!tr[i]) sload(J, ra+i);
529 lj_assertJ(tref_isnumber_str(tr[i]), "bad FORI argument type");
530 if (tref_isstr(tr[i]))
531 tr[i] = emitir(IRTG(IR_STRTO, IRT_NUM), tr[i], 0);
532 if (t == IRT_INT) {
533 if (!tref_isinteger(tr[i]))
534 tr[i] = emitir(IRTGI(IR_CONV), tr[i], IRCONV_INT_NUM|IRCONV_CHECK);
535 } else {
536 if (!tref_isnum(tr[i]))
537 tr[i] = emitir(IRTN(IR_CONV), tr[i], IRCONV_NUM_INT);
538 }
539 }
540 tr[FORL_EXT] = tr[FORL_IDX];
541 stop = tr[FORL_STOP];
542 rec_for_check(J, t, rec_for_direction(&tv[FORL_STEP]),
543 stop, tr[FORL_STEP], 1);
544 }
545
546 ev = rec_for_iter(&op, tv, isforl);
547 if (ev == LOOPEV_LEAVE) {
548 J->maxslot = ra+FORL_EXT+1;
549 J->pc = fori+1;
550 } else {
551 J->maxslot = ra;
552 J->pc = fori+bc_j(*fori)+1;
553 }
554 lj_snap_add(J);
555
556 emitir(IRTG(op, t), tr[FORL_IDX], stop);
557
558 if (ev == LOOPEV_LEAVE) {
559 J->maxslot = ra;
560 J->pc = fori+bc_j(*fori)+1;
561 } else {
562 J->maxslot = ra+FORL_EXT+1;
563 J->pc = fori+1;
564 }
565 J->needsnap = 1;
566 return ev;
567 }
568
569 /* Record ITERL/JITERL. */
570 static LoopEvent rec_iterl(jit_State *J, const BCIns iterins)
571 {
572 BCReg ra = bc_a(iterins);
573 if (!tref_isnil(getslot(J, ra))) { /* Looping back? */
574 J->base[ra-1] = J->base[ra]; /* Copy result of ITERC to control var. */
575 J->maxslot = ra-1+bc_b(J->pc[-1]);
576 J->pc += bc_j(iterins)+1;
577 return LOOPEV_ENTER;
578 } else {
579 J->maxslot = ra-3;
580 J->pc++;
581 return LOOPEV_LEAVE;
582 }
583 }
584
585 /* Record LOOP/JLOOP. Now, that was easy. */
586 static LoopEvent rec_loop(jit_State *J, BCReg ra, int skip)
587 {
588 if (ra < J->maxslot) J->maxslot = ra;
589 J->pc += skip;
590 return LOOPEV_ENTER;
591 }
592
593 /* Check if a loop repeatedly failed to trace because it didn't loop back. */
594 static int innerloopleft(jit_State *J, const BCIns *pc)
595 {
596 ptrdiff_t i;
597 for (i = 0; i < PENALTY_SLOTS; i++)
598 if (mref(J->penalty[i].pc, const BCIns) == pc) {
599 if ((J->penalty[i].reason == LJ_TRERR_LLEAVE ||
600 J->penalty[i].reason == LJ_TRERR_LINNER) &&
601 J->penalty[i].val >= 2*PENALTY_MIN)
602 return 1;
603 break;
604 }
605 return 0;
606 }
607
608 /* Handle the case when an interpreted loop op is hit. */
609 static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev)
610 {
611 if (J->parent == 0 && J->exitno == 0) {
612 if (pc == J->startpc && J->framedepth + J->retdepth == 0) {
613 if (bc_op(J->cur.startins) == BC_ITERN) return; /* See rec_itern(). */
614 /* Same loop? */
615 if (ev == LOOPEV_LEAVE) /* Must loop back to form a root trace. */
616 lj_trace_err(J, LJ_TRERR_LLEAVE);
617 lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Looping trace. */
618 } else if (ev != LOOPEV_LEAVE) { /* Entering inner loop? */
619 /* It's usually better to abort here and wait until the inner loop
620 ** is traced. But if the inner loop repeatedly didn't loop back,
621 ** this indicates a low trip count. In this case try unrolling
622 ** an inner loop even in a root trace. But it's better to be a bit
623 ** more conservative here and only do it for very short loops.
624 */
625 if (bc_j(*pc) != -1 && !innerloopleft(J, pc))
626 lj_trace_err(J, LJ_TRERR_LINNER); /* Root trace hit an inner loop. */
627 if ((ev != LOOPEV_ENTERLO &&
628 J->loopref && J->cur.nins - J->loopref > 24) || --J->loopunroll < 0)
629 lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */
630 J->loopref = J->cur.nins;
631 }
632 } else if (ev != LOOPEV_LEAVE) { /* Side trace enters an inner loop. */
633 J->loopref = J->cur.nins;
634 if (--J->loopunroll < 0)
635 lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */
636 } /* Side trace continues across a loop that's left or not entered. */
637 }
638
639 /* Handle the case when an already compiled loop op is hit. */
640 static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev)
641 {
642 if (J->parent == 0 && J->exitno == 0) { /* Root trace hit an inner loop. */
643 /* Better let the inner loop spawn a side trace back here. */
644 lj_trace_err(J, LJ_TRERR_LINNER);
645 } else if (ev != LOOPEV_LEAVE) { /* Side trace enters a compiled loop. */
646 J->instunroll = 0; /* Cannot continue across a compiled loop op. */
647 if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
648 lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Form extra loop. */
649 else
650 lj_record_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the loop. */
651 } /* Side trace continues across a loop that's left or not entered. */
652 }
653
654 /* Record ITERN. */
655 static LoopEvent rec_itern(jit_State *J, BCReg ra, BCReg rb)
656 {
657 #if LJ_BE
658 /* YAGNI: Disabled on big-endian due to issues with lj_vm_next,
659 ** IR_HIOP, RID_RETLO/RID_RETHI and ra_destpair.
660 */
661 UNUSED(ra); UNUSED(rb);
662 setintV(&J->errinfo, (int32_t)BC_ITERN);
663 lj_trace_err_info(J, LJ_TRERR_NYIBC);
664 #else
665 RecordIndex ix;
666 /* Since ITERN is recorded at the start, we need our own loop detection. */
667 if (J->pc == J->startpc &&
668 J->framedepth + J->retdepth == 0 && J->parent == 0 && J->exitno == 0) {
669 IRRef ref = REF_FIRST + LJ_HASPROFILE;
670 #ifdef LUAJIT_ENABLE_CHECKHOOK
671 ref += 3;
672 #endif
673 if (J->cur.nins > ref ||
674 (LJ_HASPROFILE && J->cur.nins == ref && J->cur.ir[ref-1].o != IR_PROF)) {
675 J->instunroll = 0; /* Cannot continue unrolling across an ITERN. */
676 lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Looping trace. */
677 return LOOPEV_ENTER;
678 }
679 }
680 J->maxslot = ra;
681 lj_snap_add(J); /* Required to make JLOOP the first ins in a side-trace. */
682 ix.tab = getslot(J, ra-2);
683 ix.key = J->base[ra-1] ? J->base[ra-1] :
684 sloadt(J, (int32_t)(ra-1), IRT_GUARD|IRT_INT,
685 IRSLOAD_TYPECHECK|IRSLOAD_KEYINDEX);
686 copyTV(J->L, &ix.tabv, &J->L->base[ra-2]);
687 copyTV(J->L, &ix.keyv, &J->L->base[ra-1]);
688 ix.idxchain = (rb < 3); /* Omit value type check, if unused. */
689 ix.mobj = 1; /* We need the next index, too. */
690 J->maxslot = ra + lj_record_next(J, &ix);
691 J->needsnap = 1;
692 if (!tref_isnil(ix.key)) { /* Looping back? */
693 J->base[ra-1] = ix.mobj | TREF_KEYINDEX; /* Control var has next index. */
694 J->base[ra] = ix.key;
695 J->base[ra+1] = ix.val;
696 J->pc += bc_j(J->pc[1])+2;
697 return LOOPEV_ENTER;
698 } else {
699 J->maxslot = ra-3;
700 J->pc += 2;
701 return LOOPEV_LEAVE;
702 }
703 #endif
704 }
705
706 /* Record ISNEXT. */
707 static void rec_isnext(jit_State *J, BCReg ra)
708 {
709 cTValue *b = &J->L->base[ra-3];
710 if (tvisfunc(b) && funcV(b)->c.ffid == FF_next &&
711 tvistab(b+1) && tvisnil(b+2)) {
712 /* These checks are folded away for a compiled pairs(). */
713 TRef func = getslot(J, ra-3);
714 TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), func, IRFL_FUNC_FFID);
715 emitir(IRTGI(IR_EQ), trid, lj_ir_kint(J, FF_next));
716 (void)getslot(J, ra-2); /* Type check for table. */
717 (void)getslot(J, ra-1); /* Type check for nil key. */
718 J->base[ra-1] = lj_ir_kint(J, 0) | TREF_KEYINDEX;
719 J->maxslot = ra;
720 } else { /* Abort trace. Interpreter will despecialize bytecode. */
721 lj_trace_err(J, LJ_TRERR_RECERR);
722 }
723 }
724
725 /* -- Record profiler hook checks ----------------------------------------- */
726
727 #if LJ_HASPROFILE
728
729 /* Need to insert profiler hook check? */
730 static int rec_profile_need(jit_State *J, GCproto *pt, const BCIns *pc)
731 {
732 GCproto *ppt;
733 lj_assertJ(J->prof_mode == 'f' || J->prof_mode == 'l',
734 "bad profiler mode %c", J->prof_mode);
735 if (!pt)
736 return 0;
737 ppt = J->prev_pt;
738 J->prev_pt = pt;
739 if (pt != ppt && ppt) {
740 J->prev_line = -1;
741 return 1;
742 }
743 if (J->prof_mode == 'l') {
744 BCLine line = lj_debug_line(pt, proto_bcpos(pt, pc));
745 BCLine pline = J->prev_line;
746 J->prev_line = line;
747 if (pline != line)
748 return 1;
749 }
750 return 0;
751 }
752
753 static void rec_profile_ins(jit_State *J, const BCIns *pc)
754 {
755 if (J->prof_mode && rec_profile_need(J, J->pt, pc)) {
756 emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
757 lj_snap_add(J);
758 }
759 }
760
761 static void rec_profile_ret(jit_State *J)
762 {
763 if (J->prof_mode == 'f') {
764 emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
765 J->prev_pt = NULL;
766 lj_snap_add(J);
767 }
768 }
769
770 #endif
771
772 /* -- Record calls and returns -------------------------------------------- */
773
774 /* Specialize to the runtime value of the called function or its prototype. */
775 static TRef rec_call_specialize(jit_State *J, GCfunc *fn, TRef tr)
776 {
777 TRef kfunc;
778 if (isluafunc(fn)) {
779 GCproto *pt = funcproto(fn);
780 /* Too many closures created? Probably not a monomorphic function. */
781 if (pt->flags >= PROTO_CLC_POLY) { /* Specialize to prototype instead. */
782 TRef trpt = emitir(IRT(IR_FLOAD, IRT_PGC), tr, IRFL_FUNC_PC);
783 emitir(IRTG(IR_EQ, IRT_PGC), trpt, lj_ir_kptr(J, proto_bc(pt)));
784 (void)lj_ir_kgc(J, obj2gco(pt), IRT_PROTO); /* Prevent GC of proto. */
785 return tr;
786 }
787 } else {
788 /* Don't specialize to non-monomorphic builtins. */
789 switch (fn->c.ffid) {
790 case FF_coroutine_wrap_aux:
791 case FF_string_gmatch_aux:
792 /* NYI: io_file_iter doesn't have an ffid, yet. */
793 { /* Specialize to the ffid. */
794 TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), tr, IRFL_FUNC_FFID);
795 emitir(IRTGI(IR_EQ), trid, lj_ir_kint(J, fn->c.ffid));
796 }
797 return tr;
798 default:
799 /* NYI: don't specialize to non-monomorphic C functions. */
800 break;
801 }
802 }
803 /* Otherwise specialize to the function (closure) value itself. */
804 kfunc = lj_ir_kfunc(J, fn);
805 emitir(IRTG(IR_EQ, IRT_FUNC), tr, kfunc);
806 return kfunc;
807 }
808
809 /* Record call setup. */
810 static void rec_call_setup(jit_State *J, BCReg func, ptrdiff_t nargs)
811 {
812 RecordIndex ix;
813 TValue *functv = &J->L->base[func];
814 TRef kfunc, *fbase = &J->base[func];
815 ptrdiff_t i;
816 (void)getslot(J, func); /* Ensure func has a reference. */
817 for (i = 1; i <= nargs; i++)
818 (void)getslot(J, func+LJ_FR2+i); /* Ensure all args have a reference. */
819 if (!tref_isfunc(fbase[0])) { /* Resolve __call metamethod. */
820 ix.tab = fbase[0];
821 copyTV(J->L, &ix.tabv, functv);
822 if (!lj_record_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj))
823 lj_trace_err(J, LJ_TRERR_NOMM);
824 for (i = ++nargs; i > LJ_FR2; i--) /* Shift arguments up. */
825 fbase[i+LJ_FR2] = fbase[i+LJ_FR2-1];
826 #if LJ_FR2
827 fbase[2] = fbase[0];
828 #endif
829 fbase[0] = ix.mobj; /* Replace function. */
830 functv = &ix.mobjv;
831 }
832 kfunc = rec_call_specialize(J, funcV(functv), fbase[0]);
833 #if LJ_FR2
834 fbase[0] = kfunc;
835 fbase[1] = TREF_FRAME;
836 #else
837 fbase[0] = kfunc | TREF_FRAME;
838 #endif
839 J->maxslot = (BCReg)nargs;
840 }
841
842 /* Record call. */
843 void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs)
844 {
845 rec_call_setup(J, func, nargs);
846 /* Bump frame. */
847 J->framedepth++;
848 J->base += func+1+LJ_FR2;
849 J->baseslot += func+1+LJ_FR2;
850 if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS)
851 lj_trace_err(J, LJ_TRERR_STACKOV);
852 }
853
854 /* Record tail call. */
855 void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs)
856 {
857 rec_call_setup(J, func, nargs);
858 if (frame_isvarg(J->L->base - 1)) {
859 BCReg cbase = (BCReg)frame_delta(J->L->base - 1);
860 if (--J->framedepth < 0)
861 lj_trace_err(J, LJ_TRERR_NYIRETL);
862 J->baseslot -= (BCReg)cbase;
863 J->base -= cbase;
864 func += cbase;
865 }
866 /* Move func + args down. */
867 if (LJ_FR2 && J->baseslot == 2)
868 J->base[func+1] = TREF_FRAME;
869 memmove(&J->base[-1-LJ_FR2], &J->base[func], sizeof(TRef)*(J->maxslot+1+LJ_FR2));
870 /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */
871 /* Tailcalls can form a loop, so count towards the loop unroll limit. */
872 if (++J->tailcalled > J->loopunroll)
873 lj_trace_err(J, LJ_TRERR_LUNROLL);
874 }
875
876 /* Check unroll limits for down-recursion. */
877 static int check_downrec_unroll(jit_State *J, GCproto *pt)
878 {
879 IRRef ptref;
880 for (ptref = J->chain[IR_KGC]; ptref; ptref = IR(ptref)->prev)
881 if (ir_kgc(IR(ptref)) == obj2gco(pt)) {
882 int count = 0;
883 IRRef ref;
884 for (ref = J->chain[IR_RETF]; ref; ref = IR(ref)->prev)
885 if (IR(ref)->op1 == ptref)
886 count++;
887 if (count) {
888 if (J->pc == J->startpc) {
889 if (count + J->tailcalled > J->param[JIT_P_recunroll])
890 return 1;
891 } else {
892 lj_trace_err(J, LJ_TRERR_DOWNREC);
893 }
894 }
895 }
896 return 0;
897 }
898
899 static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot);
900
901 /* Record return. */
902 void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
903 {
904 TValue *frame = J->L->base - 1;
905 ptrdiff_t i;
906 BCReg baseadj = 0;
907 for (i = 0; i < gotresults; i++)
908 (void)getslot(J, rbase+i); /* Ensure all results have a reference. */
909 while (frame_ispcall(frame)) { /* Immediately resolve pcall() returns. */
910 BCReg cbase = (BCReg)frame_delta(frame);
911 if (--J->framedepth <= 0)
912 lj_trace_err(J, LJ_TRERR_NYIRETL);
913 lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return");
914 gotresults++;
915 baseadj += cbase;
916 rbase += cbase;
917 J->baseslot -= (BCReg)cbase;
918 J->base -= cbase;
919 J->base[--rbase] = TREF_TRUE; /* Prepend true to results. */
920 frame = frame_prevd(frame);
921 J->needsnap = 1; /* Stop catching on-trace errors. */
922 }
923 /* Return to lower frame via interpreter for unhandled cases. */
924 if (J->framedepth == 0 && J->pt && bc_isret(bc_op(*J->pc)) &&
925 (!frame_islua(frame) ||
926 (J->parent == 0 && J->exitno == 0 &&
927 !bc_isret(bc_op(J->cur.startins))))) {
928 /* NYI: specialize to frame type and return directly, not via RET*. */
929 for (i = 0; i < (ptrdiff_t)rbase; i++)
930 J->base[i] = 0; /* Purge dead slots. */
931 J->maxslot = rbase + (BCReg)gotresults;
932 lj_record_stop(J, LJ_TRLINK_RETURN, 0); /* Return to interpreter. */
933 return;
934 }
935 if (frame_isvarg(frame)) {
936 BCReg cbase = (BCReg)frame_delta(frame);
937 if (--J->framedepth < 0) /* NYI: return of vararg func to lower frame. */
938 lj_trace_err(J, LJ_TRERR_NYIRETL);
939 lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return");
940 baseadj += cbase;
941 rbase += cbase;
942 J->baseslot -= (BCReg)cbase;
943 J->base -= cbase;
944 frame = frame_prevd(frame);
945 }
946 if (frame_islua(frame)) { /* Return to Lua frame. */
947 BCIns callins = *(frame_pc(frame)-1);
948 ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults;
949 BCReg cbase = bc_a(callins);
950 GCproto *pt = funcproto(frame_func(frame - (cbase+1+LJ_FR2)));
951 if ((pt->flags & PROTO_NOJIT))
952 lj_trace_err(J, LJ_TRERR_CJITOFF);
953 if (J->framedepth == 0 && J->pt && frame == J->L->base - 1) {
954 if (!J->cur.root && check_downrec_unroll(J, pt)) {
955 J->maxslot = (BCReg)(rbase + gotresults);
956 lj_snap_purge(J);
957 lj_record_stop(J, LJ_TRLINK_DOWNREC, J->cur.traceno); /* Down-rec. */
958 return;
959 }
960 lj_snap_add(J);
961 }
962 for (i = 0; i < nresults; i++) /* Adjust results. */
963 J->base[i-1-LJ_FR2] = i < gotresults ? J->base[rbase+i] : TREF_NIL;
964 J->maxslot = cbase+(BCReg)nresults;
965 if (J->framedepth > 0) { /* Return to a frame that is part of the trace. */
966 J->framedepth--;
967 lj_assertJ(J->baseslot > cbase+1+LJ_FR2, "bad baseslot for return");
968 J->baseslot -= cbase+1+LJ_FR2;
969 J->base -= cbase+1+LJ_FR2;
970 } else if (J->parent == 0 && J->exitno == 0 &&
971 !bc_isret(bc_op(J->cur.startins))) {
972 /* Return to lower frame would leave the loop in a root trace. */
973 lj_trace_err(J, LJ_TRERR_LLEAVE);
974 } else if (J->needsnap) { /* Tailcalled to ff with side-effects. */
975 lj_trace_err(J, LJ_TRERR_NYIRETL); /* No way to insert snapshot here. */
976 } else if (1 + pt->framesize >= LJ_MAX_JSLOTS ||
977 J->baseslot + J->maxslot >= LJ_MAX_JSLOTS) {
978 lj_trace_err(J, LJ_TRERR_STACKOV);
979 } else { /* Return to lower frame. Guard for the target we return to. */
980 TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);
981 TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame));
982 emitir(IRTG(IR_RETF, IRT_PGC), trpt, trpc);
983 J->retdepth++;
984 J->needsnap = 1;
985 J->scev.idx = REF_NIL;
986 lj_assertJ(J->baseslot == 1+LJ_FR2, "bad baseslot for return");
987 /* Shift result slots up and clear the slots of the new frame below. */
988 memmove(J->base + cbase, J->base-1-LJ_FR2, sizeof(TRef)*nresults);
989 memset(J->base-1-LJ_FR2, 0, sizeof(TRef)*(cbase+1+LJ_FR2));
990 }
991 } else if (frame_iscont(frame)) { /* Return to continuation frame. */
992 ASMFunction cont = frame_contf(frame);
993 BCReg cbase = (BCReg)frame_delta(frame);
994 if ((J->framedepth -= 2) < 0)
995 lj_trace_err(J, LJ_TRERR_NYIRETL);
996 J->baseslot -= (BCReg)cbase;
997 J->base -= cbase;
998 J->maxslot = cbase-(2<<LJ_FR2);
999 if (cont == lj_cont_ra) {
1000 /* Copy result to destination slot. */
1001 BCReg dst = bc_a(*(frame_contpc(frame)-1));
1002 J->base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL;
1003 if (dst >= J->maxslot) {
1004 J->maxslot = dst+1;
1005 }
1006 } else if (cont == lj_cont_nop) {
1007 /* Nothing to do here. */
1008 } else if (cont == lj_cont_cat) {
1009 BCReg bslot = bc_b(*(frame_contpc(frame)-1));
1010 TRef tr = gotresults ? J->base[cbase+rbase] : TREF_NIL;
1011 if (bslot != J->maxslot) { /* Concatenate the remainder. */
1012 /* Simulate lower frame and result. */
1013 TValue *b = J->L->base - baseadj, save;
1014 /* Can't handle MM_concat + CALLT + fast func side-effects. */
1015 if (J->postproc != LJ_POST_NONE)
1016 lj_trace_err(J, LJ_TRERR_NYIRETL);
1017 J->base[J->maxslot] = tr;
1018 copyTV(J->L, &save, b-(2<<LJ_FR2));
1019 if (gotresults)
1020 copyTV(J->L, b-(2<<LJ_FR2), b+rbase);
1021 else
1022 setnilV(b-(2<<LJ_FR2));
1023 J->L->base = b - cbase;
1024 tr = rec_cat(J, bslot, cbase-(2<<LJ_FR2));
1025 b = J->L->base + cbase; /* Undo. */
1026 J->L->base = b + baseadj;
1027 copyTV(J->L, b-(2<<LJ_FR2), &save);
1028 }
1029 if (tr >= 0xffffff00) {
1030 lj_err_throw(J->L, -(int32_t)tr); /* Propagate errors. */
1031 } else if (tr) { /* Store final result. */
1032 BCReg dst = bc_a(*(frame_contpc(frame)-1));
1033 J->base[dst] = tr;
1034 if (dst >= J->maxslot) {
1035 J->maxslot = dst+1;
1036 }
1037 } /* Otherwise continue with another __concat call. */
1038 } else {
1039 /* Result type already specialized. */
1040 lj_assertJ(cont == lj_cont_condf || cont == lj_cont_condt,
1041 "bad continuation type");
1042 }
1043 } else {
1044 lj_trace_err(J, LJ_TRERR_NYIRETL); /* NYI: handle return to C frame. */
1045 }
1046 lj_assertJ(J->baseslot >= 1+LJ_FR2, "bad baseslot for return");
1047 }
1048
1049 /* -- Metamethod handling ------------------------------------------------- */
1050
1051 /* Prepare to record call to metamethod. */
1052 static BCReg rec_mm_prep(jit_State *J, ASMFunction cont)
1053 {
1054 BCReg s, top = cont == lj_cont_cat ? J->maxslot : curr_proto(J->L)->framesize;
1055 #if LJ_FR2
1056 J->base[top] = lj_ir_k64(J, IR_KNUM, u64ptr(contptr(cont)));
1057 J->base[top+1] = TREF_CONT;
1058 #else
1059 J->base[top] = lj_ir_kptr(J, contptr(cont)) | TREF_CONT;
1060 #endif
1061 J->framedepth++;
1062 for (s = J->maxslot; s < top; s++)
1063 J->base[s] = 0; /* Clear frame gap to avoid resurrecting previous refs. */
1064 return top+1+LJ_FR2;
1065 }
1066
1067 /* Record metamethod lookup. */
1068 int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm)
1069 {
1070 RecordIndex mix;
1071 GCtab *mt;
1072 if (tref_istab(ix->tab)) {
1073 mt = tabref(tabV(&ix->tabv)->metatable);
1074 mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
1075 } else if (tref_isudata(ix->tab)) {
1076 int udtype = udataV(&ix->tabv)->udtype;
1077 mt = tabref(udataV(&ix->tabv)->metatable);
1078 mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_UDATA_META);
1079 /* The metatables of special userdata objects are treated as immutable. */
1080 if (udtype != UDTYPE_USERDATA) {
1081 cTValue *mo;
1082 if (LJ_HASFFI && udtype == UDTYPE_FFI_CLIB) {
1083 /* Specialize to the C library namespace object. */
1084 emitir(IRTG(IR_EQ, IRT_PGC), ix->tab, lj_ir_kptr(J, udataV(&ix->tabv)));
1085 } else {
1086 /* Specialize to the type of userdata. */
1087 TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), ix->tab, IRFL_UDATA_UDTYPE);
1088 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, udtype));
1089 }
1090 immutable_mt:
1091 mo = lj_tab_getstr(mt, mmname_str(J2G(J), mm));
1092 ix->mt = mix.tab;
1093 ix->mtv = mt;
1094 if (!mo || tvisnil(mo))
1095 return 0; /* No metamethod. */
1096 /* Treat metamethod or index table as immutable, too. */
1097 if (!(tvisfunc(mo) || tvistab(mo)))
1098 lj_trace_err(J, LJ_TRERR_BADTYPE);
1099 copyTV(J->L, &ix->mobjv, mo);
1100 ix->mobj = lj_ir_kgc(J, gcV(mo), tvisfunc(mo) ? IRT_FUNC : IRT_TAB);
1101 return 1; /* Got metamethod or index table. */
1102 }
1103 } else {
1104 /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */
1105 mt = tabref(basemt_obj(J2G(J), &ix->tabv));
1106 if (mt == NULL) {
1107 ix->mt = TREF_NIL;
1108 return 0; /* No metamethod. */
1109 }
1110 /* The cdata metatable is treated as immutable. */
1111 if (LJ_HASFFI && tref_iscdata(ix->tab)) {
1112 mix.tab = TREF_NIL;
1113 goto immutable_mt;
1114 }
1115 ix->mt = mix.tab = lj_ir_ggfload(J, IRT_TAB,
1116 GG_OFS(g.gcroot[GCROOT_BASEMT+itypemap(&ix->tabv)]));
1117 goto nocheck;
1118 }
1119 ix->mt = mt ? mix.tab : TREF_NIL;
1120 emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mix.tab, lj_ir_knull(J, IRT_TAB));
1121 nocheck:
1122 if (mt) {
1123 GCstr *mmstr = mmname_str(J2G(J), mm);
1124 cTValue *mo = lj_tab_getstr(mt, mmstr);
1125 if (mo && !tvisnil(mo))
1126 copyTV(J->L, &ix->mobjv, mo);
1127 ix->mtv = mt;
1128 settabV(J->L, &mix.tabv, mt);
1129 setstrV(J->L, &mix.keyv, mmstr);
1130 mix.key = lj_ir_kstr(J, mmstr);
1131 mix.val = 0;
1132 mix.idxchain = 0;
1133 ix->mobj = lj_record_idx(J, &mix);
1134 return !tref_isnil(ix->mobj); /* 1 if metamethod found, 0 if not. */
1135 }
1136 return 0; /* No metamethod. */
1137 }
1138
1139 /* Record call to arithmetic metamethod. */
1140 static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm)
1141 {
1142 /* Set up metamethod call first to save ix->tab and ix->tabv. */
1143 BCReg func = rec_mm_prep(J, mm == MM_concat ? lj_cont_cat : lj_cont_ra);
1144 TRef *base = J->base + func;
1145 TValue *basev = J->L->base + func;
1146 base[1+LJ_FR2] = ix->tab; base[2+LJ_FR2] = ix->key;
1147 copyTV(J->L, basev+1+LJ_FR2, &ix->tabv);
1148 copyTV(J->L, basev+2+LJ_FR2, &ix->keyv);
1149 if (!lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */
1150 if (mm != MM_unm) {
1151 ix->tab = ix->key;
1152 copyTV(J->L, &ix->tabv, &ix->keyv);
1153 if (lj_record_mm_lookup(J, ix, mm)) /* Lookup mm on 2nd operand. */
1154 goto ok;
1155 }
1156 lj_trace_err(J, LJ_TRERR_NOMM);
1157 }
1158 ok:
1159 base[0] = ix->mobj;
1160 #if LJ_FR2
1161 base[1] = 0;
1162 #endif
1163 copyTV(J->L, basev+0, &ix->mobjv);
1164 lj_record_call(J, func, 2);
1165 return 0; /* No result yet. */
1166 }
1167
1168 /* Record call to __len metamethod. */
1169 static TRef rec_mm_len(jit_State *J, TRef tr, TValue *tv)
1170 {
1171 RecordIndex ix;
1172 ix.tab = tr;
1173 copyTV(J->L, &ix.tabv, tv);
1174 if (lj_record_mm_lookup(J, &ix, MM_len)) {
1175 BCReg func = rec_mm_prep(J, lj_cont_ra);
1176 TRef *base = J->base + func;
1177 TValue *basev = J->L->base + func;
1178 base[0] = ix.mobj; copyTV(J->L, basev+0, &ix.mobjv);
1179 base += LJ_FR2;
1180 basev += LJ_FR2;
1181 base[1] = tr; copyTV(J->L, basev+1, tv);
1182 #if LJ_52
1183 base[2] = tr; copyTV(J->L, basev+2, tv);
1184 #else
1185 base[2] = TREF_NIL; setnilV(basev+2);
1186 #endif
1187 lj_record_call(J, func, 2);
1188 } else {
1189 if (LJ_52 && tref_istab(tr))
1190 return emitir(IRTI(IR_ALEN), tr, TREF_NIL);
1191 lj_trace_err(J, LJ_TRERR_NOMM);
1192 }
1193 return 0; /* No result yet. */
1194 }
1195
1196 /* Call a comparison metamethod. */
1197 static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op)
1198 {
1199 BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt);
1200 TRef *base = J->base + func + LJ_FR2;
1201 TValue *tv = J->L->base + func + LJ_FR2;
1202 base[-LJ_FR2] = ix->mobj; base[1] = ix->val; base[2] = ix->key;
1203 copyTV(J->L, tv-LJ_FR2, &ix->mobjv);
1204 copyTV(J->L, tv+1, &ix->valv);
1205 copyTV(J->L, tv+2, &ix->keyv);
1206 lj_record_call(J, func, 2);
1207 }
1208
1209 /* Record call to equality comparison metamethod (for tab and udata only). */
1210 static void rec_mm_equal(jit_State *J, RecordIndex *ix, int op)
1211 {
1212 ix->tab = ix->val;
1213 copyTV(J->L, &ix->tabv, &ix->valv);
1214 if (lj_record_mm_lookup(J, ix, MM_eq)) { /* Lookup mm on 1st operand. */
1215 cTValue *bv;
1216 TRef mo1 = ix->mobj;
1217 TValue mo1v;
1218 copyTV(J->L, &mo1v, &ix->mobjv);
1219 /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
1220 bv = &ix->keyv;
1221 if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
1222 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
1223 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1224 } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
1225 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
1226 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1227 } else { /* Lookup metamethod on 2nd operand and compare both. */
1228 ix->tab = ix->key;
1229 copyTV(J->L, &ix->tabv, bv);
1230 if (!lj_record_mm_lookup(J, ix, MM_eq) ||
1231 lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
1232 return;
1233 }
1234 rec_mm_callcomp(J, ix, op);
1235 }
1236 }
1237
1238 /* Record call to ordered comparison metamethods (for arbitrary objects). */
1239 static void rec_mm_comp(jit_State *J, RecordIndex *ix, int op)
1240 {
1241 ix->tab = ix->val;
1242 copyTV(J->L, &ix->tabv, &ix->valv);
1243 while (1) {
1244 MMS mm = (op & 2) ? MM_le : MM_lt; /* Try __le + __lt or only __lt. */
1245 #if LJ_52
1246 if (!lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */
1247 ix->tab = ix->key;
1248 copyTV(J->L, &ix->tabv, &ix->keyv);
1249 if (!lj_record_mm_lookup(J, ix, mm)) /* Lookup mm on 2nd operand. */
1250 goto nomatch;
1251 }
1252 rec_mm_callcomp(J, ix, op);
1253 return;
1254 #else
1255 if (lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */
1256 cTValue *bv;
1257 TRef mo1 = ix->mobj;
1258 TValue mo1v;
1259 copyTV(J->L, &mo1v, &ix->mobjv);
1260 /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
1261 bv = &ix->keyv;
1262 if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
1263 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
1264 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1265 } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
1266 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
1267 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1268 } else { /* Lookup metamethod on 2nd operand and compare both. */
1269 ix->tab = ix->key;
1270 copyTV(J->L, &ix->tabv, bv);
1271 if (!lj_record_mm_lookup(J, ix, mm) ||
1272 lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
1273 goto nomatch;
1274 }
1275 rec_mm_callcomp(J, ix, op);
1276 return;
1277 }
1278 #endif
1279 nomatch:
1280 /* Lookup failed. Retry with __lt and swapped operands. */
1281 if (!(op & 2)) break; /* Already at __lt. Interpreter will throw. */
1282 ix->tab = ix->key; ix->key = ix->val; ix->val = ix->tab;
1283 copyTV(J->L, &ix->tabv, &ix->keyv);
1284 copyTV(J->L, &ix->keyv, &ix->valv);
1285 copyTV(J->L, &ix->valv, &ix->tabv);
1286 op ^= 3;
1287 }
1288 }
1289
1290 #if LJ_HASFFI
1291 /* Setup call to cdata comparison metamethod. */
1292 static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm)
1293 {
1294 lj_snap_add(J);
1295 if (tref_iscdata(ix->val)) {
1296 ix->tab = ix->val;
1297 copyTV(J->L, &ix->tabv, &ix->valv);
1298 } else {
1299 lj_assertJ(tref_iscdata(ix->key), "cdata expected");
1300 ix->tab = ix->key;
1301 copyTV(J->L, &ix->tabv, &ix->keyv);
1302 }
1303 lj_record_mm_lookup(J, ix, mm);
1304 rec_mm_callcomp(J, ix, op);
1305 }
1306 #endif
1307
1308 /* -- Indexed access ------------------------------------------------------ */
1309
1310 #ifdef LUAJIT_ENABLE_TABLE_BUMP
1311 /* Bump table allocations in bytecode when they grow during recording. */
1312 static void rec_idx_bump(jit_State *J, RecordIndex *ix)
1313 {
1314 RBCHashEntry *rbc = &J->rbchash[(ix->tab & (RBCHASH_SLOTS-1))];
1315 if (tref_ref(ix->tab) == rbc->ref) {
1316 const BCIns *pc = mref(rbc->pc, const BCIns);
1317 GCtab *tb = tabV(&ix->tabv);
1318 uint32_t nhbits;
1319 IRIns *ir;
1320 if (!tvisnil(&ix->keyv))
1321 (void)lj_tab_set(J->L, tb, &ix->keyv); /* Grow table right now. */
1322 nhbits = tb->hmask > 0 ? lj_fls(tb->hmask)+1 : 0;
1323 ir = IR(tref_ref(ix->tab));
1324 if (ir->o == IR_TNEW) {
1325 uint32_t ah = bc_d(*pc);
1326 uint32_t asize = ah & 0x7ff, hbits = ah >> 11;
1327 if (nhbits > hbits) hbits = nhbits;
1328 if (tb->asize > asize) {
1329 asize = tb->asize <= 0x7ff ? tb->asize : 0x7ff;
1330 }
1331 if ((asize | (hbits<<11)) != ah) { /* Has the size changed? */
1332 /* Patch bytecode, but continue recording (for more patching). */
1333 setbc_d(pc, (asize | (hbits<<11)));
1334 /* Patching TNEW operands is only safe if the trace is aborted. */
1335 ir->op1 = asize; ir->op2 = hbits;
1336 J->retryrec = 1; /* Abort the trace at the end of recording. */
1337 }
1338 } else if (ir->o == IR_TDUP) {
1339 GCtab *tpl = gco2tab(proto_kgc(&gcref(rbc->pt)->pt, ~(ptrdiff_t)bc_d(*pc)));
1340 /* Grow template table, but preserve keys with nil values. */
1341 if ((tb->asize > tpl->asize && (1u << nhbits)-1 == tpl->hmask) ||
1342 (tb->asize == tpl->asize && (1u << nhbits)-1 > tpl->hmask)) {
1343 Node *node = noderef(tpl->node);
1344 uint32_t i, hmask = tpl->hmask, asize;
1345 TValue *array;
1346 for (i = 0; i <= hmask; i++) {
1347 if (!tvisnil(&node[i].key) && tvisnil(&node[i].val))
1348 settabV(J->L, &node[i].val, tpl);
1349 }
1350 if (!tvisnil(&ix->keyv) && tref_isk(ix->key)) {
1351 TValue *o = lj_tab_set(J->L, tpl, &ix->keyv);
1352 if (tvisnil(o)) settabV(J->L, o, tpl);
1353 }
1354 lj_tab_resize(J->L, tpl, tb->asize, nhbits);
1355 node = noderef(tpl->node);
1356 hmask = tpl->hmask;
1357 for (i = 0; i <= hmask; i++) {
1358 /* This is safe, since template tables only hold immutable values. */
1359 if (tvistab(&node[i].val))
1360 setnilV(&node[i].val);
1361 }
1362 /* The shape of the table may have changed. Clean up array part, too. */
1363 asize = tpl->asize;
1364 array = tvref(tpl->array);
1365 for (i = 0; i < asize; i++) {
1366 if (tvistab(&array[i]))
1367 setnilV(&array[i]);
1368 }
1369 J->retryrec = 1; /* Abort the trace at the end of recording. */
1370 }
1371 }
1372 }
1373 }
1374 #endif
1375
1376 /* Record bounds-check. */
1377 static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize)
1378 {
1379 /* Try to emit invariant bounds checks. */
1380 if ((J->flags & (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) ==
1381 (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) {
1382 IRRef ref = tref_ref(ikey);
1383 IRIns *ir = IR(ref);
1384 int32_t ofs = 0;
1385 IRRef ofsref = 0;
1386 /* Handle constant offsets. */
1387 if (ir->o == IR_ADD && irref_isk(ir->op2)) {
1388 ofsref = ir->op2;
1389 ofs = IR(ofsref)->i;
1390 ref = ir->op1;
1391 ir = IR(ref);
1392 }
1393 /* Got scalar evolution analysis results for this reference? */
1394 if (ref == J->scev.idx) {
1395 int32_t stop;
1396 lj_assertJ(irt_isint(J->scev.t) && ir->o == IR_SLOAD,
1397 "only int SCEV supported");
1398 stop = numberVint(&(J->L->base - J->baseslot)[ir->op1 + FORL_STOP]);
1399 /* Runtime value for stop of loop is within bounds? */
1400 if ((uint64_t)stop + ofs < (uint64_t)asize) {
1401 /* Emit invariant bounds check for stop. */
1402 uint32_t abc = IRTG(IR_ABC, tref_isk(asizeref) ? IRT_U32 : IRT_P32);
1403 emitir(abc, asizeref, ofs == 0 ? J->scev.stop :
1404 emitir(IRTI(IR_ADD), J->scev.stop, ofsref));
1405 /* Emit invariant bounds check for start, if not const or negative. */
1406 if (!(J->scev.dir && J->scev.start &&
1407 (int64_t)IR(J->scev.start)->i + ofs >= 0))
1408 emitir(abc, asizeref, ikey);
1409 return;
1410 }
1411 }
1412 }
1413 emitir(IRTGI(IR_ABC), asizeref, ikey); /* Emit regular bounds check. */
1414 }
1415
1416 /* Record indexed key lookup. */
1417 static TRef rec_idx_key(jit_State *J, RecordIndex *ix, IRRef *rbref,
1418 IRType1 *rbguard)
1419 {
1420 TRef key;
1421 GCtab *t = tabV(&ix->tabv);
1422 ix->oldv = lj_tab_get(J->L, t, &ix->keyv); /* Lookup previous value. */
1423 *rbref = 0;
1424 rbguard->irt = 0;
1425
1426 /* Integer keys are looked up in the array part first. */
1427 key = ix->key;
1428 if (tref_isnumber(key)) {
1429 int32_t k = numberVint(&ix->keyv);
1430 if (!tvisint(&ix->keyv) && numV(&ix->keyv) != (lua_Number)k)
1431 k = LJ_MAX_ASIZE;
1432 if ((MSize)k < LJ_MAX_ASIZE) { /* Potential array key? */
1433 TRef ikey = lj_opt_narrow_index(J, key);
1434 TRef asizeref = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
1435 if ((MSize)k < t->asize) { /* Currently an array key? */
1436 TRef arrayref;
1437 rec_idx_abc(J, asizeref, ikey, t->asize);
1438 arrayref = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_ARRAY);
1439 return emitir(IRT(IR_AREF, IRT_PGC), arrayref, ikey);
1440 } else { /* Currently not in array (may be an array extension)? */
1441 emitir(IRTGI(IR_ULE), asizeref, ikey); /* Inv. bounds check. */
1442 if (k == 0 && tref_isk(key))
1443 key = lj_ir_knum_zero(J); /* Canonicalize 0 or +-0.0 to +0.0. */
1444 /* And continue with the hash lookup. */
1445 }
1446 } else if (!tref_isk(key)) {
1447 /* We can rule out const numbers which failed the integerness test
1448 ** above. But all other numbers are potential array keys.
1449 */
1450 if (t->asize == 0) { /* True sparse tables have an empty array part. */
1451 /* Guard that the array part stays empty. */
1452 TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
1453 emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
1454 } else {
1455 lj_trace_err(J, LJ_TRERR_NYITMIX);
1456 }
1457 }
1458 }
1459
1460 /* Otherwise the key is located in the hash part. */
1461 if (t->hmask == 0) { /* Shortcut for empty hash part. */
1462 /* Guard that the hash part stays empty. */
1463 TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
1464 emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
1465 return lj_ir_kkptr(J, niltvg(J2G(J)));
1466 }
1467 if (tref_isinteger(key)) /* Hash keys are based on numbers, not ints. */
1468 key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
1469 if (tref_isk(key)) {
1470 /* Optimize lookup of constant hash keys. */
1471 GCSize hslot = (GCSize)((char *)ix->oldv-(char *)&noderef(t->node)[0].val);
1472 if (hslot <= t->hmask*(GCSize)sizeof(Node) &&
1473 hslot <= 65535*(GCSize)sizeof(Node)) {
1474 TRef node, kslot, hm;
1475 *rbref = J->cur.nins; /* Mark possible rollback point. */
1476 *rbguard = J->guardemit;
1477 hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
1478 emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask));
1479 node = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_NODE);
1480 kslot = lj_ir_kslot(J, key, (IRRef)(hslot / sizeof(Node)));
1481 return emitir(IRTG(IR_HREFK, IRT_PGC), node, kslot);
1482 }
1483 }
1484 /* Fall back to a regular hash lookup. */
1485 return emitir(IRT(IR_HREF, IRT_PGC), ix->tab, key);
1486 }
1487
1488 /* Determine whether a key is NOT one of the fast metamethod names. */
1489 static int nommstr(jit_State *J, TRef key)
1490 {
1491 if (tref_isstr(key)) {
1492 if (tref_isk(key)) {
1493 GCstr *str = ir_kstr(IR(tref_ref(key)));
1494 uint32_t mm;
1495 for (mm = 0; mm <= MM_FAST; mm++)
1496 if (mmname_str(J2G(J), mm) == str)
1497 return 0; /* MUST be one the fast metamethod names. */
1498 } else {
1499 return 0; /* Variable string key MAY be a metamethod name. */
1500 }
1501 }
1502 return 1; /* CANNOT be a metamethod name. */
1503 }
1504
1505 /* Record indexed load/store. */
1506 TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1507 {
1508 TRef xref;
1509 IROp xrefop, loadop;
1510 IRRef rbref;
1511 IRType1 rbguard;
1512 cTValue *oldv;
1513
1514 while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */
1515 /* Never call raw lj_record_idx() on non-table. */
1516 lj_assertJ(ix->idxchain != 0, "bad usage");
1517 if (!lj_record_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index))
1518 lj_trace_err(J, LJ_TRERR_NOMM);
1519 handlemm:
1520 if (tref_isfunc(ix->mobj)) { /* Handle metamethod call. */
1521 BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra);
1522 TRef *base = J->base + func + LJ_FR2;
1523 TValue *tv = J->L->base + func + LJ_FR2;
1524 base[-LJ_FR2] = ix->mobj; base[1] = ix->tab; base[2] = ix->key;
1525 setfuncV(J->L, tv-LJ_FR2, funcV(&ix->mobjv));
1526 copyTV(J->L, tv+1, &ix->tabv);
1527 copyTV(J->L, tv+2, &ix->keyv);
1528 if (ix->val) {
1529 base[3] = ix->val;
1530 copyTV(J->L, tv+3, &ix->valv);
1531 lj_record_call(J, func, 3); /* mobj(tab, key, val) */
1532 return 0;
1533 } else {
1534 lj_record_call(J, func, 2); /* res = mobj(tab, key) */
1535 return 0; /* No result yet. */
1536 }
1537 }
1538 #if LJ_HASBUFFER
1539 /* The index table of buffer objects is treated as immutable. */
1540 if (ix->mt == TREF_NIL && !ix->val &&
1541 tref_isudata(ix->tab) && udataV(&ix->tabv)->udtype == UDTYPE_BUFFER &&
1542 tref_istab(ix->mobj) && tref_isstr(ix->key) && tref_isk(ix->key)) {
1543 cTValue *val = lj_tab_getstr(tabV(&ix->mobjv), strV(&ix->keyv));
1544 TRef tr = lj_record_constify(J, val);
1545 if (tr) return tr; /* Specialize to the value, i.e. a method. */
1546 }
1547 #endif
1548 /* Otherwise retry lookup with metaobject. */
1549 ix->tab = ix->mobj;
1550 copyTV(J->L, &ix->tabv, &ix->mobjv);
1551 if (--ix->idxchain == 0)
1552 lj_trace_err(J, LJ_TRERR_IDXLOOP);
1553 }
1554
1555 /* First catch nil and NaN keys for tables. */
1556 if (tvisnil(&ix->keyv) || (tvisnum(&ix->keyv) && tvisnan(&ix->keyv))) {
1557 if (ix->val) /* Better fail early. */
1558 lj_trace_err(J, LJ_TRERR_STORENN);
1559 if (tref_isk(ix->key)) {
1560 if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
1561 goto handlemm;
1562 return TREF_NIL;
1563 }
1564 }
1565
1566 /* Record the key lookup. */
1567 xref = rec_idx_key(J, ix, &rbref, &rbguard);
1568 xrefop = IR(tref_ref(xref))->o;
1569 loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD;
1570 /* The lj_meta_tset() inconsistency is gone, but better play safe. */
1571 oldv = xrefop == IR_KKPTR ? (cTValue *)ir_kptr(IR(tref_ref(xref))) : ix->oldv;
1572
1573 if (ix->val == 0) { /* Indexed load */
1574 IRType t = itype2irt(oldv);
1575 TRef res;
1576 if (oldv == niltvg(J2G(J))) {
1577 emitir(IRTG(IR_EQ, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1578 res = TREF_NIL;
1579 } else {
1580 res = emitir(IRTG(loadop, t), xref, 0);
1581 }
1582 if (tref_ref(res) < rbref) { /* HREFK + load forwarded? */
1583 lj_ir_rollback(J, rbref); /* Rollback to eliminate hmask guard. */
1584 J->guardemit = rbguard;
1585 }
1586 if (t == IRT_NIL && ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
1587 goto handlemm;
1588 if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitives. */
1589 return res;
1590 } else { /* Indexed store. */
1591 GCtab *mt = tabref(tabV(&ix->tabv)->metatable);
1592 int keybarrier = tref_isgcv(ix->key) && !tref_isnil(ix->val);
1593 if (tref_ref(xref) < rbref) { /* HREFK forwarded? */
1594 lj_ir_rollback(J, rbref); /* Rollback to eliminate hmask guard. */
1595 J->guardemit = rbguard;
1596 }
1597 if (tvisnil(oldv)) { /* Previous value was nil? */
1598 /* Need to duplicate the hasmm check for the early guards. */
1599 int hasmm = 0;
1600 if (ix->idxchain && mt) {
1601 cTValue *mo = lj_tab_getstr(mt, mmname_str(J2G(J), MM_newindex));
1602 hasmm = mo && !tvisnil(mo);
1603 }
1604 if (hasmm)
1605 emitir(IRTG(loadop, IRT_NIL), xref, 0); /* Guard for nil value. */
1606 else if (xrefop == IR_HREF)
1607 emitir(IRTG(oldv == niltvg(J2G(J)) ? IR_EQ : IR_NE, IRT_PGC),
1608 xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1609 if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_newindex)) {
1610 lj_assertJ(hasmm, "inconsistent metamethod handling");
1611 goto handlemm;
1612 }
1613 lj_assertJ(!hasmm, "inconsistent metamethod handling");
1614 if (oldv == niltvg(J2G(J))) { /* Need to insert a new key. */
1615 TRef key = ix->key;
1616 if (tref_isinteger(key)) { /* NEWREF needs a TValue as a key. */
1617 key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
1618 } else if (tref_isnum(key)) {
1619 if (tref_isk(key)) {
1620 if (tvismzero(&ix->keyv))
1621 key = lj_ir_knum_zero(J); /* Canonicalize -0.0 to +0.0. */
1622 } else {
1623 emitir(IRTG(IR_EQ, IRT_NUM), key, key); /* Check for !NaN. */
1624 }
1625 }
1626 xref = emitir(IRT(IR_NEWREF, IRT_PGC), ix->tab, key);
1627 keybarrier = 0; /* NEWREF already takes care of the key barrier. */
1628 #ifdef LUAJIT_ENABLE_TABLE_BUMP
1629 if ((J->flags & JIT_F_OPT_SINK)) /* Avoid a separate flag. */
1630 rec_idx_bump(J, ix);
1631 #endif
1632 }
1633 } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) {
1634 /* Cannot derive that the previous value was non-nil, must do checks. */
1635 if (xrefop == IR_HREF) /* Guard against store to niltv. */
1636 emitir(IRTG(IR_NE, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1637 if (ix->idxchain) { /* Metamethod lookup required? */
1638 /* A check for NULL metatable is cheaper (hoistable) than a load. */
1639 if (!mt) {
1640 TRef mtref = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
1641 emitir(IRTG(IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB));
1642 } else {
1643 IRType t = itype2irt(oldv);
1644 emitir(IRTG(loadop, t), xref, 0); /* Guard for non-nil value. */
1645 }
1646 }
1647 } else {
1648 keybarrier = 0; /* Previous non-nil value kept the key alive. */
1649 }
1650 /* Convert int to number before storing. */
1651 if (!LJ_DUALNUM && tref_isinteger(ix->val))
1652 ix->val = emitir(IRTN(IR_CONV), ix->val, IRCONV_NUM_INT);
1653 emitir(IRT(loadop+IRDELTA_L2S, tref_type(ix->val)), xref, ix->val);
1654 if (keybarrier || tref_isgcv(ix->val))
1655 emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0);
1656 /* Invalidate neg. metamethod cache for stores with certain string keys. */
1657 if (!nommstr(J, ix->key)) {
1658 TRef fref = emitir(IRT(IR_FREF, IRT_PGC), ix->tab, IRFL_TAB_NOMM);
1659 emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0));
1660 }
1661 J->needsnap = 1;
1662 return 0;
1663 }
1664 }
1665
1666 /* Determine result type of table traversal. */
1667 static IRType rec_next_types(GCtab *t, uint32_t idx)
1668 {
1669 for (; idx < t->asize; idx++) {
1670 cTValue *a = arrayslot(t, idx);
1671 if (LJ_LIKELY(!tvisnil(a)))
1672 return (LJ_DUALNUM ? IRT_INT : IRT_NUM) + (itype2irt(a) << 8);
1673 }
1674 idx -= t->asize;
1675 for (; idx <= t->hmask; idx++) {
1676 Node *n = &noderef(t->node)[idx];
1677 if (!tvisnil(&n->val))
1678 return itype2irt(&n->key) + (itype2irt(&n->val) << 8);
1679 }
1680 return IRT_NIL + (IRT_NIL << 8);
1681 }
1682
1683 /* Record a table traversal step aka next(). */
1684 int lj_record_next(jit_State *J, RecordIndex *ix)
1685 {
1686 IRType t, tkey, tval;
1687 TRef trvk;
1688 t = rec_next_types(tabV(&ix->tabv), ix->keyv.u32.lo);
1689 tkey = (t & 0xff); tval = (t >> 8);
1690 trvk = lj_ir_call(J, IRCALL_lj_vm_next, ix->tab, ix->key);
1691 if (ix->mobj || tkey == IRT_NIL) {
1692 TRef idx = emitir(IRTI(IR_HIOP), trvk, trvk);
1693 /* Always check for invalid key from next() for nil result. */
1694 if (!ix->mobj) emitir(IRTGI(IR_NE), idx, lj_ir_kint(J, -1));
1695 ix->mobj = idx;
1696 }
1697 ix->key = lj_record_vload(J, trvk, 1, tkey);
1698 if (tkey == IRT_NIL || ix->idxchain) { /* Omit value type check. */
1699 ix->val = TREF_NIL;
1700 return 1;
1701 } else { /* Need value. */
1702 ix->val = lj_record_vload(J, trvk, 0, tval);
1703 return 2;
1704 }
1705 }
1706
1707 static void rec_tsetm(jit_State *J, BCReg ra, BCReg rn, int32_t i)
1708 {
1709 RecordIndex ix;
1710 cTValue *basev = J->L->base;
1711 GCtab *t = tabV(&basev[ra-1]);
1712 settabV(J->L, &ix.tabv, t);
1713 ix.tab = getslot(J, ra-1);
1714 ix.idxchain = 0;
1715 #ifdef LUAJIT_ENABLE_TABLE_BUMP
1716 if ((J->flags & JIT_F_OPT_SINK)) {
1717 if (t->asize < i+rn-ra)
1718 lj_tab_reasize(J->L, t, i+rn-ra);
1719 setnilV(&ix.keyv);
1720 rec_idx_bump(J, &ix);
1721 }
1722 #endif
1723 for (; ra < rn; i++, ra++) {
1724 setintV(&ix.keyv, i);
1725 ix.key = lj_ir_kint(J, i);
1726 copyTV(J->L, &ix.valv, &basev[ra]);
1727 ix.val = getslot(J, ra);
1728 lj_record_idx(J, &ix);
1729 }
1730 }
1731
1732 /* -- Upvalue access ------------------------------------------------------ */
1733
1734 /* Check whether upvalue is immutable and ok to constify. */
1735 static int rec_upvalue_constify(jit_State *J, GCupval *uvp)
1736 {
1737 if (uvp->immutable) {
1738 cTValue *o = uvval(uvp);
1739 /* Don't constify objects that may retain large amounts of memory. */
1740 #if LJ_HASFFI
1741 if (tviscdata(o)) {
1742 GCcdata *cd = cdataV(o);
1743 if (!cdataisv(cd) && !(cd->marked & LJ_GC_CDATA_FIN)) {
1744 CType *ct = ctype_raw(ctype_ctsG(J2G(J)), cd->ctypeid);
1745 if (!ctype_hassize(ct->info) || ct->size <= 16)
1746 return 1;
1747 }
1748 return 0;
1749 }
1750 #else
1751 UNUSED(J);
1752 #endif
1753 if (!(tvistab(o) || tvisudata(o) || tvisthread(o)))
1754 return 1;
1755 }
1756 return 0;
1757 }
1758
1759 /* Record upvalue load/store. */
1760 static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val)
1761 {
1762 GCupval *uvp = &gcref(J->fn->l.uvptr[uv])->uv;
1763 TRef fn = getcurrf(J);
1764 IRRef uref;
1765 int needbarrier = 0;
1766 if (rec_upvalue_constify(J, uvp)) { /* Try to constify immutable upvalue. */
1767 TRef tr, kfunc;
1768 lj_assertJ(val == 0, "bad usage");
1769 if (!tref_isk(fn)) { /* Late specialization of current function. */
1770 if (J->pt->flags >= PROTO_CLC_POLY)
1771 goto noconstify;
1772 kfunc = lj_ir_kfunc(J, J->fn);
1773 emitir(IRTG(IR_EQ, IRT_FUNC), fn, kfunc);
1774 #if LJ_FR2
1775 J->base[-2] = kfunc;
1776 #else
1777 J->base[-1] = kfunc | TREF_FRAME;
1778 #endif
1779 fn = kfunc;
1780 }
1781 tr = lj_record_constify(J, uvval(uvp));
1782 if (tr)
1783 return tr;
1784 }
1785 noconstify:
1786 /* Note: this effectively limits LJ_MAX_UPVAL to 127. */
1787 uv = (uv << 8) | (hashrot(uvp->dhash, uvp->dhash + HASH_BIAS) & 0xff);
1788 if (!uvp->closed) {
1789 /* In current stack? */
1790 if (uvval(uvp) >= tvref(J->L->stack) &&
1791 uvval(uvp) < tvref(J->L->maxstack)) {
1792 int32_t slot = (int32_t)(uvval(uvp) - (J->L->base - J->baseslot));
1793 if (slot >= 0) { /* Aliases an SSA slot? */
1794 uref = tref_ref(emitir(IRT(IR_UREFO, IRT_PGC), fn, uv));
1795 emitir(IRTG(IR_EQ, IRT_PGC),
1796 REF_BASE,
1797 emitir(IRT(IR_ADD, IRT_PGC), uref,
1798 lj_ir_kintpgc(J, (slot - 1 - LJ_FR2) * -8)));
1799 slot -= (int32_t)J->baseslot; /* Note: slot number may be negative! */
1800 if (val == 0) {
1801 return getslot(J, slot);
1802 } else {
1803 J->base[slot] = val;
1804 if (slot >= (int32_t)J->maxslot) J->maxslot = (BCReg)(slot+1);
1805 return 0;
1806 }
1807 }
1808 }
1809 /* IR_UREFO+IRT_IGC is not checked for open-ness at runtime.
1810 ** Always marked as a guard, since it might get promoted to IRT_PGC later.
1811 */
1812 uref = emitir(IRTG(IR_UREFO, tref_isgcv(val) ? IRT_PGC : IRT_IGC), fn, uv);
1813 uref = tref_ref(uref);
1814 emitir(IRTG(IR_UGT, IRT_PGC),
1815 emitir(IRT(IR_SUB, IRT_PGC), uref, REF_BASE),
1816 lj_ir_kintpgc(J, (J->baseslot + J->maxslot) * 8));
1817 } else {
1818 /* If fn is constant, then so is the GCupval*, and the upvalue cannot
1819 ** transition back to open, so no guard is required in this case.
1820 */
1821 IRType t = (tref_isk(fn) ? 0 : IRT_GUARD) | IRT_PGC;
1822 uref = tref_ref(emitir(IRT(IR_UREFC, t), fn, uv));
1823 needbarrier = 1;
1824 }
1825 if (val == 0) { /* Upvalue load */
1826 IRType t = itype2irt(uvval(uvp));
1827 TRef res = emitir(IRTG(IR_ULOAD, t), uref, 0);
1828 if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitive refs. */
1829 return res;
1830 } else { /* Upvalue store. */
1831 /* Convert int to number before storing. */
1832 if (!LJ_DUALNUM && tref_isinteger(val))
1833 val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
1834 emitir(IRT(IR_USTORE, tref_type(val)), uref, val);
1835 if (needbarrier && tref_isgcv(val))
1836 emitir(IRT(IR_OBAR, IRT_NIL), uref, val);
1837 J->needsnap = 1;
1838 return 0;
1839 }
1840 }
1841
1842 /* -- Record calls to Lua functions --------------------------------------- */
1843
1844 /* Check unroll limits for calls. */
1845 static void check_call_unroll(jit_State *J, TraceNo lnk)
1846 {
1847 cTValue *frame = J->L->base - 1;
1848 void *pc = mref(frame_func(frame)->l.pc, void);
1849 int32_t depth = J->framedepth;
1850 int32_t count = 0;
1851 if ((J->pt->flags & PROTO_VARARG)) depth--; /* Vararg frame still missing. */
1852 for (; depth > 0; depth--) { /* Count frames with same prototype. */
1853 if (frame_iscont(frame)) depth--;
1854 frame = frame_prev(frame);
1855 if (mref(frame_func(frame)->l.pc, void) == pc)
1856 count++;
1857 }
1858 if (J->pc == J->startpc) {
1859 if (count + J->tailcalled > J->param[JIT_P_recunroll]) {
1860 J->pc++;
1861 if (J->framedepth + J->retdepth == 0)
1862 lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Tail-rec. */
1863 else
1864 lj_record_stop(J, LJ_TRLINK_UPREC, J->cur.traceno); /* Up-recursion. */
1865 }
1866 } else {
1867 if (count > J->param[JIT_P_callunroll]) {
1868 if (lnk) { /* Possible tail- or up-recursion. */
1869 lj_trace_flush(J, lnk); /* Flush trace that only returns. */
1870 /* Set a small, pseudo-random hotcount for a quick retry of JFUNC*. */
1871 hotcount_set(J2GG(J), J->pc+1, lj_prng_u64(&J2G(J)->prng) & 15u);
1872 }
1873 lj_trace_err(J, LJ_TRERR_CUNROLL);
1874 }
1875 }
1876 }
1877
1878 /* Record Lua function setup. */
1879 static void rec_func_setup(jit_State *J)
1880 {
1881 GCproto *pt = J->pt;
1882 BCReg s, numparams = pt->numparams;
1883 if ((pt->flags & PROTO_NOJIT))
1884 lj_trace_err(J, LJ_TRERR_CJITOFF);
1885 if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS)
1886 lj_trace_err(J, LJ_TRERR_STACKOV);
1887 /* Fill up missing parameters with nil. */
1888 for (s = J->maxslot; s < numparams; s++)
1889 J->base[s] = TREF_NIL;
1890 /* The remaining slots should never be read before they are written. */
1891 J->maxslot = numparams;
1892 }
1893
1894 /* Record Lua vararg function setup. */
1895 static void rec_func_vararg(jit_State *J)
1896 {
1897 GCproto *pt = J->pt;
1898 BCReg s, fixargs, vframe = J->maxslot+1+LJ_FR2;
1899 lj_assertJ((pt->flags & PROTO_VARARG), "FUNCV in non-vararg function");
1900 if (J->baseslot + vframe + pt->framesize >= LJ_MAX_JSLOTS)
1901 lj_trace_err(J, LJ_TRERR_STACKOV);
1902 J->base[vframe-1-LJ_FR2] = J->base[-1-LJ_FR2]; /* Copy function up. */
1903 #if LJ_FR2
1904 J->base[vframe-1] = TREF_FRAME;
1905 #endif
1906 /* Copy fixarg slots up and set their original slots to nil. */
1907 fixargs = pt->numparams < J->maxslot ? pt->numparams : J->maxslot;
1908 for (s = 0; s < fixargs; s++) {
1909 J->base[vframe+s] = J->base[s];
1910 J->base[s] = TREF_NIL;
1911 }
1912 J->maxslot = fixargs;
1913 J->framedepth++;
1914 J->base += vframe;
1915 J->baseslot += vframe;
1916 }
1917
1918 /* Record entry to a Lua function. */
1919 static void rec_func_lua(jit_State *J)
1920 {
1921 rec_func_setup(J);
1922 check_call_unroll(J, 0);
1923 }
1924
1925 /* Record entry to an already compiled function. */
1926 static void rec_func_jit(jit_State *J, TraceNo lnk)
1927 {
1928 GCtrace *T;
1929 rec_func_setup(J);
1930 T = traceref(J, lnk);
1931 if (T->linktype == LJ_TRLINK_RETURN) { /* Trace returns to interpreter? */
1932 check_call_unroll(J, lnk);
1933 /* Temporarily unpatch JFUNC* to continue recording across function. */
1934 J->patchins = *J->pc;
1935 J->patchpc = (BCIns *)J->pc;
1936 *J->patchpc = T->startins;
1937 return;
1938 }
1939 J->instunroll = 0; /* Cannot continue across a compiled function. */
1940 if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
1941 lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Extra tail-rec. */
1942 else
1943 lj_record_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the function. */
1944 }
1945
1946 /* -- Vararg handling ----------------------------------------------------- */
1947
1948 /* Detect y = select(x, ...) idiom. */
1949 static int select_detect(jit_State *J)
1950 {
1951 BCIns ins = J->pc[1];
1952 if (bc_op(ins) == BC_CALLM && bc_b(ins) == 2 && bc_c(ins) == 1) {
1953 cTValue *func = &J->L->base[bc_a(ins)];
1954 if (tvisfunc(func) && funcV(func)->c.ffid == FF_select) {
1955 TRef kfunc = lj_ir_kfunc(J, funcV(func));
1956 emitir(IRTG(IR_EQ, IRT_FUNC), getslot(J, bc_a(ins)), kfunc);
1957 return 1;
1958 }
1959 }
1960 return 0;
1961 }
1962
1963 /* Record vararg instruction. */
1964 static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults)
1965 {
1966 int32_t numparams = J->pt->numparams;
1967 ptrdiff_t nvararg = frame_delta(J->L->base-1) - numparams - 1 - LJ_FR2;
1968 lj_assertJ(frame_isvarg(J->L->base-1), "VARG in non-vararg frame");
1969 if (LJ_FR2 && dst > J->maxslot)
1970 J->base[dst-1] = 0; /* Prevent resurrection of unrelated slot. */
1971 if (J->framedepth > 0) { /* Simple case: varargs defined on-trace. */
1972 ptrdiff_t i;
1973 if (nvararg < 0) nvararg = 0;
1974 if (nresults != 1) {
1975 if (nresults == -1) nresults = nvararg;
1976 J->maxslot = dst + (BCReg)nresults;
1977 } else if (dst >= J->maxslot) {
1978 J->maxslot = dst + 1;
1979 }
1980 if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS)
1981 lj_trace_err(J, LJ_TRERR_STACKOV);
1982 for (i = 0; i < nresults; i++)
1983 J->base[dst+i] = i < nvararg ? getslot(J, i - nvararg - 1 - LJ_FR2) : TREF_NIL;
1984 } else { /* Unknown number of varargs passed to trace. */
1985 TRef fr = emitir(IRTI(IR_SLOAD), LJ_FR2, IRSLOAD_READONLY|IRSLOAD_FRAME);
1986 int32_t frofs = 8*(1+LJ_FR2+numparams)+FRAME_VARG;
1987 if (nresults >= 0) { /* Known fixed number of results. */
1988 ptrdiff_t i;
1989 if (nvararg > 0) {
1990 ptrdiff_t nload = nvararg >= nresults ? nresults : nvararg;
1991 TRef vbase;
1992 if (nvararg >= nresults)
1993 emitir(IRTGI(IR_GE), fr, lj_ir_kint(J, frofs+8*(int32_t)nresults));
1994 else
1995 emitir(IRTGI(IR_EQ), fr,
1996 lj_ir_kint(J, (int32_t)frame_ftsz(J->L->base-1)));
1997 vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr);
1998 vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase,
1999 lj_ir_kintpgc(J, frofs-8*(1+LJ_FR2)));
2000 for (i = 0; i < nload; i++) {
2001 IRType t = itype2irt(&J->L->base[i-1-LJ_FR2-nvararg]);
2002 J->base[dst+i] = lj_record_vload(J, vbase, (MSize)i, t);
2003 }
2004 } else {
2005 emitir(IRTGI(IR_LE), fr, lj_ir_kint(J, frofs));
2006 nvararg = 0;
2007 }
2008 for (i = nvararg; i < nresults; i++)
2009 J->base[dst+i] = TREF_NIL;
2010 if (nresults != 1 || dst >= J->maxslot) {
2011 J->maxslot = dst + (BCReg)nresults;
2012 }
2013 } else if (select_detect(J)) { /* y = select(x, ...) */
2014 TRef tridx = getslot(J, dst-1);
2015 TRef tr = TREF_NIL;
2016 ptrdiff_t idx = lj_ffrecord_select_mode(J, tridx, &J->L->base[dst-1]);
2017 if (idx < 0) goto nyivarg;
2018 if (idx != 0 && !tref_isinteger(tridx)) {
2019 if (tref_isstr(tridx))
2020 tridx = emitir(IRTG(IR_STRTO, IRT_NUM), tridx, 0);
2021 tridx = emitir(IRTGI(IR_CONV), tridx, IRCONV_INT_NUM|IRCONV_INDEX);
2022 }
2023 if (idx != 0 && tref_isk(tridx)) {
2024 emitir(IRTGI(idx <= nvararg ? IR_GE : IR_LT),
2025 fr, lj_ir_kint(J, frofs+8*(int32_t)idx));
2026 frofs -= 8; /* Bias for 1-based index. */
2027 } else if (idx <= nvararg) { /* Compute size. */
2028 TRef tmp = emitir(IRTI(IR_ADD), fr, lj_ir_kint(J, -frofs));
2029 if (numparams)
2030 emitir(IRTGI(IR_GE), tmp, lj_ir_kint(J, 0));
2031 tr = emitir(IRTI(IR_BSHR), tmp, lj_ir_kint(J, 3));
2032 if (idx != 0) {
2033 tridx = emitir(IRTI(IR_ADD), tridx, lj_ir_kint(J, -1));
2034 rec_idx_abc(J, tr, tridx, (uint32_t)nvararg);
2035 }
2036 } else {
2037 TRef tmp = lj_ir_kint(J, frofs);
2038 if (idx != 0) {
2039 TRef tmp2 = emitir(IRTI(IR_BSHL), tridx, lj_ir_kint(J, 3));
2040 tmp = emitir(IRTI(IR_ADD), tmp2, tmp);
2041 } else {
2042 tr = lj_ir_kint(J, 0);
2043 }
2044 emitir(IRTGI(IR_LT), fr, tmp);
2045 }
2046 if (idx != 0 && idx <= nvararg) {
2047 IRType t;
2048 TRef aref, vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr);
2049 vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase,
2050 lj_ir_kintpgc(J, frofs-(8<<LJ_FR2)));
2051 t = itype2irt(&J->L->base[idx-2-LJ_FR2-nvararg]);
2052 aref = emitir(IRT(IR_AREF, IRT_PGC), vbase, tridx);
2053 tr = lj_record_vload(J, aref, 0, t);
2054 }
2055 J->base[dst-2-LJ_FR2] = tr;
2056 J->maxslot = dst-1-LJ_FR2;
2057 J->bcskip = 2; /* Skip CALLM + select. */
2058 } else {
2059 nyivarg:
2060 setintV(&J->errinfo, BC_VARG);
2061 lj_trace_err_info(J, LJ_TRERR_NYIBC);
2062 }
2063 }
2064 }
2065
2066 /* -- Record allocations -------------------------------------------------- */
2067
2068 static TRef rec_tnew(jit_State *J, uint32_t ah)
2069 {
2070 uint32_t asize = ah & 0x7ff;
2071 uint32_t hbits = ah >> 11;
2072 TRef tr;
2073 if (asize == 0x7ff) asize = 0x801;
2074 tr = emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits);
2075 #ifdef LUAJIT_ENABLE_TABLE_BUMP
2076 J->rbchash[(tr & (RBCHASH_SLOTS-1))].ref = tref_ref(tr);
2077 setmref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pc, J->pc);
2078 setgcref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt));
2079 #endif
2080 return tr;
2081 }
2082
2083 /* -- Concatenation ------------------------------------------------------- */
2084
2085 typedef struct RecCatDataCP {
2086 TValue savetv[5+LJ_FR2];
2087 jit_State *J;
2088 BCReg baseslot, topslot;
2089 TRef tr;
2090 } RecCatDataCP;
2091
2092 static TValue *rec_mm_concat_cp(lua_State *L, lua_CFunction dummy, void *ud)
2093 {
2094 RecCatDataCP *rcd = (RecCatDataCP *)ud;
2095 jit_State *J = rcd->J;
2096 BCReg baseslot = rcd->baseslot, topslot = rcd->topslot;
2097 TRef *top = &J->base[topslot];
2098 BCReg s;
2099 RecordIndex ix;
2100 UNUSED(L); UNUSED(dummy);
2101 lj_assertJ(baseslot < topslot, "bad CAT arg");
2102 for (s = baseslot; s <= topslot; s++)
2103 (void)getslot(J, s); /* Ensure all arguments have a reference. */
2104 if (tref_isnumber_str(top[0]) && tref_isnumber_str(top[-1])) {
2105 TRef tr, hdr, *trp, *xbase, *base = &J->base[baseslot];
2106 /* First convert numbers to strings. */
2107 for (trp = top; trp >= base; trp--) {
2108 if (tref_isnumber(*trp))
2109 *trp = emitir(IRT(IR_TOSTR, IRT_STR), *trp,
2110 tref_isnum(*trp) ? IRTOSTR_NUM : IRTOSTR_INT);
2111 else if (!tref_isstr(*trp))
2112 break;
2113 }
2114 xbase = ++trp;
2115 tr = hdr = emitir(IRT(IR_BUFHDR, IRT_PGC),
2116 lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
2117 do {
2118 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, *trp++);
2119 } while (trp <= top);
2120 tr = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
2121 J->maxslot = (BCReg)(xbase - J->base);
2122 if (xbase == base) {
2123 rcd->tr = tr; /* Return simple concatenation result. */
2124 return NULL;
2125 }
2126 /* Pass partial result. */
2127 rcd->topslot = topslot = J->maxslot--;
2128 /* Save updated range of slots. */
2129 memcpy(rcd->savetv, &L->base[topslot-1], sizeof(rcd->savetv));
2130 *xbase = tr;
2131 top = xbase;
2132 setstrV(J->L, &ix.keyv, &J2G(J)->strempty); /* Simulate string result. */
2133 } else {
2134 J->maxslot = topslot-1;
2135 copyTV(J->L, &ix.keyv, &J->L->base[topslot]);
2136 }
2137 copyTV(J->L, &ix.tabv, &J->L->base[topslot-1]);
2138 ix.tab = top[-1];
2139 ix.key = top[0];
2140 rec_mm_arith(J, &ix, MM_concat); /* Call __concat metamethod. */
2141 rcd->tr = 0; /* No result yet. */
2142 return NULL;
2143 }
2144
2145 static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot)
2146 {
2147 lua_State *L = J->L;
2148 ptrdiff_t delta = L->top - L->base;
2149 TValue errobj;
2150 RecCatDataCP rcd;
2151 int errcode;
2152 rcd.J = J;
2153 rcd.baseslot = baseslot;
2154 rcd.topslot = topslot;
2155 /* Save slots. */
2156 memcpy(rcd.savetv, &L->base[topslot-1], sizeof(rcd.savetv));
2157 errcode = lj_vm_cpcall(L, NULL, &rcd, rec_mm_concat_cp);
2158 if (errcode) copyTV(L, &errobj, L->top-1);
2159 /* Restore slots. */
2160 memcpy(&L->base[rcd.topslot-1], rcd.savetv, sizeof(rcd.savetv));
2161 if (errcode) {
2162 L->top = L->base + delta;
2163 copyTV(L, L->top++, &errobj);
2164 return (TRef)(-errcode);
2165 }
2166 return rcd.tr;
2167 }
2168
2169 /* -- Record bytecode ops ------------------------------------------------- */
2170
2171 /* Prepare for comparison. */
2172 static void rec_comp_prep(jit_State *J)
2173 {
2174 /* Prevent merging with snapshot #0 (GC exit) since we fixup the PC. */
2175 if (J->cur.nsnap == 1 && J->cur.snap[0].ref == J->cur.nins)
2176 emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0);
2177 lj_snap_add(J);
2178 }
2179
2180 /* Fixup comparison. */
2181 static void rec_comp_fixup(jit_State *J, const BCIns *pc, int cond)
2182 {
2183 BCIns jmpins = pc[1];
2184 const BCIns *npc = pc + 2 + (cond ? bc_j(jmpins) : 0);
2185 SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
2186 /* Set PC to opposite target to avoid re-recording the comp. in side trace. */
2187 #if LJ_FR2
2188 SnapEntry *flink = &J->cur.snapmap[snap->mapofs + snap->nent];
2189 uint64_t pcbase;
2190 memcpy(&pcbase, flink, sizeof(uint64_t));
2191 pcbase = (pcbase & 0xff) | (u64ptr(npc) << 8);
2192 memcpy(flink, &pcbase, sizeof(uint64_t));
2193 #else
2194 J->cur.snapmap[snap->mapofs + snap->nent] = SNAP_MKPC(npc);
2195 #endif
2196 J->needsnap = 1;
2197 if (bc_a(jmpins) < J->maxslot) J->maxslot = bc_a(jmpins);
2198 lj_snap_shrink(J); /* Shrink last snapshot if possible. */
2199 }
2200
2201 /* Record the next bytecode instruction (_before_ it's executed). */
2202 void lj_record_ins(jit_State *J)
2203 {
2204 cTValue *lbase;
2205 RecordIndex ix;
2206 const BCIns *pc;
2207 BCIns ins;
2208 BCOp op;
2209 TRef ra, rb, rc;
2210
2211 /* Perform post-processing action before recording the next instruction. */
2212 if (LJ_UNLIKELY(J->postproc != LJ_POST_NONE)) {
2213 switch (J->postproc) {
2214 case LJ_POST_FIXCOMP: /* Fixup comparison. */
2215 pc = (const BCIns *)(uintptr_t)J2G(J)->tmptv.u64;
2216 rec_comp_fixup(J, pc, (!tvistruecond(&J2G(J)->tmptv2) ^ (bc_op(*pc)&1)));
2217 /* fallthrough */
2218 case LJ_POST_FIXGUARD: /* Fixup and emit pending guard. */
2219 case LJ_POST_FIXGUARDSNAP: /* Fixup and emit pending guard and snapshot. */
2220 if (!tvistruecond(&J2G(J)->tmptv2)) {
2221 J->fold.ins.o ^= 1; /* Flip guard to opposite. */
2222 if (J->postproc == LJ_POST_FIXGUARDSNAP) {
2223 SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
2224 J->cur.snapmap[snap->mapofs+snap->nent-1]--; /* False -> true. */
2225 }
2226 }
2227 lj_opt_fold(J); /* Emit pending guard. */
2228 /* fallthrough */
2229 case LJ_POST_FIXBOOL:
2230 if (!tvistruecond(&J2G(J)->tmptv2)) {
2231 BCReg s;
2232 TValue *tv = J->L->base;
2233 for (s = 0; s < J->maxslot; s++) /* Fixup stack slot (if any). */
2234 if (J->base[s] == TREF_TRUE && tvisfalse(&tv[s])) {
2235 J->base[s] = TREF_FALSE;
2236 break;
2237 }
2238 }
2239 break;
2240 case LJ_POST_FIXCONST:
2241 {
2242 BCReg s;
2243 TValue *tv = J->L->base;
2244 for (s = 0; s < J->maxslot; s++) /* Constify stack slots (if any). */
2245 if (J->base[s] == TREF_NIL && !tvisnil(&tv[s]))
2246 J->base[s] = lj_record_constify(J, &tv[s]);
2247 }
2248 break;
2249 case LJ_POST_FFRETRY: /* Suppress recording of retried fast function. */
2250 if (bc_op(*J->pc) >= BC__MAX)
2251 return;
2252 break;
2253 default: lj_assertJ(0, "bad post-processing mode"); break;
2254 }
2255 J->postproc = LJ_POST_NONE;
2256 }
2257
2258 /* Need snapshot before recording next bytecode (e.g. after a store). */
2259 if (J->needsnap) {
2260 J->needsnap = 0;
2261 if (J->pt) lj_snap_purge(J);
2262 lj_snap_add(J);
2263 J->mergesnap = 1;
2264 }
2265
2266 /* Skip some bytecodes. */
2267 if (LJ_UNLIKELY(J->bcskip > 0)) {
2268 J->bcskip--;
2269 return;
2270 }
2271
2272 /* Record only closed loops for root traces. */
2273 pc = J->pc;
2274 if (J->framedepth == 0 &&
2275 (MSize)((char *)pc - (char *)J->bc_min) >= J->bc_extent)
2276 lj_trace_err(J, LJ_TRERR_LLEAVE);
2277
2278 #ifdef LUA_USE_ASSERT
2279 rec_check_slots(J);
2280 rec_check_ir(J);
2281 #endif
2282
2283 #if LJ_HASPROFILE
2284 rec_profile_ins(J, pc);
2285 #endif
2286
2287 /* Keep a copy of the runtime values of var/num/str operands. */
2288 #define rav (&ix.valv)
2289 #define rbv (&ix.tabv)
2290 #define rcv (&ix.keyv)
2291
2292 lbase = J->L->base;
2293 ins = *pc;
2294 op = bc_op(ins);
2295 ra = bc_a(ins);
2296 ix.val = 0;
2297 switch (bcmode_a(op)) {
2298 case BCMvar:
2299 copyTV(J->L, rav, &lbase[ra]); ix.val = ra = getslot(J, ra); break;
2300 default: break; /* Handled later. */
2301 }
2302 rb = bc_b(ins);
2303 rc = bc_c(ins);
2304 switch (bcmode_b(op)) {
2305 case BCMnone: rb = 0; rc = bc_d(ins); break; /* Upgrade rc to 'rd'. */
2306 case BCMvar:
2307 copyTV(J->L, rbv, &lbase[rb]); ix.tab = rb = getslot(J, rb); break;
2308 default: break; /* Handled later. */
2309 }
2310 switch (bcmode_c(op)) {
2311 case BCMvar:
2312 copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break;
2313 case BCMpri: setpriV(rcv, ~rc); ix.key = rc = TREF_PRI(IRT_NIL+rc); break;
2314 case BCMnum: { cTValue *tv = proto_knumtv(J->pt, rc);
2315 copyTV(J->L, rcv, tv); ix.key = rc = tvisint(tv) ? lj_ir_kint(J, intV(tv)) :
2316 tv->u32.hi == LJ_KEYINDEX ? (lj_ir_kint(J, 0) | TREF_KEYINDEX) :
2317 lj_ir_knumint(J, numV(tv)); } break;
2318 case BCMstr: { GCstr *s = gco2str(proto_kgc(J->pt, ~(ptrdiff_t)rc));
2319 setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break;
2320 default: break; /* Handled later. */
2321 }
2322
2323 switch (op) {
2324
2325 /* -- Comparison ops ---------------------------------------------------- */
2326
2327 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2328 #if LJ_HASFFI
2329 if (tref_iscdata(ra) || tref_iscdata(rc)) {
2330 rec_mm_comp_cdata(J, &ix, op, ((int)op & 2) ? MM_le : MM_lt);
2331 break;
2332 }
2333 #endif
2334 /* Emit nothing for two numeric or string consts. */
2335 if (!(tref_isk2(ra,rc) && tref_isnumber_str(ra) && tref_isnumber_str(rc))) {
2336 IRType ta = tref_isinteger(ra) ? IRT_INT : tref_type(ra);
2337 IRType tc = tref_isinteger(rc) ? IRT_INT : tref_type(rc);
2338 int irop;
2339 if (ta != tc) {
2340 /* Widen mixed number/int comparisons to number/number comparison. */
2341 if (ta == IRT_INT && tc == IRT_NUM) {
2342 ra = emitir(IRTN(IR_CONV), ra, IRCONV_NUM_INT);
2343 ta = IRT_NUM;
2344 } else if (ta == IRT_NUM && tc == IRT_INT) {
2345 rc = emitir(IRTN(IR_CONV), rc, IRCONV_NUM_INT);
2346 } else if (LJ_52) {
2347 ta = IRT_NIL; /* Force metamethod for different types. */
2348 } else if (!((ta == IRT_FALSE || ta == IRT_TRUE) &&
2349 (tc == IRT_FALSE || tc == IRT_TRUE))) {
2350 break; /* Interpreter will throw for two different types. */
2351 }
2352 }
2353 rec_comp_prep(J);
2354 irop = (int)op - (int)BC_ISLT + (int)IR_LT;
2355 if (ta == IRT_NUM) {
2356 if ((irop & 1)) irop ^= 4; /* ISGE/ISGT are unordered. */
2357 if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop))
2358 irop ^= 5;
2359 } else if (ta == IRT_INT) {
2360 if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop))
2361 irop ^= 1;
2362 } else if (ta == IRT_STR) {
2363 if (!lj_ir_strcmp(strV(rav), strV(rcv), (IROp)irop)) irop ^= 1;
2364 ra = lj_ir_call(J, IRCALL_lj_str_cmp, ra, rc);
2365 rc = lj_ir_kint(J, 0);
2366 ta = IRT_INT;
2367 } else {
2368 rec_mm_comp(J, &ix, (int)op);
2369 break;
2370 }
2371 emitir(IRTG(irop, ta), ra, rc);
2372 rec_comp_fixup(J, J->pc, ((int)op ^ irop) & 1);
2373 }
2374 break;
2375
2376 case BC_ISEQV: case BC_ISNEV:
2377 case BC_ISEQS: case BC_ISNES:
2378 case BC_ISEQN: case BC_ISNEN:
2379 case BC_ISEQP: case BC_ISNEP:
2380 #if LJ_HASFFI
2381 if (tref_iscdata(ra) || tref_iscdata(rc)) {
2382 rec_mm_comp_cdata(J, &ix, op, MM_eq);
2383 break;
2384 }
2385 #endif
2386 /* Emit nothing for two non-table, non-udata consts. */
2387 if (!(tref_isk2(ra, rc) && !(tref_istab(ra) || tref_isudata(ra)))) {
2388 int diff;
2389 rec_comp_prep(J);
2390 diff = lj_record_objcmp(J, ra, rc, rav, rcv);
2391 if (diff == 2 || !(tref_istab(ra) || tref_isudata(ra)))
2392 rec_comp_fixup(J, J->pc, ((int)op & 1) == !diff);
2393 else if (diff == 1) /* Only check __eq if different, but same type. */
2394 rec_mm_equal(J, &ix, (int)op);
2395 }
2396 break;
2397
2398 /* -- Unary test and copy ops ------------------------------------------- */
2399
2400 case BC_ISTC: case BC_ISFC:
2401 if ((op & 1) == tref_istruecond(rc))
2402 rc = 0; /* Don't store if condition is not true. */
2403 /* fallthrough */
2404 case BC_IST: case BC_ISF: /* Type specialization suffices. */
2405 if (bc_a(pc[1]) < J->maxslot)
2406 J->maxslot = bc_a(pc[1]); /* Shrink used slots. */
2407 break;
2408
2409 case BC_ISTYPE: case BC_ISNUM:
2410 /* These coercions need to correspond with lj_meta_istype(). */
2411 if (LJ_DUALNUM && rc == ~LJ_TNUMX+1)
2412 ra = lj_opt_narrow_toint(J, ra);
2413 else if (rc == ~LJ_TNUMX+2)
2414 ra = lj_ir_tonum(J, ra);
2415 else if (rc == ~LJ_TSTR+1)
2416 ra = lj_ir_tostr(J, ra);
2417 /* else: type specialization suffices. */
2418 J->base[bc_a(ins)] = ra;
2419 break;
2420
2421 /* -- Unary ops --------------------------------------------------------- */
2422
2423 case BC_NOT:
2424 /* Type specialization already forces const result. */
2425 rc = tref_istruecond(rc) ? TREF_FALSE : TREF_TRUE;
2426 break;
2427
2428 case BC_LEN:
2429 if (tref_isstr(rc))
2430 rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN);
2431 else if (!LJ_52 && tref_istab(rc))
2432 rc = emitir(IRTI(IR_ALEN), rc, TREF_NIL);
2433 else
2434 rc = rec_mm_len(J, rc, rcv);
2435 break;
2436
2437 /* -- Arithmetic ops ---------------------------------------------------- */
2438
2439 case BC_UNM:
2440 if (tref_isnumber_str(rc)) {
2441 rc = lj_opt_narrow_unm(J, rc, rcv);
2442 } else {
2443 ix.tab = rc;
2444 copyTV(J->L, &ix.tabv, rcv);
2445 rc = rec_mm_arith(J, &ix, MM_unm);
2446 }
2447 break;
2448
2449 case BC_ADDNV: case BC_SUBNV: case BC_MULNV: case BC_DIVNV: case BC_MODNV:
2450 /* Swap rb/rc and rbv/rcv. rav is temp. */
2451 ix.tab = rc; ix.key = rc = rb; rb = ix.tab;
2452 copyTV(J->L, rav, rbv);
2453 copyTV(J->L, rbv, rcv);
2454 copyTV(J->L, rcv, rav);
2455 if (op == BC_MODNV)
2456 goto recmod;
2457 /* fallthrough */
2458 case BC_ADDVN: case BC_SUBVN: case BC_MULVN: case BC_DIVVN:
2459 case BC_ADDVV: case BC_SUBVV: case BC_MULVV: case BC_DIVVV: {
2460 MMS mm = bcmode_mm(op);
2461 if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2462 rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv,
2463 (int)mm - (int)MM_add + (int)IR_ADD);
2464 else
2465 rc = rec_mm_arith(J, &ix, mm);
2466 break;
2467 }
2468
2469 case BC_MODVN: case BC_MODVV:
2470 recmod:
2471 if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2472 rc = lj_opt_narrow_mod(J, rb, rc, rbv, rcv);
2473 else
2474 rc = rec_mm_arith(J, &ix, MM_mod);
2475 break;
2476
2477 case BC_POW:
2478 if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2479 rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv, IR_POW);
2480 else
2481 rc = rec_mm_arith(J, &ix, MM_pow);
2482 break;
2483
2484 /* -- Miscellaneous ops ------------------------------------------------- */
2485
2486 case BC_CAT:
2487 rc = rec_cat(J, rb, rc);
2488 if (rc >= 0xffffff00)
2489 lj_err_throw(J->L, -(int32_t)rc); /* Propagate errors. */
2490 break;
2491
2492 /* -- Constant and move ops --------------------------------------------- */
2493
2494 case BC_MOV:
2495 /* Clear gap of method call to avoid resurrecting previous refs. */
2496 if (ra > J->maxslot) {
2497 #if LJ_FR2
2498 memset(J->base + J->maxslot, 0, (ra - J->maxslot) * sizeof(TRef));
2499 #else
2500 J->base[ra-1] = 0;
2501 #endif
2502 }
2503 break;
2504 case BC_KSTR: case BC_KNUM: case BC_KPRI:
2505 break;
2506 case BC_KSHORT:
2507 rc = lj_ir_kint(J, (int32_t)(int16_t)rc);
2508 break;
2509 case BC_KNIL:
2510 if (LJ_FR2 && ra > J->maxslot)
2511 J->base[ra-1] = 0;
2512 while (ra <= rc)
2513 J->base[ra++] = TREF_NIL;
2514 if (rc >= J->maxslot) J->maxslot = rc+1;
2515 break;
2516 #if LJ_HASFFI
2517 case BC_KCDATA:
2518 rc = lj_ir_kgc(J, proto_kgc(J->pt, ~(ptrdiff_t)rc), IRT_CDATA);
2519 break;
2520 #endif
2521
2522 /* -- Upvalue and function ops ------------------------------------------ */
2523
2524 case BC_UGET:
2525 rc = rec_upvalue(J, rc, 0);
2526 break;
2527 case BC_USETV: case BC_USETS: case BC_USETN: case BC_USETP:
2528 rec_upvalue(J, ra, rc);
2529 break;
2530
2531 /* -- Table ops --------------------------------------------------------- */
2532
2533 case BC_GGET: case BC_GSET:
2534 settabV(J->L, &ix.tabv, tabref(J->fn->l.env));
2535 ix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), getcurrf(J), IRFL_FUNC_ENV);
2536 ix.idxchain = LJ_MAX_IDXCHAIN;
2537 rc = lj_record_idx(J, &ix);
2538 break;
2539
2540 case BC_TGETB: case BC_TSETB:
2541 setintV(&ix.keyv, (int32_t)rc);
2542 ix.key = lj_ir_kint(J, (int32_t)rc);
2543 /* fallthrough */
2544 case BC_TGETV: case BC_TGETS: case BC_TSETV: case BC_TSETS:
2545 ix.idxchain = LJ_MAX_IDXCHAIN;
2546 rc = lj_record_idx(J, &ix);
2547 break;
2548 case BC_TGETR: case BC_TSETR:
2549 ix.idxchain = 0;
2550 rc = lj_record_idx(J, &ix);
2551 break;
2552
2553 case BC_TSETM:
2554 rec_tsetm(J, ra, (BCReg)(J->L->top - J->L->base), (int32_t)rcv->u32.lo);
2555 J->maxslot = ra; /* The table slot at ra-1 is the highest used slot. */
2556 break;
2557
2558 case BC_TNEW:
2559 rc = rec_tnew(J, rc);
2560 break;
2561 case BC_TDUP:
2562 rc = emitir(IRTG(IR_TDUP, IRT_TAB),
2563 lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0);
2564 #ifdef LUAJIT_ENABLE_TABLE_BUMP
2565 J->rbchash[(rc & (RBCHASH_SLOTS-1))].ref = tref_ref(rc);
2566 setmref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pc, pc);
2567 setgcref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt));
2568 #endif
2569 break;
2570
2571 /* -- Calls and vararg handling ----------------------------------------- */
2572
2573 case BC_ITERC:
2574 J->base[ra] = getslot(J, ra-3);
2575 J->base[ra+1+LJ_FR2] = getslot(J, ra-2);
2576 J->base[ra+2+LJ_FR2] = getslot(J, ra-1);
2577 { /* Do the actual copy now because lj_record_call needs the values. */
2578 TValue *b = &J->L->base[ra];
2579 copyTV(J->L, b, b-3);
2580 copyTV(J->L, b+1+LJ_FR2, b-2);
2581 copyTV(J->L, b+2+LJ_FR2, b-1);
2582 }
2583 lj_record_call(J, ra, (ptrdiff_t)rc-1);
2584 break;
2585
2586 /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */
2587 case BC_CALLM:
2588 rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
2589 /* fallthrough */
2590 case BC_CALL:
2591 lj_record_call(J, ra, (ptrdiff_t)rc-1);
2592 break;
2593
2594 case BC_CALLMT:
2595 rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
2596 /* fallthrough */
2597 case BC_CALLT:
2598 lj_record_tailcall(J, ra, (ptrdiff_t)rc-1);
2599 break;
2600
2601 case BC_VARG:
2602 rec_varg(J, ra, (ptrdiff_t)rb-1);
2603 break;
2604
2605 /* -- Returns ----------------------------------------------------------- */
2606
2607 case BC_RETM:
2608 /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */
2609 rc = (BCReg)(J->L->top - J->L->base) - ra + 1;
2610 /* fallthrough */
2611 case BC_RET: case BC_RET0: case BC_RET1:
2612 #if LJ_HASPROFILE
2613 rec_profile_ret(J);
2614 #endif
2615 lj_record_ret(J, ra, (ptrdiff_t)rc-1);
2616 break;
2617
2618 /* -- Loops and branches ------------------------------------------------ */
2619
2620 case BC_FORI:
2621 if (rec_for(J, pc, 0) != LOOPEV_LEAVE)
2622 J->loopref = J->cur.nins;
2623 break;
2624 case BC_JFORI:
2625 lj_assertJ(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL,
2626 "JFORI does not point to JFORL");
2627 if (rec_for(J, pc, 0) != LOOPEV_LEAVE) /* Link to existing loop. */
2628 lj_record_stop(J, LJ_TRLINK_ROOT, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J]));
2629 /* Continue tracing if the loop is not entered. */
2630 break;
2631
2632 case BC_FORL:
2633 rec_loop_interp(J, pc, rec_for(J, pc+((ptrdiff_t)rc-BCBIAS_J), 1));
2634 break;
2635 case BC_ITERL:
2636 rec_loop_interp(J, pc, rec_iterl(J, *pc));
2637 break;
2638 case BC_ITERN:
2639 rec_loop_interp(J, pc, rec_itern(J, ra, rb));
2640 break;
2641 case BC_LOOP:
2642 rec_loop_interp(J, pc, rec_loop(J, ra, 1));
2643 break;
2644
2645 case BC_JFORL:
2646 rec_loop_jit(J, rc, rec_for(J, pc+bc_j(traceref(J, rc)->startins), 1));
2647 break;
2648 case BC_JITERL:
2649 rec_loop_jit(J, rc, rec_iterl(J, traceref(J, rc)->startins));
2650 break;
2651 case BC_JLOOP:
2652 rec_loop_jit(J, rc, rec_loop(J, ra,
2653 !bc_isret(bc_op(traceref(J, rc)->startins)) &&
2654 bc_op(traceref(J, rc)->startins) != BC_ITERN));
2655 break;
2656
2657 case BC_IFORL:
2658 case BC_IITERL:
2659 case BC_ILOOP:
2660 case BC_IFUNCF:
2661 case BC_IFUNCV:
2662 lj_trace_err(J, LJ_TRERR_BLACKL);
2663 break;
2664
2665 case BC_JMP:
2666 if (ra < J->maxslot)
2667 J->maxslot = ra; /* Shrink used slots. */
2668 break;
2669
2670 case BC_ISNEXT:
2671 rec_isnext(J, ra);
2672 break;
2673
2674 /* -- Function headers -------------------------------------------------- */
2675
2676 case BC_FUNCF:
2677 rec_func_lua(J);
2678 break;
2679 case BC_JFUNCF:
2680 rec_func_jit(J, rc);
2681 break;
2682
2683 case BC_FUNCV:
2684 rec_func_vararg(J);
2685 rec_func_lua(J);
2686 break;
2687 case BC_JFUNCV:
2688 /* Cannot happen. No hotcall counting for varag funcs. */
2689 lj_assertJ(0, "unsupported vararg hotcall");
2690 break;
2691
2692 case BC_FUNCC:
2693 case BC_FUNCCW:
2694 lj_ffrecord_func(J);
2695 break;
2696
2697 default:
2698 if (op >= BC__MAX) {
2699 lj_ffrecord_func(J);
2700 break;
2701 }
2702 /* fallthrough */
2703 case BC_UCLO:
2704 case BC_FNEW:
2705 setintV(&J->errinfo, (int32_t)op);
2706 lj_trace_err_info(J, LJ_TRERR_NYIBC);
2707 break;
2708 }
2709
2710 /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */
2711 if (bcmode_a(op) == BCMdst && rc) {
2712 J->base[ra] = rc;
2713 if (ra >= J->maxslot) {
2714 #if LJ_FR2
2715 if (ra > J->maxslot) J->base[ra-1] = 0;
2716 #endif
2717 J->maxslot = ra+1;
2718 }
2719 }
2720
2721 #undef rav
2722 #undef rbv
2723 #undef rcv
2724
2725 /* Limit the number of recorded IR instructions and constants. */
2726 if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord] ||
2727 J->cur.nk < REF_BIAS-(IRRef)J->param[JIT_P_maxirconst])
2728 lj_trace_err(J, LJ_TRERR_TRACEOV);
2729 }
2730
2731 /* -- Recording setup ----------------------------------------------------- */
2732
2733 /* Setup recording for a root trace started by a hot loop. */
2734 static const BCIns *rec_setup_root(jit_State *J)
2735 {
2736 /* Determine the next PC and the bytecode range for the loop. */
2737 const BCIns *pcj, *pc = J->pc;
2738 BCIns ins = *pc;
2739 BCReg ra = bc_a(ins);
2740 switch (bc_op(ins)) {
2741 case BC_FORL:
2742 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2743 pc += 1+bc_j(ins);
2744 J->bc_min = pc;
2745 break;
2746 case BC_ITERL:
2747 if (bc_op(pc[-1]) == BC_JLOOP)
2748 lj_trace_err(J, LJ_TRERR_LINNER);
2749 lj_assertJ(bc_op(pc[-1]) == BC_ITERC, "no ITERC before ITERL");
2750 J->maxslot = ra + bc_b(pc[-1]) - 1;
2751 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2752 pc += 1+bc_j(ins);
2753 lj_assertJ(bc_op(pc[-1]) == BC_JMP, "ITERL does not point to JMP+1");
2754 J->bc_min = pc;
2755 break;
2756 case BC_ITERN:
2757 lj_assertJ(bc_op(pc[1]) == BC_ITERL, "no ITERL after ITERN");
2758 J->maxslot = ra;
2759 J->bc_extent = (MSize)(-bc_j(pc[1]))*sizeof(BCIns);
2760 J->bc_min = pc+2 + bc_j(pc[1]);
2761 J->state = LJ_TRACE_RECORD_1ST; /* Record the first ITERN, too. */
2762 break;
2763 case BC_LOOP:
2764 /* Only check BC range for real loops, but not for "repeat until true". */
2765 pcj = pc + bc_j(ins);
2766 ins = *pcj;
2767 if (bc_op(ins) == BC_JMP && bc_j(ins) < 0) {
2768 J->bc_min = pcj+1 + bc_j(ins);
2769 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2770 }
2771 J->maxslot = ra;
2772 pc++;
2773 break;
2774 case BC_RET:
2775 case BC_RET0:
2776 case BC_RET1:
2777 /* No bytecode range check for down-recursive root traces. */
2778 J->maxslot = ra + bc_d(ins) - 1;
2779 break;
2780 case BC_FUNCF:
2781 /* No bytecode range check for root traces started by a hot call. */
2782 J->maxslot = J->pt->numparams;
2783 pc++;
2784 break;
2785 case BC_CALLM:
2786 case BC_CALL:
2787 case BC_ITERC:
2788 /* No bytecode range check for stitched traces. */
2789 pc++;
2790 break;
2791 default:
2792 lj_assertJ(0, "bad root trace start bytecode %d", bc_op(ins));
2793 break;
2794 }
2795 return pc;
2796 }
2797
2798 /* Setup for recording a new trace. */
2799 void lj_record_setup(jit_State *J)
2800 {
2801 uint32_t i;
2802
2803 /* Initialize state related to current trace. */
2804 memset(J->slot, 0, sizeof(J->slot));
2805 memset(J->chain, 0, sizeof(J->chain));
2806 #ifdef LUAJIT_ENABLE_TABLE_BUMP
2807 memset(J->rbchash, 0, sizeof(J->rbchash));
2808 #endif
2809 memset(J->bpropcache, 0, sizeof(J->bpropcache));
2810 J->scev.idx = REF_NIL;
2811 setmref(J->scev.pc, NULL);
2812
2813 J->baseslot = 1+LJ_FR2; /* Invoking function is at base[-1-LJ_FR2]. */
2814 J->base = J->slot + J->baseslot;
2815 J->maxslot = 0;
2816 J->framedepth = 0;
2817 J->retdepth = 0;
2818
2819 J->instunroll = J->param[JIT_P_instunroll];
2820 J->loopunroll = J->param[JIT_P_loopunroll];
2821 J->tailcalled = 0;
2822 J->loopref = 0;
2823
2824 J->bc_min = NULL; /* Means no limit. */
2825 J->bc_extent = ~(MSize)0;
2826
2827 /* Emit instructions for fixed references. Also triggers initial IR alloc. */
2828 emitir_raw(IRT(IR_BASE, IRT_PGC), J->parent, J->exitno);
2829 for (i = 0; i <= 2; i++) {
2830 IRIns *ir = IR(REF_NIL-i);
2831 ir->i = 0;
2832 ir->t.irt = (uint8_t)(IRT_NIL+i);
2833 ir->o = IR_KPRI;
2834 ir->prev = 0;
2835 }
2836 J->cur.nk = REF_TRUE;
2837
2838 J->startpc = J->pc;
2839 setmref(J->cur.startpc, J->pc);
2840 if (J->parent) { /* Side trace. */
2841 GCtrace *T = traceref(J, J->parent);
2842 TraceNo root = T->root ? T->root : J->parent;
2843 J->cur.root = (uint16_t)root;
2844 J->cur.startins = BCINS_AD(BC_JMP, 0, 0);
2845 /* Check whether we could at least potentially form an extra loop. */
2846 if (J->exitno == 0 && T->snap[0].nent == 0) {
2847 /* We can narrow a FORL for some side traces, too. */
2848 if (J->pc > proto_bc(J->pt) && bc_op(J->pc[-1]) == BC_JFORI &&
2849 bc_d(J->pc[bc_j(J->pc[-1])-1]) == root) {
2850 lj_snap_add(J);
2851 rec_for_loop(J, J->pc-1, &J->scev, 1);
2852 goto sidecheck;
2853 }
2854 } else {
2855 J->startpc = NULL; /* Prevent forming an extra loop. */
2856 }
2857 lj_snap_replay(J, T);
2858 sidecheck:
2859 if ((traceref(J, J->cur.root)->nchild >= J->param[JIT_P_maxside] ||
2860 T->snap[J->exitno].count >= J->param[JIT_P_hotexit] +
2861 J->param[JIT_P_tryside])) {
2862 if (bc_op(*J->pc) == BC_JLOOP) {
2863 BCIns startins = traceref(J, bc_d(*J->pc))->startins;
2864 if (bc_op(startins) == BC_ITERN)
2865 rec_itern(J, bc_a(startins), bc_b(startins));
2866 }
2867 lj_record_stop(J, LJ_TRLINK_INTERP, 0);
2868 }
2869 } else { /* Root trace. */
2870 J->cur.root = 0;
2871 J->cur.startins = *J->pc;
2872 J->pc = rec_setup_root(J);
2873 /* Note: the loop instruction itself is recorded at the end and not
2874 ** at the start! So snapshot #0 needs to point to the *next* instruction.
2875 ** The one exception is BC_ITERN, which sets LJ_TRACE_RECORD_1ST.
2876 */
2877 lj_snap_add(J);
2878 if (bc_op(J->cur.startins) == BC_FORL)
2879 rec_for_loop(J, J->pc-1, &J->scev, 1);
2880 else if (bc_op(J->cur.startins) == BC_ITERC)
2881 J->startpc = NULL;
2882 if (1 + J->pt->framesize >= LJ_MAX_JSLOTS)
2883 lj_trace_err(J, LJ_TRERR_STACKOV);
2884 }
2885 #if LJ_HASPROFILE
2886 J->prev_pt = NULL;
2887 J->prev_line = -1;
2888 #endif
2889 #ifdef LUAJIT_ENABLE_CHECKHOOK
2890 /* Regularly check for instruction/line hooks from compiled code and
2891 ** exit to the interpreter if the hooks are set.
2892 **
2893 ** This is a compile-time option and disabled by default, since the
2894 ** hook checks may be quite expensive in tight loops.
2895 **
2896 ** Note this is only useful if hooks are *not* set most of the time.
2897 ** Use this only if you want to *asynchronously* interrupt the execution.
2898 **
2899 ** You can set the instruction hook via lua_sethook() with a count of 1
2900 ** from a signal handler or another native thread. Please have a look
2901 ** at the first few functions in luajit.c for an example (Ctrl-C handler).
2902 */
2903 {
2904 TRef tr = emitir(IRT(IR_XLOAD, IRT_U8),
2905 lj_ir_kptr(J, &J2G(J)->hookmask), IRXLOAD_VOLATILE);
2906 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (LUA_MASKLINE|LUA_MASKCOUNT)));
2907 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0));
2908 }
2909 #endif
2910 }
2911
2912 #undef IR
2913 #undef emitir_raw
2914 #undef emitir
2915
2916 #endif
LuaJIT is a Just-In-Time Compiler for the Lua programming language.