[php-src] master: Update IR

From: Date: Tue, 09 Apr 2024 19:42:09 +0000
Subject: [php-src] master: Update IR
Groups: php.cvs 
Request: Send a blank email to php-cvs+get-128005@lists.php.net to get a copy of this message
Author: Dmitry Stogov (dstogov) Date: 2024-04-09T22:41:33+03:00 Commit: https://github.com/php/php-src/commit/5708df71a7e9b0e3852ac1d4723e1c1113b98500 Raw diff: https://github.com/php/php-src/commit/5708df71a7e9b0e3852ac1d4723e1c1113b98500.diff Update IR IR commit: babd9980d84be8b804b16859ed52704e7895f0e5 Changed paths: M ext/opcache/jit/ir/ir_aarch64.dasc M ext/opcache/jit/ir/ir_emit.c M ext/opcache/jit/ir/ir_private.h M ext/opcache/jit/ir/ir_ra.c M ext/opcache/jit/ir/ir_x86.dasc Diff: diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 663482022f791..4f0ac2ee69796 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -318,6 +318,8 @@ const char *ir_reg_name(int8_t reg, ir_type type) #define IR_RULE_ENUM(name) IR_ ## name, +#define IR_STATIC_ALLOCA (IR_SKIPPED | IR_FUSED | IR_SIMPLE | IR_ALLOCA) + enum _ir_rule { IR_FIRST_RULE = IR_LAST_OP, IR_RULES(IR_RULE_ENUM) @@ -348,6 +350,9 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co if (IR_IS_CONST_REF(insn->op1)) { constraints->tmp_regs[n] = IR_TMP_REG(1, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n++; + } else if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[n] = IR_TMP_REG(1, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; } if (IR_IS_CONST_REF(insn->op2) && insn->op1 != insn->op2) { const ir_insn *val_insn = &ctx->ir_base[insn->op2]; @@ -380,6 +385,9 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co n++; break; } + } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[n] = IR_TMP_REG(2, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; } if (insn->op == IR_MOD) { constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); @@ -475,6 +483,9 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co if (IR_IS_CONST_REF(insn->op1)) { constraints->tmp_regs[n] = IR_TMP_REG(1, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n++; + } else if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[0] = IR_TMP_REG(1, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n = 1; } if (IR_IS_CONST_REF(insn->op2) && insn->op1 != insn->op2) { insn = &ctx->ir_base[insn->op2]; @@ -482,6 +493,9 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n++; } + } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[n] = IR_TMP_REG(2, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; } break; case IR_CMP_FP: @@ -530,6 +544,9 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n++; } + } else if (ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[n] = IR_TMP_REG(3, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; } break; case IR_SWITCH: @@ -624,17 +641,17 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co flags = 0; break; case IR_VA_START: - flags = IR_OP1_MUST_BE_IN_REG; + flags = IR_OP2_MUST_BE_IN_REG; constraints->tmp_regs[0] = IR_TMP_REG(3, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n = 1; break; case IR_VA_ARG: - flags = IR_USE_MUST_BE_IN_REG | IR_OP1_MUST_BE_IN_REG; + flags = IR_USE_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG; constraints->tmp_regs[0] = IR_TMP_REG(3, IR_ADDR, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n = 1; break; case IR_VA_COPY: - flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG; + flags = IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG; constraints->tmp_regs[0] = IR_TMP_REG(1, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n = 1; break; @@ -902,7 +919,7 @@ binop_fp: return insn->op; } case IR_CALL: - ctx->flags2 |= IR_HAS_CALLS; + ctx->flags2 |= IR_HAS_CALLS | IR_16B_FRAME_ALIGNMENT; return IR_CALL; case IR_VAR: return IR_SKIPPED | IR_VAR; @@ -910,6 +927,13 @@ binop_fp: return ctx->use_lists[ref].count > 0 ? IR_PARAM : IR_SKIPPED | IR_PARAM; case IR_ALLOCA: if (ctx->flags & IR_FUNCTION) { + if (IR_IS_CONST_REF(insn->op2) && ctx->cfg_map[ref] == 1) { + ir_insn *val = &ctx->ir_base[insn->op2]; + + if (!IR_IS_SYM_CONST(val->op)) { + return IR_STATIC_ALLOCA; + } + } ctx->flags |= IR_USE_FRAME_POINTER; ctx->flags2 |= IR_HAS_ALLOCA; } @@ -1348,6 +1372,23 @@ static void ir_emit_load_mem(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem) } } +static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + int32_t offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[src].op3); + + IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA); + if (aarch64_may_encode_imm12(offset)) { + | add Rx(reg), Rx(base), #offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | add sp, sp, Rx(IR_REG_INT_TMP) + } +} + + static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) { if (IR_IS_CONST_REF(src)) { @@ -1370,9 +1411,11 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) } else { ir_emit_load_imm_fp(ctx, type, reg, src); } - } else { + } else if (ctx->vregs[src]) { ir_mem mem = ir_ref_spill_slot(ctx, src); ir_emit_load_mem(ctx, type, reg, mem); + } else { + ir_load_local_addr(ctx, reg, src); } } @@ -1752,12 +1795,12 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -1900,11 +1943,11 @@ static void ir_emit_min_max_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE && op2_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, op2); } @@ -2055,7 +2098,7 @@ static void ir_emit_reg_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) break; } } else { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, op2); } @@ -2097,7 +2140,7 @@ static void ir_emit_mul_div_mod_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op)); IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } @@ -2138,12 +2181,13 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op)); IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE && def_reg != op1_reg); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } if (op2_reg != IR_REG_NONE) { + op2_reg = IR_REG_NUM(op2_reg); ir_emit_load_imm_int(ctx, type, op2_reg, offset); } @@ -2234,7 +2278,7 @@ static void ir_emit_shift(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg tmp_reg; IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE && op2_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(insn->op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, insn->op1); } @@ -2343,7 +2387,7 @@ static void ir_emit_shift_const(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op)); IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } @@ -2424,7 +2468,7 @@ static void ir_emit_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } @@ -2483,7 +2527,7 @@ static void ir_emit_ctpop(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } @@ -2533,7 +2577,7 @@ static void ir_emit_op_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } @@ -2570,11 +2614,11 @@ static void ir_emit_binop_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg op2_reg = ctx->regs[def][2]; IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE && op2_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -2683,7 +2727,7 @@ static void ir_emit_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg op2_reg = ctx->regs[def][2]; IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } @@ -2694,9 +2738,6 @@ static void ir_emit_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_load(ctx, type, op2_reg, op2); } } - if (IR_IS_CONST_REF(op2)) { - ir_emit_load(ctx, type, op2_reg, op2); - } } if (IR_IS_CONST_REF(insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op) @@ -2785,11 +2826,11 @@ static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref cmp_ref, ir_insn *cmp_ins } IR_ASSERT(op1_reg != IR_REG_NONE && op2_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -3025,9 +3066,6 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } - if (op1_reg != IR_REG_NONE && IR_IS_CONST_REF(op1)) { - ir_emit_load(ctx, type, op1_reg, op1); - } if (op2_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); @@ -3035,9 +3073,6 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i ir_emit_load(ctx, type, op2_reg, op2); } } - if (IR_IS_CONST_REF(op2)) { - ir_emit_load(ctx, type, op2_reg, op2); - } } if (IR_IS_CONST_REF(op2) && !IR_IS_SYM_CONST(ctx->ir_base[op2].op) @@ -3091,9 +3126,17 @@ static void ir_emit_if_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, u } } return; + } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + uint32_t true_block, false_block; + + ir_get_true_false_blocks(ctx, b, &true_block, &false_block); + if (true_block != next_block) { + | b =>true_block + } + return; } IR_ASSERT(op2_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(insn->op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, insn->op2); } @@ -3117,7 +3160,7 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, op2); if (op1 == op2) { @@ -3127,14 +3170,14 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) op3_reg = op2_reg; } } - if (op3 != op2 && (IR_REG_SPILLED(op3_reg) || IR_IS_CONST_REF(op3))) { + if (op3 != op2 && IR_REG_SPILLED(op3_reg)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, op3); if (op1 == op2) { op1_reg = op3_reg; } } - if (op1 != op2 && op1 != op3 && (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1))) { + if (op1 != op2 && op1 != op3 && IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, op1_type, op1_reg, op1); } @@ -3217,7 +3260,7 @@ static void ir_emit_sext(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(IR_IS_TYPE_INT(dst_type)); IR_ASSERT(ir_type_size[dst_type] > ir_type_size[src_type]); IR_ASSERT(def_reg != IR_REG_NONE); - if ((op1_reg != IR_REG_NONE) && (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(insn->op1))) { + if ((op1_reg != IR_REG_NONE) && IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, src_type, op1_reg, insn->op1); } @@ -3290,7 +3333,7 @@ static void ir_emit_zext(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(IR_IS_TYPE_INT(dst_type)); IR_ASSERT(ir_type_size[dst_type] > ir_type_size[src_type]); IR_ASSERT(def_reg != IR_REG_NONE); - if ((op1_reg != IR_REG_NONE) && (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(insn->op1))) { + if ((op1_reg != IR_REG_NONE) && IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, src_type, op1_reg, insn->op1); } @@ -3466,7 +3509,7 @@ static void ir_emit_int2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(IR_IS_TYPE_INT(src_type)); IR_ASSERT(IR_IS_TYPE_FP(dst_type)); IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(insn->op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, src_type, op1_reg, insn->op1); } @@ -3531,7 +3574,7 @@ static void ir_emit_fp2int(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(IR_IS_TYPE_FP(src_type)); IR_ASSERT(IR_IS_TYPE_INT(dst_type)); IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(insn->op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, src_type, op1_reg, insn->op1); } @@ -3585,7 +3628,7 @@ static void ir_emit_fp2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(IR_IS_TYPE_FP(src_type)); IR_ASSERT(IR_IS_TYPE_FP(dst_type)); IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(insn->op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, src_type, op1_reg, insn->op1); } @@ -3715,7 +3758,7 @@ static void ir_emit_vstore(ir_ctx *ctx, ir_ref ref, ir_insn *insn) if (IR_REG_SPILLED(op3_reg) && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) { return; // fake store } - if (IR_REG_SPILLED(op3_reg) || IR_IS_CONST_REF(insn->op3)) { + if (IR_REG_SPILLED(op3_reg)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, insn->op3); } @@ -3727,20 +3770,34 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) { ir_insn *addr_insn = &ctx->ir_base[ref]; ir_reg reg; + int32_t offset; - IR_ASSERT(addr_insn->op == IR_ADD); - IR_ASSERT(!IR_IS_CONST_REF(addr_insn->op1) && IR_IS_CONST_REF(addr_insn->op2)); - IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[addr_insn->op2].op)); - if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { - reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 1); + if (addr_insn->op == IR_ADD) { + IR_ASSERT(!IR_IS_CONST_REF(addr_insn->op1) && IR_IS_CONST_REF(addr_insn->op2)); + IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[addr_insn->op2].op)); + if (ir_rule(ctx, addr_insn->op1) == IR_STATIC_ALLOCA) { + reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[addr_insn->op1].op3); + offset += ctx->ir_base[addr_insn->op2].val.i32; + return IR_MEM_BO(reg, offset); + } else { + if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { + reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 1); + } else { + reg = ctx->regs[ref][1]; + } + if (IR_REG_SPILLED(reg)) { + reg = IR_REG_NUM(reg); + ir_emit_load(ctx, IR_ADDR, reg, addr_insn->op1); + } + return IR_MEM_BO(reg, ctx->ir_base[addr_insn->op2].val.i32); + } } else { - reg = ctx->regs[ref][1]; + IR_ASSERT(addr_insn->op == IR_ALLOCA); + reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[ref].op3); + return IR_MEM_BO(reg, offset); } - if (IR_REG_SPILLED(reg)) { - reg = IR_REG_NUM(reg); - ir_emit_load(ctx, IR_ADDR, reg, addr_insn->op1); - } - return IR_MEM_BO(reg, ctx->ir_base[addr_insn->op2].val.i32); } static void ir_emit_load_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) @@ -3748,15 +3805,27 @@ static void ir_emit_load_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_ref type = insn->type; ir_reg op2_reg = ctx->regs[def][2]; ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_mem mem; if (ctx->use_lists[def].count == 1) { /* dead load */ return; } IR_ASSERT(def_reg != IR_REG_NONE); - if (!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)) { - ir_mem mem = ir_fuse_addr(ctx, def, insn->op2); - + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + mem = IR_MEM_B(op2_reg); + } else if (IR_IS_CONST_REF(insn->op2)) { + op2_reg = def_reg; + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + mem = IR_MEM_B(op2_reg); + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED); + mem = ir_fuse_addr(ctx, def, insn->op2); if (IR_REG_SPILLED(ctx->regs[def][0]) && ir_is_same_spill_slot(ctx, def, mem)) { if (!ir_may_avoid_spill_load(ctx, def, def)) { ir_emit_load_mem_int(ctx, type, def_reg, mem); @@ -3764,18 +3833,8 @@ static void ir_emit_load_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) /* avoid load to the same location (valid only when register is not reused) */ return; } - ir_emit_load_mem_int(ctx, type, def_reg, mem); - } else { - if (op2_reg == IR_REG_NONE) { - op2_reg = def_reg; - } - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(insn->op2)) { - op2_reg = IR_REG_NUM(op2_reg); - IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); - } - ir_emit_load_mem_int(ctx, type, def_reg, IR_MEM_B(op2_reg)); } + ir_emit_load_mem_int(ctx, type, def_reg, mem); if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, type, def, def_reg); } @@ -3786,15 +3845,27 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_ref type = insn->type; ir_reg op2_reg = ctx->regs[def][2]; ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_mem mem; if (ctx->use_lists[def].count == 1) { /* dead load */ return; } IR_ASSERT(def_reg != IR_REG_NONE); - if (!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)) { - ir_mem mem = ir_fuse_addr(ctx, def, insn->op2); - + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + mem = IR_MEM_B(op2_reg); + } else if (IR_IS_CONST_REF(insn->op2)) { + op2_reg = def_reg; + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + mem = IR_MEM_B(op2_reg); + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED); + mem = ir_fuse_addr(ctx, def, insn->op2); if (IR_REG_SPILLED(ctx->regs[def][0]) && ir_is_same_spill_slot(ctx, def, mem)) { if (!ir_may_avoid_spill_load(ctx, def, def)) { ir_emit_load_mem_fp(ctx, type, def_reg, mem); @@ -3802,19 +3873,8 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) /* avoid load to the same location (valid only when register is not reused) */ return; } - ir_emit_load_mem_fp(ctx, type, def_reg, mem); - } else { - if (op2_reg != IR_REG_NONE && (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(insn->op2))) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); - } - if (op2_reg == IR_REG_NONE) { - op2_reg = def_reg; - IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); - } - ir_emit_load_mem_fp(ctx, type, def_reg, IR_MEM_B(op2_reg)); } + ir_emit_load_mem_fp(ctx, type, def_reg, mem); if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, type, def, def_reg); } @@ -3826,11 +3886,22 @@ static void ir_emit_store_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) ir_ref type = val_insn->type; ir_reg op2_reg = ctx->regs[ref][2]; ir_reg op3_reg = ctx->regs[ref][3]; + ir_mem mem; - if (!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)) { - ir_mem mem = ir_fuse_addr(ctx, ref, insn->op2); - - if (!IR_IS_CONST_REF(insn->op3) && IR_REG_SPILLED(op3_reg) && ir_is_same_spill_slot(ctx, insn->op3, mem)) { + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + mem = IR_MEM_B(op2_reg); + } else { + IR_ASSERT(!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)); + mem = ir_fuse_addr(ctx, ref, insn->op2); + if (!IR_IS_CONST_REF(insn->op3) + && IR_REG_SPILLED(op3_reg) + && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA + && ir_is_same_spill_slot(ctx, insn->op3, mem)) { if (!ir_may_avoid_spill_load(ctx, insn->op3, ref)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, insn->op3); @@ -3838,30 +3909,18 @@ static void ir_emit_store_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) /* avoid store to the same location */ return; } - if (op3_reg == IR_REG_NONE) { - IR_ASSERT(IR_IS_CONST_REF(insn->op3) && !IR_IS_SYM_CONST(ctx->ir_base[insn->op3].op) && ctx->ir_base[insn->op3].val.i64 == 0); - op3_reg = IR_REG_ZR; - } else if (IR_REG_SPILLED(op3_reg) || IR_IS_CONST_REF(insn->op3)) { + } + + if (op3_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op3_reg)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, insn->op3); } - ir_emit_store_mem_int(ctx, type, mem, op3_reg); } else { - IR_ASSERT(op2_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(insn->op2)) { - op2_reg = IR_REG_NUM(op2_reg); - IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); - } - if (op3_reg == IR_REG_NONE) { - IR_ASSERT(IR_IS_CONST_REF(insn->op3) && !IR_IS_SYM_CONST(ctx->ir_base[insn->op3].op) && ctx->ir_base[insn->op3].val.i64 == 0); - op3_reg = IR_REG_ZR; - } else if (IR_REG_SPILLED(op3_reg) || IR_IS_CONST_REF(insn->op3)) { - op3_reg = IR_REG_NUM(op3_reg); - ir_emit_load(ctx, type, op3_reg, insn->op3); - } - ir_emit_store_mem_int(ctx, type, IR_MEM_B(op2_reg), op3_reg); + IR_ASSERT(IR_IS_CONST_REF(insn->op3) && !IR_IS_SYM_CONST(ctx->ir_base[insn->op3].op) && ctx->ir_base[insn->op3].val.i64 == 0); + op3_reg = IR_REG_ZR; } + ir_emit_store_mem_int(ctx, type, mem, op3_reg); } static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) @@ -3869,12 +3928,23 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) ir_ref type = ctx->ir_base[insn->op3].type; ir_reg op2_reg = ctx->regs[ref][2]; ir_reg op3_reg = ctx->regs[ref][3]; + ir_mem mem; IR_ASSERT(op3_reg != IR_REG_NONE); - if (!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)) { - ir_mem mem = ir_fuse_addr(ctx, ref, insn->op2); - - if (!IR_IS_CONST_REF(insn->op3) && IR_REG_SPILLED(op3_reg) && ir_is_same_spill_slot(ctx, insn->op3, mem)) { + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + mem = IR_MEM_B(op2_reg); + } else { + IR_ASSERT(!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)); + mem = ir_fuse_addr(ctx, ref, insn->op2); + if (!IR_IS_CONST_REF(insn->op3) + && IR_REG_SPILLED(op3_reg) + && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA + && ir_is_same_spill_slot(ctx, insn->op3, mem)) { if (!ir_may_avoid_spill_load(ctx, insn->op3, ref)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, insn->op3); @@ -3882,23 +3952,13 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) /* avoid store to the same location */ return; } - if (IR_REG_SPILLED(op3_reg) || IR_IS_CONST_REF(insn->op3)) { - op3_reg = IR_REG_NUM(op3_reg); - ir_emit_load(ctx, type, op3_reg, insn->op3); - } - ir_emit_store_mem_fp(ctx, type, mem, op3_reg); - } else { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(insn->op2)) { - op2_reg = IR_REG_NUM(op2_reg); - IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); - } - if (IR_REG_SPILLED(op3_reg) || IR_IS_CONST_REF(insn->op3)) { - op3_reg = IR_REG_NUM(op3_reg); - ir_emit_load(ctx, type, op3_reg, insn->op3); - } - ir_emit_store_mem_fp(ctx, type, IR_MEM_B(op2_reg), op3_reg); } + + if (IR_REG_SPILLED(op3_reg)) { + op3_reg = IR_REG_NUM(op3_reg); + ir_emit_load(ctx, type, op3_reg, insn->op3); + } + ir_emit_store_mem_fp(ctx, type, mem, op3_reg); } static void ir_emit_rload(ir_ctx *ctx, ir_ref def, ir_insn *insn) @@ -3981,7 +4041,7 @@ static void ir_emit_alloca(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(!IR_IS_SYM_CONST(val->op)); IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); - if (ctx->flags2 & IR_HAS_CALLS) { + if (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) { /* Stack must be 16 byte aligned */ size = IR_ALIGNED_SIZE(size, 16); } else { @@ -3997,7 +4057,7 @@ static void ir_emit_alloca(ir_ctx *ctx, ir_ref def, ir_insn *insn) ctx->call_stack_size += size; } } else { - int32_t alignment = (ctx->flags2 & IR_HAS_CALLS) ? 16 : 8; + int32_t alignment = (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) ? 16 : 8; ir_reg op2_reg = ctx->regs[def][2]; ir_type type = ctx->ir_base[insn->op2].type; @@ -4035,7 +4095,7 @@ static void ir_emit_afree(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(!IR_IS_SYM_CONST(val->op)); IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 > 0); - if (ctx->flags2 & IR_HAS_CALLS) { + if (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) { /* Stack must be 16 byte aligned */ size = IR_ALIGNED_SIZE(size, 16); } else { @@ -4046,7 +4106,7 @@ static void ir_emit_afree(ir_ctx *ctx, ir_ref def, ir_insn *insn) ctx->call_stack_size -= size; } } else { -// int32_t alignment = (ctx->flags2 & IR_HAS_CALLS) ? 16 : 8; +// int32_t alignment = (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) ? 16 : 8; ir_reg op2_reg = ctx->regs[def][2]; ir_type type = ctx->ir_base[insn->op2].type; @@ -4116,11 +4176,19 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) int arg_area_offset; ir_reg op2_reg = ctx->regs[def][2]; ir_reg tmp_reg = ctx->regs[def][3]; + int32_t offset; IR_ASSERT(tmp_reg != IR_REG_NONE); - if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -4131,13 +4199,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) arg_area_offset = ctx->call_stack_size + ctx->stack_frame_size + ctx->param_stack_size; } | add Rx(tmp_reg), Rx(fp), #arg_area_offset - if (op2_reg != IR_REG_NONE) { - | str Rx(tmp_reg), [Rx(op2_reg)] - } else { - int32_t offset = ir_ref_spill_slot_offset(ctx, insn->op2, &op2_reg); - - | str Rx(tmp_reg), [Rx(op2_reg), #offset] - } + | str Rx(tmp_reg), [Rx(op2_reg), #offset] #else ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; @@ -4146,11 +4208,19 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) int overflow_arg_area_offset; ir_reg op2_reg = ctx->regs[def][2]; ir_reg tmp_reg = ctx->regs[def][3]; + int32_t offset; - IR_ASSERT(op2_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + IR_ASSERT(tmp_reg != IR_REG_NONE); + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -4165,20 +4235,20 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) /* Set va_list.stack */ | add Rx(tmp_reg), Rx(fp), #overflow_arg_area_offset - | str Rx(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, stack)] + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, stack))] if ((ctx->flags2 & (IR_HAS_VA_ARG_GP|IR_HAS_VA_COPY)) && ctx->gp_reg_params < IR_REG_INT_ARGS) { reg_save_area_offset += sizeof(void*) * IR_REG_INT_ARGS; /* Set va_list.gr_top */ if (overflow_arg_area_offset != reg_save_area_offset) { | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset } - | str Rx(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, gr_top)] + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, gr_top))] /* Set va_list.gr_offset */ | movn Rw(tmp_reg), #~(0 - (sizeof(void*) * (IR_REG_INT_ARGS - ctx->gp_reg_params))) - | str Rw(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, gr_offset)] + | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, gr_offset))] } else { /* Set va_list.gr_offset */ - | str wzr, [Rx(op2_reg), #offsetof(ir_va_list, gr_offset)] + | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_va_list, gr_offset))] } if ((ctx->flags2 & (IR_HAS_VA_ARG_FP|IR_HAS_VA_COPY)) && ctx->fp_reg_params < IR_REG_FP_ARGS) { reg_save_area_offset += 16 * IR_REG_FP_ARGS; @@ -4186,13 +4256,13 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (overflow_arg_area_offset != reg_save_area_offset) { | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset } - | str Rx(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, vr_top)] + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, vr_top))] /* Set va_list.vr_offset */ | movn Rw(tmp_reg), #~(0 - (16 * (IR_REG_FP_ARGS - ctx->fp_reg_params))) - | str Rw(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, vr_offset)] + | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, vr_offset))] } else { /* Set va_list.vr_offset */ - | str wzr, [Rx(op2_reg), #offsetof(ir_va_list, vr_offset)] + | str wzr, [Rx(op2_reg), #(offset+offsetof(ir_va_list, vr_offset))] } #endif } @@ -4205,42 +4275,72 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg tmp_reg = ctx->regs[def][1]; ir_reg op2_reg = ctx->regs[def][2]; ir_reg op3_reg = ctx->regs[def][3]; + int32_t op2_offset, op3_offset; - IR_ASSERT(tmp_reg != IR_REG_NONE && op2_reg != IR_REG_NONE && op3_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + IR_ASSERT(tmp_reg != IR_REG_NONE); + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + op2_offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + op2_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } - if (IR_REG_SPILLED(op3_reg)) { - op3_reg = IR_REG_NUM(op3_reg); - ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3); + if (op3_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op3_reg)) { + op3_reg = IR_REG_NUM(op3_reg); + ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3); + } + op3_offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); + op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + op3_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op3].op3); } - | ldr Rx(tmp_reg), [Rx(op3_reg)] - | str Rx(tmp_reg), [Rx(op2_reg)] + | ldr Rx(tmp_reg), [Rx(op3_reg), #op3_offset] + | str Rx(tmp_reg), [Rx(op2_reg), #op2_offset] #else ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; ir_reg tmp_reg = ctx->regs[def][1]; ir_reg op2_reg = ctx->regs[def][2]; ir_reg op3_reg = ctx->regs[def][3]; + int32_t op2_offset, op3_offset; - IR_ASSERT(tmp_reg != IR_REG_NONE && op2_reg != IR_REG_NONE && op3_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + IR_ASSERT(tmp_reg != IR_REG_NONE); + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + op2_offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + op2_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } - if (IR_REG_SPILLED(op3_reg)) { - op3_reg = IR_REG_NUM(op3_reg); - ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3); - } - | ldr Rx(tmp_reg), [Rx(op3_reg)] - | str Rx(tmp_reg), [Rx(op2_reg)] - | ldr Rx(tmp_reg), [Rx(op3_reg), #8] - | str Rx(tmp_reg), [Rx(op2_reg), #8] - | ldr Rx(tmp_reg), [Rx(op3_reg), #16] - | str Rx(tmp_reg), [Rx(op2_reg), #16] - | ldr Rx(tmp_reg), [Rx(op3_reg), #24] - | str Rx(tmp_reg), [Rx(op2_reg), #24] + if (op3_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op3_reg)) { + op3_reg = IR_REG_NUM(op3_reg); + ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3); + } + op3_offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); + op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + op3_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op3].op3); + } + | ldr Rx(tmp_reg), [Rx(op3_reg), #op3_offset] + | str Rx(tmp_reg), [Rx(op2_reg), #op2_offset] + | ldr Rx(tmp_reg), [Rx(op3_reg), #(op3_offset+8)] + | str Rx(tmp_reg), [Rx(op2_reg), #(op2_offset+8)] + | ldr Rx(tmp_reg), [Rx(op3_reg), #(op3_offset+16)] + | str Rx(tmp_reg), [Rx(op2_reg), #(op2_offset+16)] + | ldr Rx(tmp_reg), [Rx(op3_reg), #(op3_offset+24)] + | str Rx(tmp_reg), [Rx(op2_reg), #(op2_offset+24)] #endif } @@ -4253,22 +4353,24 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg def_reg = ctx->regs[def][0]; ir_reg op2_reg = ctx->regs[def][2]; ir_reg tmp_reg = ctx->regs[def][3]; + int32_t offset; IR_ASSERT(def_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); - if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); - } - | ldr Rx(tmp_reg), [Rx(op2_reg)] - ir_emit_load_mem(ctx, type, def_reg, IR_MEM_BO(tmp_reg, 0)); - | add Rx(tmp_reg), Rx(tmp_reg), #IR_MAX(ir_type_size[type], sizeof(void*)) if (op2_reg != IR_REG_NONE) { - | str Rx(tmp_reg), [Rx(op2_reg)] + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + offset = 0; } else { - int32_t offset = ir_ref_spill_slot_offset(ctx, insn->op2, &op2_reg); - - | str Rx(tmp_reg), [Rx(op2_reg), #offset] + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } + | ldr Rx(tmp_reg), [Rx(op2_reg), #offset] + ir_emit_load_mem(ctx, type, def_reg, IR_MEM_BO(tmp_reg, 0)); + | add Rx(tmp_reg), Rx(tmp_reg), #IR_MAX(ir_type_size[type], sizeof(void*)) + | str Rx(tmp_reg), [Rx(op2_reg), #offset] if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, type, def, def_reg); } @@ -4279,45 +4381,53 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg def_reg = ctx->regs[def][0]; ir_reg op2_reg = ctx->regs[def][2]; ir_reg tmp_reg = ctx->regs[def][3]; + int32_t offset; - IR_ASSERT(def_reg != IR_REG_NONE && op2_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + IR_ASSERT(def_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } if (IR_IS_TYPE_INT(type)) { - | ldr Rw(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, gr_offset)] + | ldr Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, gr_offset))] | cmp Rw(tmp_reg), wzr | bge >1 - | ldr Rx(IR_REG_INT_TMP), [Rx(op2_reg), #offsetof(ir_va_list, gr_top)] + | ldr Rx(IR_REG_INT_TMP), [Rx(op2_reg), #(offset+offsetof(ir_va_list, gr_top))] | sxtw Rx(tmp_reg), Rw(tmp_reg) | add Rx(IR_REG_INT_TMP), Rx(tmp_reg), Rx(IR_REG_INT_TMP) | ldr Rx(def_reg), [Rx(IR_REG_INT_TMP)] | add Rw(tmp_reg), Rw(tmp_reg), #sizeof(void*) - | str Rw(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, gr_offset)] + | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, gr_offset))] | b >2 |1: - | ldr Rx(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, stack)] + | ldr Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, stack))] | ldr Rx(def_reg), [Rx(tmp_reg)] | add Rx(tmp_reg), Rx(tmp_reg), #sizeof(void*) - | str Rx(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, stack)] + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, stack))] |2: } else { - | ldr Rw(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, vr_offset)] + | ldr Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, vr_offset))] | cmp Rw(tmp_reg), wzr | bge >1 - | ldr Rx(IR_REG_INT_TMP), [Rx(op2_reg), #offsetof(ir_va_list, vr_top)] + | ldr Rx(IR_REG_INT_TMP), [Rx(op2_reg), #(offset+offsetof(ir_va_list, vr_top))] | sxtw Rx(tmp_reg), Rw(tmp_reg) | add Rx(IR_REG_INT_TMP), Rx(tmp_reg), Rx(IR_REG_INT_TMP) | ldr Rd(def_reg-IR_REG_FP_FIRST), [Rx(IR_REG_INT_TMP)] | add Rw(tmp_reg), Rw(tmp_reg), #16 - | str Rw(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, vr_offset)] + | str Rw(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, vr_offset))] | b >2 |1: - | ldr Rx(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, stack)] + | ldr Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, stack))] | ldr Rd(def_reg-IR_REG_FP_FIRST), [Rx(tmp_reg)] | add Rx(tmp_reg), Rx(tmp_reg), #sizeof(void*) - | str Rx(tmp_reg), [Rx(op2_reg), #offsetof(ir_va_list, stack)] + | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, stack))] |2: } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -4381,8 +4491,6 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, insn->op2); - } else if (IR_IS_CONST_REF(insn->op2)) { - ir_emit_load(ctx, type, op2_reg, insn->op2); } /* Generate a table jmp or a sequence of calls */ @@ -4626,7 +4734,7 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, ir_reg fp_param++; } if (dst_reg != IR_REG_NONE) { - if (IR_IS_CONST_REF(arg) || src_reg == IR_REG_NONE) { + if (src_reg == IR_REG_NONE) { /* delay CONST->REG and MEM->REG moves to third pass */ do_pass3 = 1; } else { @@ -4692,7 +4800,7 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, ir_reg fp_param++; } if (dst_reg != IR_REG_NONE) { - if (IR_IS_CONST_REF(arg) || src_reg == IR_REG_NONE) { + if (src_reg == IR_REG_NONE) { if (IR_IS_CONST_REF(arg) && IR_IS_TYPE_INT(type)) { if (ir_type_size[type] == 1) { type = IR_ADDR; @@ -4702,7 +4810,7 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, ir_reg } } else { if (IR_IS_TYPE_INT(type)) { - if (IR_IS_CONST_REF(arg) || src_reg == IR_REG_NONE) { + if (src_reg == IR_REG_NONE) { IR_ASSERT(tmp_reg != IR_REG_NONE); ir_emit_load(ctx, type, tmp_reg, arg); if (IR_IS_CONST_REF(arg)) { @@ -4715,10 +4823,7 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, ir_reg ir_emit_store_mem_int(ctx, type, IR_MEM_BO(IR_REG_STACK_POINTER, stack_offset), src_reg); } } else { - if (IR_IS_CONST_REF(arg)) { - ir_emit_load(ctx, type, tmp_fp_reg, arg); - ir_emit_store_mem_fp(ctx, IR_DOUBLE, IR_MEM_BO(IR_REG_STACK_POINTER, stack_offset), tmp_fp_reg); - } else if (src_reg == IR_REG_NONE) { + if (src_reg == IR_REG_NONE) { IR_ASSERT(tmp_fp_reg != IR_REG_NONE); ir_emit_load(ctx, type, tmp_fp_reg, arg); ir_emit_store_mem_fp(ctx, IR_DOUBLE, IR_MEM_BO(IR_REG_STACK_POINTER, stack_offset), tmp_fp_reg); @@ -5022,11 +5127,11 @@ static void ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn * ir_reg op2_reg = ctx->regs[insn->op2][2]; void *addr; - if (op1_reg != IR_REG_NONE && (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1))) { + if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } - if (op2_reg != IR_REG_NONE && (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2))) { + if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -5545,7 +5650,13 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) n--; if (constraints.tmp_regs[n].type) { ir_reg reg = ir_get_free_reg(constraints.tmp_regs[n].type, available); + ir_ref *ops = insn->ops; IR_REGSET_EXCL(available, reg); + if (constraints.tmp_regs[n].num > 0 + && IR_IS_CONST_REF(ops[constraints.tmp_regs[n].num])) { + /* rematerialization */ + reg |= IR_REG_SPILL_LOAD; + } ctx->regs[i][constraints.tmp_regs[n].num] = reg; } else if (constraints.tmp_regs[n].reg == IR_REG_SCRATCH) { available = IR_REGSET_DIFFERENCE(available, IR_REGSET_SCRATCH); @@ -5649,12 +5760,12 @@ void ir_fix_stack_frame(ir_ctx *ctx) ctx->stack_frame_alignment = 0; ctx->call_stack_size = 0; - if ((ctx->flags2 & IR_HAS_CALLS) && !(ctx->flags & IR_FUNCTION)) { + if ((ctx->flags2 & IR_16B_FRAME_ALIGNMENT) && !(ctx->flags & IR_FUNCTION)) { while (IR_ALIGNED_SIZE(ctx->stack_frame_size, 16) != ctx->stack_frame_size) { ctx->stack_frame_size += sizeof(void*); ctx->stack_frame_alignment += sizeof(void*); } - } else if (ctx->flags2 & IR_HAS_CALLS) { + } else if (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) { ctx->flags |= IR_USE_FRAME_POINTER; /* Stack must be 16 byte aligned */ if (!(ctx->flags & IR_FUNCTION)) { diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index a3630da62a4d6..c117ba5becb0c 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -623,7 +623,13 @@ static void ir_emit_dessa_move(ir_ctx *ctx, ir_type type, ir_ref to, ir_ref from IR_ASSERT(from != to); if (to < IR_REG_NUM) { if (IR_IS_CONST_REF(from)) { - ir_emit_load(ctx, type, to, from); + if (-from < ctx->consts_count) { + /* constant reference */ + ir_emit_load(ctx, type, to, from); + } else { + /* local variable address */ + ir_load_local_addr(ctx, to, -from - ctx->consts_count); + } } else if (from < IR_REG_NUM) { if (IR_IS_TYPE_INT(type)) { ir_emit_mov(ctx, type, to, from); @@ -637,18 +643,27 @@ static void ir_emit_dessa_move(ir_ctx *ctx, ir_type type, ir_ref to, ir_ref from } else { mem_to = ir_vreg_spill_slot(ctx, to - IR_REG_NUM); if (IR_IS_CONST_REF(from)) { + if (-from < ctx->consts_count) { + /* constant reference */ #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) - if (IR_IS_TYPE_INT(type) - && !IR_IS_SYM_CONST(ctx->ir_base[from].op) - && (ir_type_size[type] != 8 || IR_IS_SIGNED_32BIT(ctx->ir_base[from].val.i64))) { - ir_emit_store_mem_imm(ctx, type, mem_to, ctx->ir_base[from].val.i32); - return; - } + if (IR_IS_TYPE_INT(type) + && !IR_IS_SYM_CONST(ctx->ir_base[from].op) + && (ir_type_size[type] != 8 || IR_IS_SIGNED_32BIT(ctx->ir_base[from].val.i64))) { + ir_emit_store_mem_imm(ctx, type, mem_to, ctx->ir_base[from].val.i32); + return; + } #endif - ir_reg tmp = IR_IS_TYPE_INT(type) ? tmp_reg : tmp_fp_reg; - IR_ASSERT(tmp != IR_REG_NONE); - ir_emit_load(ctx, type, tmp, from); - ir_emit_store_mem(ctx, type, mem_to, tmp); + ir_reg tmp = IR_IS_TYPE_INT(type) ? tmp_reg : tmp_fp_reg; + IR_ASSERT(tmp != IR_REG_NONE); + ir_emit_load(ctx, type, tmp, from); + ir_emit_store_mem(ctx, type, mem_to, tmp); + } else { + /* local variable address */ + IR_ASSERT(IR_IS_TYPE_INT(type)); + IR_ASSERT(tmp_reg != IR_REG_NONE); + ir_load_local_addr(ctx, tmp_reg, -from - ctx->consts_count); + ir_emit_store_mem(ctx, type, mem_to, tmp_reg); + } } else if (from < IR_REG_NUM) { ir_emit_store_mem(ctx, type, mem_to, from); } else { @@ -909,6 +924,9 @@ static void ir_emit_dessa_moves(ir_ctx *ctx, int b, ir_block *bb) IR_ASSERT(dst == IR_REG_NONE || !IR_REG_SPILLED(dst)); if (IR_IS_CONST_REF(input)) { from = input; + } else if (ir_rule(ctx, input) == IR_STATIC_ALLOCA) { + /* encode local variable address */ + from = -(ctx->consts_count + input); } else { from = (src != IR_REG_NONE && !IR_REG_SPILLED(src)) ? (ir_ref)src : (ir_ref)(IR_REG_NUM + ctx->vregs[input]); diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index e3fb8f9a6ebb8..42624574585b9 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -981,6 +981,7 @@ IR_ALWAYS_INLINE uint32_t ir_insn_len(const ir_insn *insn) #define IR_HAS_VA_ARG_GP (1<<8) #define IR_HAS_VA_ARG_FP (1<<9) #define IR_HAS_FP_RET_SLOT (1<<10) +#define IR_16B_FRAME_ALIGNMENT (1<<11) /* Temporary: SCCP -> CFG */ #define IR_SCCP_DONE (1<<25) diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index f265567d84b20..a7c0403502ad1 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -718,7 +718,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) int n; if (ctx->rules[ref] & (IR_FUSED|IR_SKIPPED)) { - if (ctx->rules[ref] == (IR_SKIPPED|IR_VAR) && ctx->use_lists[ref].count > 0) { + if (((ctx->rules[ref] & IR_RULE_MASK) == IR_VAR + || (ctx->rules[ref] & IR_RULE_MASK) == IR_ALLOCA) + && ctx->use_lists[ref].count > 0) { insn = &ctx->ir_base[ref]; insn->op3 = ctx->vars; ctx->vars = ref; @@ -1305,9 +1307,10 @@ int ir_compute_live_ranges(ir_ctx *ctx) if (input > 0) { uint32_t v = ctx->vregs[input]; - IR_ASSERT(v); - ival = ctx->live_intervals[v]; - ir_add_phi_use(ctx, ival, k, IR_DEF_LIVE_POS_FROM_REF(bb->end), use); + if (v) { + ival = ctx->live_intervals[v]; + ir_add_phi_use(ctx, ival, k, IR_DEF_LIVE_POS_FROM_REF(bb->end), use); + } } } } @@ -1331,7 +1334,9 @@ int ir_compute_live_ranges(ir_ctx *ctx) int n; if (ctx->rules[ref] & (IR_FUSED|IR_SKIPPED)) { - if (ctx->rules[ref] == (IR_SKIPPED|IR_VAR) && ctx->use_lists[ref].count > 0) { + if (((ctx->rules[ref] & IR_RULE_MASK) == IR_VAR + || (ctx->rules[ref] & IR_RULE_MASK) == IR_ALLOCA) + && ctx->use_lists[ref].count > 0) { insn = &ctx->ir_base[ref]; insn->op3 = ctx->vars; ctx->vars = ref; @@ -1909,7 +1914,7 @@ int ir_coalesce(ir_ctx *ctx) insn = &ctx->ir_base[use]; if (insn->op == IR_PHI) { input = ir_insn_op(insn, k); - if (input > 0) { + if (input > 0 && ctx->vregs[input]) { uint32_t v1 = ctx->vregs[input]; uint32_t v2 = ctx->vregs[use]; @@ -2486,12 +2491,11 @@ static ir_live_interval *ir_split_interval_at(ir_ctx *ctx, ir_live_interval *iva return child; } -int32_t ir_allocate_spill_slot(ir_ctx *ctx, ir_type type, ir_reg_alloc_data *data) +static int32_t ir_allocate_small_spill_slot(ir_ctx *ctx, size_t size, ir_reg_alloc_data *data) { int32_t ret; - uint8_t size = ir_type_size[type]; - IR_ASSERT(size == 1 || size == 2 || size == 4 || size == 8); + IR_ASSERT(size == 0 || size == 1 || size == 2 || size == 4 || size == 8); if (data->handled && data->handled[size]) { ret = data->handled[size]->stack_spill_pos; data->handled[size] = data->handled[size]->list_next; @@ -2542,8 +2546,7 @@ int32_t ir_allocate_spill_slot(ir_ctx *ctx, ir_type type, ir_reg_alloc_data *dat ctx->stack_frame_size += 4; } } - } else { - IR_ASSERT(size == 1); + } else if (size == 1) { if (data->unused_slot_1) { ret = data->unused_slot_1; data->unused_slot_1 = 0; @@ -2582,10 +2585,42 @@ int32_t ir_allocate_spill_slot(ir_ctx *ctx, ir_type type, ir_reg_alloc_data *dat ctx->stack_frame_size += 4; } } + } else { + ret = IR_NULL; } return ret; } +int32_t ir_allocate_spill_slot(ir_ctx *ctx, ir_type type, ir_reg_alloc_data *data) +{ + return ir_allocate_small_spill_slot(ctx, ir_type_size[type], data); +} + +static int32_t ir_allocate_big_spill_slot(ir_ctx *ctx, int32_t size, ir_reg_alloc_data *data) +{ + int32_t ret; + + if (size <= 8) { + if (size == 3) { + size = 4; + } else if (size > 4 && size < 8) { + size = 8; + } + return ir_allocate_small_spill_slot(ctx, size, data); + } + + if (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) { + /* Stack must be 16 byte aligned */ + size = IR_ALIGNED_SIZE(size, 16); + } else { + size = IR_ALIGNED_SIZE(size, 8); + } + ret = ctx->stack_frame_size; + ctx->stack_frame_size += size; + + return ret; +} + static ir_reg ir_get_first_reg_hint(ir_ctx *ctx, ir_live_interval *ival, ir_regset available) { ir_use_pos *use_pos; @@ -3464,10 +3499,22 @@ static int ir_linear_scan(ir_ctx *ctx) while (vars) { ir_insn *insn = &ctx->ir_base[vars]; - IR_ASSERT(insn->op == IR_VAR); + IR_ASSERT(insn->op == IR_VAR || insn->op == IR_ALLOCA); vars = insn->op3; /* list next */ - insn->op3 = ir_allocate_spill_slot(ctx, insn->type, &data); + if (insn->op == IR_VAR) { + insn->op3 = ir_allocate_spill_slot(ctx, insn->type, &data); + } else { + ir_insn *val = &ctx->ir_base[insn->op2]; + + IR_ASSERT(IR_IS_CONST_REF(insn->op2)); + IR_ASSERT(IR_IS_TYPE_INT(val->type)); + IR_ASSERT(!IR_IS_SYM_CONST(val->op)); + IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); + IR_ASSERT(val->val.i64 < 0x7fffffff); + + insn->op3 = ir_allocate_big_spill_slot(ctx, val->val.i32, &data); + } } for (j = ctx->vregs_count; j != 0; j--) { @@ -3994,7 +4041,22 @@ static void assign_regs(ir_ctx *ctx) do { IR_ASSERT(ival->reg != IR_REG_NONE); IR_REGSET_INCL(used_regs, ival->reg); - ir_set_alocated_reg(ctx, ival->tmp_ref, ival->tmp_op_num, ival->reg); + reg = ival->reg; + if (ival->tmp_op_num > 0) { + ir_insn *insn = &ctx->ir_base[ival->tmp_ref]; + + if (ival->tmp_op_num <= insn->inputs_count) { + ir_ref *ops = insn->ops; + if (IR_IS_CONST_REF(ops[ival->tmp_op_num])) { + /* constant rematerialization */ + reg |= IR_REG_SPILL_LOAD; + } else if (ctx->ir_base[ops[ival->tmp_op_num]].op == IR_ALLOCA) { + /* local address rematerialization */ + reg |= IR_REG_SPILL_LOAD; + } + } + } + ir_set_alocated_reg(ctx, ival->tmp_ref, ival->tmp_op_num, reg); ival = ival->next; } while (ival); } diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index 89bab16300720..a975392d00f80 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -1064,6 +1064,8 @@ const char *ir_reg_name(int8_t reg, ir_type type) #define IR_RULE_ENUM(name) IR_ ## name, +#define IR_STATIC_ALLOCA (IR_SKIPPED | IR_FUSED | IR_SIMPLE | IR_ALLOCA) + enum _ir_rule { IR_FIRST_RULE = IR_LAST_OP, IR_RULES(IR_RULE_ENUM) @@ -1149,6 +1151,9 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co } if (IR_IS_CONST_REF(insn->op2) && insn->op1 != insn->op2) { n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints); + } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[n] = IR_TMP_REG(2, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; } break; case IR_IMUL3: @@ -1212,12 +1217,18 @@ op2_const: const ir_insn *val_insn = &ctx->ir_base[insn->op1]; constraints->tmp_regs[0] = IR_TMP_REG(1, val_insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n = 1; + } else if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[0] = IR_TMP_REG(1, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n = 1; } else if (ir_rule(ctx, insn->op1) & IR_FUSED) { flags = IR_USE_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG; } if (IR_IS_CONST_REF(insn->op2) && insn->op1 != insn->op2) { flags = IR_USE_MUST_BE_IN_REG | IR_OP1_SHOULD_BE_IN_REG; n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints); + } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[n] = IR_TMP_REG(2, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; } break; case IR_CMP_FP: @@ -1284,6 +1295,9 @@ op2_const: } if (IR_IS_CONST_REF(insn->op3)) { n = ir_add_const_tmp_reg(ctx, insn->op3, 3, n, constraints); + } else if (ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[n] = IR_TMP_REG(3, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; } break; case IR_VSTORE_FP: @@ -1486,17 +1500,17 @@ op2_const: flags = 0; break; case IR_VA_START: - flags = IR_OP1_MUST_BE_IN_REG; + flags = IR_OP2_MUST_BE_IN_REG; constraints->tmp_regs[0] = IR_TMP_REG(3, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n = 1; break; case IR_VA_ARG: - flags = IR_USE_MUST_BE_IN_REG | IR_OP1_MUST_BE_IN_REG; + flags = IR_USE_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG; constraints->tmp_regs[0] = IR_TMP_REG(3, IR_ADDR, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n = 1; break; case IR_VA_COPY: - flags = IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG; + flags = IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG; constraints->tmp_regs[0] = IR_TMP_REG(1, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n = 1; break; @@ -2241,7 +2255,7 @@ binop_fp: } } } - ctx->flags2 |= IR_HAS_CALLS; + ctx->flags2 |= IR_HAS_CALLS | IR_16B_FRAME_ALIGNMENT; #ifndef IR_REG_FP_RET1 if (IR_IS_TYPE_FP(insn->type)) { ctx->flags2 |= IR_HAS_FP_RET_SLOT; @@ -2259,6 +2273,13 @@ binop_fp: case IR_ALLOCA: /* alloca() may be used only in functions */ if (ctx->flags & IR_FUNCTION) { + if (IR_IS_CONST_REF(insn->op2) && ctx->cfg_map[ref] == 1) { + ir_insn *val = &ctx->ir_base[insn->op2]; + + if (!IR_IS_SYM_CONST(val->op)) { + return IR_STATIC_ALLOCA; + } + } ctx->flags |= IR_USE_FRAME_POINTER; ctx->flags2 |= IR_HAS_ALLOCA; } @@ -2966,6 +2987,21 @@ static void ir_emit_load_mem(ir_ctx *ctx, ir_type type, ir_reg reg, ir_mem mem) } } +static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_reg base = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + int32_t offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[src].op3); + + IR_ASSERT(ir_rule(ctx, src) == IR_STATIC_ALLOCA); + if (offset == 0) { + | mov Ra(reg), Ra(base) + } else { + | lea Ra(reg), [Ra(base)+offset] + } +} + static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) { if (IR_IS_CONST_REF(src)) { @@ -2987,8 +3023,10 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) } else { ir_emit_load_imm_fp(ctx, type, reg, src); } - } else { + } else if (ctx->vregs[src]) { ir_emit_load_mem(ctx, type, reg, ir_ref_spill_slot(ctx, src)); + } else { + ir_load_local_addr(ctx, reg, src); } } @@ -3047,6 +3085,7 @@ static void ir_emit_store_mem_int_const(ir_ctx *ctx, ir_type type, ir_mem mem, i ir_emit_store_mem_imm(ctx, type, mem, val); } else { IR_ASSERT(tmp_reg != IR_REG_NONE); + tmp_reg = IR_REG_NUM(tmp_reg); ir_emit_load_imm_int(ctx, type, tmp_reg, val); ir_emit_store_mem_int(ctx, type, mem, tmp_reg); } @@ -3065,6 +3104,7 @@ static void ir_emit_store_mem_fp_const(ir_ctx *ctx, ir_type type, ir_mem mem, ir ir_emit_load_imm_int(ctx, IR_U64, tmp_reg, val->i64); ir_emit_store_mem_int(ctx, IR_U64, mem, tmp_reg); } else { + tmp_fp_reg = IR_REG_NUM(tmp_fp_reg); ir_emit_load(ctx, type, tmp_fp_reg, src); ir_emit_store_mem_fp(ctx, IR_DOUBLE, mem, tmp_fp_reg); } @@ -3156,16 +3196,24 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) ir_insn *insn = &ctx->ir_base[ref]; ir_insn *op1_insn, *op2_insn, *offset_insn; ir_ref base_reg_ref, index_reg_ref; - ir_reg base_reg, index_reg; - int32_t offset, scale; + ir_reg base_reg = IR_REG_NONE, index_reg; + int32_t offset = 0, scale; - IR_ASSERT((rule & IR_RULE_MASK) >= IR_LEA_OB && (rule & IR_RULE_MASK) <= IR_LEA_SI_B); + IR_ASSERT(((rule & IR_RULE_MASK) >= IR_LEA_OB && + (rule & IR_RULE_MASK) <= IR_LEA_SI_B) || + rule == IR_STATIC_ALLOCA); switch (rule & IR_RULE_MASK) { default: IR_ASSERT(0); case IR_LEA_OB: offset_insn = insn; - base_reg_ref = ref * sizeof(ir_ref) + 1; + if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op1].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + } else { + base_reg_ref = ref * sizeof(ir_ref) + 1; + } index_reg_ref = IR_UNUSED; scale = 1; break; @@ -3181,24 +3229,60 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) offset_insn = NULL; break; case IR_LEA_IB: - base_reg_ref = ref * sizeof(ir_ref) + 1; - index_reg_ref = ref * sizeof(ir_ref) + 2; + if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op1].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + index_reg_ref = ref * sizeof(ir_ref) + 2; + } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + index_reg_ref = ref * sizeof(ir_ref) + 1; + } else { + base_reg_ref = ref * sizeof(ir_ref) + 1; + index_reg_ref = ref * sizeof(ir_ref) + 2; + } offset_insn = NULL; scale = 1; break; case IR_LEA_OB_I: - base_reg_ref = insn->op1 * sizeof(ir_ref) + 1; - index_reg_ref = ref * sizeof(ir_ref) + 2; op1_insn = &ctx->ir_base[insn->op1]; offset_insn = op1_insn; scale = 1; + if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + index_reg_ref = insn->op1 * sizeof(ir_ref) + 1; + } else if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[op1_insn->op1].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + index_reg_ref = ref * sizeof(ir_ref) + 2; + } else { + base_reg_ref = insn->op1 * sizeof(ir_ref) + 1; + index_reg_ref = ref * sizeof(ir_ref) + 2; + } break; case IR_LEA_I_OB: - base_reg_ref = ref * sizeof(ir_ref) + 1; - index_reg_ref = insn->op2 * sizeof(ir_ref) + 1; op2_insn = &ctx->ir_base[insn->op2]; offset_insn = op2_insn; scale = 1; + if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op1].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + index_reg_ref = insn->op2 * sizeof(ir_ref) + 1; + } else if (ir_rule(ctx, op2_insn->op1) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[op2_insn->op1].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + index_reg_ref = ref * sizeof(ir_ref) + 1; + } else { + base_reg_ref = ref * sizeof(ir_ref) + 1; + index_reg_ref = insn->op2 * sizeof(ir_ref) + 1; + } break; case IR_LEA_SI_O: index_reg_ref = insn->op1 * sizeof(ir_ref) + 1; @@ -3220,23 +3304,41 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) scale = 1; break; case IR_LEA_OB_SI: - base_reg_ref = insn->op1 * sizeof(ir_ref) + 1; index_reg_ref = insn->op2 * sizeof(ir_ref) + 1; op1_insn = &ctx->ir_base[insn->op1]; offset_insn = op1_insn; op2_insn = &ctx->ir_base[insn->op2]; scale = ctx->ir_base[op2_insn->op2].val.i32; + if (ir_rule(ctx, op1_insn->op1) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[op1_insn->op1].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + } else { + base_reg_ref = insn->op1 * sizeof(ir_ref) + 1; + } break; case IR_LEA_SI_OB: index_reg_ref = insn->op1 * sizeof(ir_ref) + 1; - base_reg_ref = insn->op2 * sizeof(ir_ref) + 1; op1_insn = &ctx->ir_base[insn->op1]; scale = ctx->ir_base[op1_insn->op2].val.i32; op2_insn = &ctx->ir_base[insn->op2]; offset_insn = op2_insn; + if (ir_rule(ctx, op2_insn->op1) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[op2_insn->op1].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + } else { + base_reg_ref = insn->op2 * sizeof(ir_ref) + 1; + } break; case IR_LEA_B_SI: - base_reg_ref = ref * sizeof(ir_ref) + 1; + if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op1].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + } else { + base_reg_ref = ref * sizeof(ir_ref) + 1; + } index_reg_ref = insn->op2 * sizeof(ir_ref) + 1; op2_insn = &ctx->ir_base[insn->op2]; scale = ctx->ir_base[op2_insn->op2].val.i32; @@ -3244,30 +3346,42 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) break; case IR_LEA_SI_B: index_reg_ref = insn->op1 * sizeof(ir_ref) + 1; - base_reg_ref = ref * sizeof(ir_ref) + 2; + if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = IR_UNUSED; + } else { + base_reg_ref = ref * sizeof(ir_ref) + 2; + } op1_insn = &ctx->ir_base[insn->op1]; scale = ctx->ir_base[op1_insn->op2].val.i32; offset_insn = NULL; break; + case IR_ALLOCA: + offset = IR_SPILL_POS_TO_OFFSET(insn->op3); + base_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + base_reg_ref = index_reg_ref = IR_UNUSED; + scale = 1; + offset_insn = NULL; + break; } - offset = 0; if (offset_insn) { ir_insn *addr_insn = &ctx->ir_base[offset_insn->op2]; if (IR_IS_SYM_CONST(addr_insn->op)) { void *addr = ir_sym_val(ctx, addr_insn); IR_ASSERT(sizeof(void*) != 8 || IR_IS_SIGNED_32BIT((intptr_t)addr)); - offset = (int64_t)(intptr_t)(addr); + offset += (int64_t)(intptr_t)(addr); } else { - offset = addr_insn->val.i32; if (offset_insn->op == IR_SUB) { - offset = -offset; + offset = -addr_insn->val.i32; + } else { + offset += addr_insn->val.i32; } } } - base_reg = IR_REG_NONE; if (base_reg_ref) { if (UNEXPECTED(ctx->rules[base_reg_ref / sizeof(ir_ref)] & IR_FUSED_REG)) { base_reg = ir_get_fused_reg(ctx, root, base_reg_ref); @@ -3307,7 +3421,7 @@ static ir_mem ir_fuse_addr(ir_ctx *ctx, ir_ref root, ir_ref ref) static ir_mem ir_fuse_mem(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_insn *mem_insn, ir_reg reg) { if (reg != IR_REG_NONE) { - if (IR_REG_SPILLED(reg) || IR_IS_CONST_REF(mem_insn->op2)) { + if (IR_REG_SPILLED(reg)) { reg = IR_REG_NUM(reg); ir_emit_load(ctx, IR_ADDR, reg, mem_insn->op2); } @@ -3348,6 +3462,41 @@ static int32_t ir_fuse_imm(ir_ctx *ctx, ir_ref ref) } } +static void ir_emit_load_ex(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src, ir_ref root) +{ + if (IR_IS_CONST_REF(src)) { + if (IR_IS_TYPE_INT(type)) { + ir_insn *insn = &ctx->ir_base[src]; + + if (insn->op == IR_SYM || insn->op == IR_FUNC) { + void *addr = ir_sym_val(ctx, insn); + ir_emit_load_imm_int(ctx, type, reg, (intptr_t)addr); + } else if (insn->op == IR_STR) { + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + int label = ir_const_label(ctx, src); + + | lea Ra(reg), aword [=>label] + } else { + ir_emit_load_imm_int(ctx, type, reg, insn->val.i64); + } + } else { + ir_emit_load_imm_fp(ctx, type, reg, src); + } + } else if (ir_rule(ctx, src) == IR_STATIC_ALLOCA) { + ir_load_local_addr(ctx, reg, src); + } else { + ir_mem mem; + + if (ir_rule(ctx, src) & IR_FUSED) { + mem = ir_fuse_load(ctx, root, src); + } else { + mem = ir_ref_spill_slot(ctx, src); + } + ir_emit_load_mem(ctx, type, reg, mem); + } +} + static void ir_emit_prologue(ir_ctx *ctx) { ir_backend_data *data = ctx->data; @@ -3563,7 +3712,7 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) } if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -3735,7 +3884,7 @@ static void ir_emit_min_max_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -3864,7 +4013,7 @@ static void ir_emit_mem_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) break; } } else { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, op2); } @@ -3928,7 +4077,7 @@ static void ir_emit_reg_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) break; } } else { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, op2); } @@ -3998,6 +4147,7 @@ static void ir_emit_mul_div_mod_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) || if (ir_type_size[type] == 8 && ctx->regs[def][2] != IR_REG_NONE) { || ir_reg op2_reg = ctx->regs[def][2]; || +|| op2_reg = IR_REG_NUM(op2_reg); || ir_emit_load_imm_int(ctx, type, op2_reg, mask); | ASM_REG_REG_OP and, type, def_reg, op2_reg || } else { @@ -4028,7 +4178,7 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op)); IR_ASSERT(op1_reg != IR_REG_NONE && def_reg != IR_REG_NONE && op1_reg != def_reg); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } @@ -4053,6 +4203,7 @@ static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) || ir_reg op2_reg = ctx->regs[def][2]; || || if (op2_reg != IR_REG_NONE) { +|| op2_reg = IR_REG_NUM(op2_reg); || ir_emit_load_imm_int(ctx, type, op2_reg, offset); | lea Rq(def_reg), [Rq(op1_reg)+Rq(op2_reg)] || } else { @@ -4119,6 +4270,7 @@ static void ir_emit_smod_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn) || if (ir_type_size[type] == 8 && ctx->regs[def][2] != IR_REG_NONE) { || ir_reg op2_reg = ctx->regs[def][2]; || +|| op2_reg = IR_REG_NUM(op2_reg); || ir_emit_load_imm_int(ctx, type, op2_reg, mask); | ASM_REG_REG_OP and, type, def_reg, op2_reg || } else { @@ -4430,7 +4582,7 @@ static void ir_emit_bit_count(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE); if (op1_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } @@ -4599,7 +4751,7 @@ static void ir_emit_ctpop(ir_ctx *ctx, ir_ref def, ir_insn *insn) |.endif IR_ASSERT(def_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); - if (IR_IS_CONST_REF(op1) || op1_reg == IR_REG_NONE) { + if (op1_reg == IR_REG_NONE) { ir_emit_load(ctx, type, def_reg, op1); if (ir_type_size[insn->type] == 1) { | movzx Rd(def_reg), Rb(def_reg) @@ -4823,21 +4975,19 @@ static void ir_emit_mul_div_mod(ir_ctx *ctx, ir_ref def, ir_insn *insn) } if (op2_reg == IR_REG_NONE && op1 == op2) { op2_reg = IR_REG_RAX; - } else if (IR_IS_CONST_REF(op2)) { - if (insn->op == IR_MUL || insn->op == IR_MUL_OV) { - op2_reg = IR_REG_RDX; - } else { - IR_ASSERT(op2_reg != IR_REG_NONE); + } else if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, type, op2_reg, op2); } + } else if (IR_IS_CONST_REF(op2) + && (insn->op == IR_MUL || insn->op == IR_MUL_OV)) { + op2_reg = IR_REG_RDX; ir_emit_load(ctx, type, op2_reg, op2); } if (insn->op == IR_MUL || insn->op == IR_MUL_OV) { if (IR_IS_TYPE_SIGNED(insn->type)) { if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, type, op2_reg, op2); - } | ASM_REG_OP imul, type, op2_reg } else { if (ir_rule(ctx, op2) & IR_FUSED) { @@ -4849,10 +4999,6 @@ static void ir_emit_mul_div_mod(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } else { if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, type, op2_reg, op2); - } | ASM_REG_OP mul, type, op2_reg } else { if (ir_rule(ctx, op2) & IR_FUSED) { @@ -4875,10 +5021,6 @@ static void ir_emit_mul_div_mod(ir_ctx *ctx, ir_ref def, ir_insn *insn) | movsx ax, al } if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, type, op2_reg, op2); - } | ASM_REG_OP idiv, type, op2_reg } else { if (ir_rule(ctx, op2) & IR_FUSED) { @@ -4895,10 +5037,6 @@ static void ir_emit_mul_div_mod(ir_ctx *ctx, ir_ref def, ir_insn *insn) | ASM_REG_REG_OP xor, type, IR_REG_RDX, IR_REG_RDX } if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, type, op2_reg, op2); - } | ASM_REG_OP div, type, op2_reg } else { if (ir_rule(ctx, op2) & IR_FUSED) { @@ -5086,7 +5224,7 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -5188,12 +5326,12 @@ static void ir_emit_binop_avx(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE && op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -5305,20 +5443,19 @@ static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_type type, ir_ref root, ir_in } | ASM_REG_MEM_OP cmp, type, op1_reg, mem } - } else if (IR_IS_CONST_REF(insn->op1)) { + } else if (IR_IS_CONST_REF(op1)) { IR_ASSERT(0); } else { ir_mem mem; - if (ir_rule(ctx, insn->op1) & IR_FUSED) { - mem = ir_fuse_load(ctx, root, insn->op1); + if (ir_rule(ctx, op1) & IR_FUSED) { + mem = ir_fuse_load(ctx, root, op1); } else { - mem = ir_ref_spill_slot(ctx, insn->op1); + mem = ir_ref_spill_slot(ctx, op1); } if (op2_reg != IR_REG_NONE) { | ASM_MEM_REG_OP cmp, type, mem, op2_reg } else { - IR_ASSERT(!IR_IS_CONST_REF(op1)); int32_t val = ir_fuse_imm(ctx, op2); | ASM_MEM_IMM_OP cmp, type, mem, val } @@ -5333,11 +5470,11 @@ static void ir_emit_cmp_int_common2(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_ins ir_reg op1_reg = ctx->regs[ref][1]; ir_reg op2_reg = ctx->regs[ref][2]; - if (op1_reg != IR_REG_NONE && (IR_IS_CONST_REF(op1) || IR_REG_SPILLED(op1_reg))) { + if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } - if (op2_reg != IR_REG_NONE && (IR_IS_CONST_REF(op2) || IR_REG_SPILLED(op2_reg))) { + if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -5443,11 +5580,11 @@ static void ir_emit_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg op2_reg = ctx->regs[def][2]; IR_ASSERT(def_reg != IR_REG_NONE); - if (op1_reg != IR_REG_NONE && (IR_IS_CONST_REF(op1) || IR_REG_SPILLED(op1_reg))) { + if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } - if (op2_reg != IR_REG_NONE && (IR_IS_CONST_REF(op2) || IR_REG_SPILLED(op2_reg))) { + if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -5499,7 +5636,7 @@ static void ir_emit_test_int_common(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_op ir_emit_load(ctx, type, op1_reg, op1); } if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -5551,7 +5688,7 @@ static void ir_emit_test_int_common(ir_ctx *ctx, ir_ref root, ir_ref ref, ir_op mem = ir_ref_spill_slot(ctx, op1); } if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -5617,12 +5754,12 @@ static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref root, ir_ref cmp_ref, ir_ IR_ASSERT(op1_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -5860,11 +5997,11 @@ static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_i ir_reg op1_reg = ctx->regs[insn->op2][1]; ir_reg op2_reg = ctx->regs[insn->op2][2]; - if (op1_reg != IR_REG_NONE && (IR_IS_CONST_REF(op1) || IR_REG_SPILLED(op1_reg))) { + if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } - if (op2_reg != IR_REG_NONE && (IR_IS_CONST_REF(op2) || IR_REG_SPILLED(op2_reg))) { + if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -5952,6 +6089,14 @@ static void ir_emit_if_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, u } } return; + } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + uint32_t true_block, false_block; + + ir_get_true_false_blocks(ctx, b, &true_block, &false_block); + if (true_block != next_block) { + | jmp =>true_block + } + return; } else { ir_mem mem; @@ -5981,7 +6126,7 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE); - if (op2_reg != IR_REG_NONE && (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2))) { + if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, op2); if (op1 == op2) { @@ -5991,14 +6136,14 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) op3_reg = op2_reg; } } - if (op3_reg != IR_REG_NONE && op3 != op2 && (IR_REG_SPILLED(op3_reg) || IR_IS_CONST_REF(op3))) { + if (op3_reg != IR_REG_NONE && op3 != op2 && IR_REG_SPILLED(op3_reg)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, op3); if (op1 == op2) { op1_reg = op3_reg; } } - if (op1_reg != IR_REG_NONE && op1 != op2 && op1 != op3 && (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(op1))) { + if (op1_reg != IR_REG_NONE && op1 != op2 && op1 != op3 && IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, op1_type, op1_reg, op1); } @@ -6029,10 +6174,8 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else if (IR_IS_CONST_REF(op2) && !IR_IS_SYM_CONST(ctx->ir_base[op2].op)) { /* prevent "xor" and flags clobbering */ ir_emit_mov_imm_int(ctx, type, def_reg, ctx->ir_base[op2].val.i64); - } else if (IR_IS_CONST_REF(op2) || !(ir_rule(ctx, op2) & IR_FUSED)) { - ir_emit_load(ctx, type, def_reg, op2); } else { - ir_emit_load_mem(ctx, type, def_reg, ir_fuse_load(ctx, def, op2)); + ir_emit_load_ex(ctx, type, def_reg, op2, def); } | ASM_REG_REG_OP2 cmove, type, def_reg, op3_reg } @@ -6041,10 +6184,8 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (IR_IS_CONST_REF(op3) && !IR_IS_SYM_CONST(ctx->ir_base[op3].op)) { /* prevent "xor" and flags clobbering */ ir_emit_mov_imm_int(ctx, type, def_reg, ctx->ir_base[op3].val.i64); - } else if (IR_IS_CONST_REF(op3) || !(ir_rule(ctx, op3) & IR_FUSED)) { - ir_emit_load(ctx, type, def_reg, op3); } else { - ir_emit_load_mem(ctx, type, def_reg, ir_fuse_load(ctx, def, op3)); + ir_emit_load_ex(ctx, type, def_reg, op3, def); } | ASM_REG_REG_OP2 cmovne, type, def_reg, op2_reg } @@ -6078,10 +6219,8 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_fp_mov(ctx, type, def_reg, op2_reg); } } - } else if (IR_IS_CONST_REF(op2) || !(ir_rule(ctx, op2) & IR_FUSED)) { - ir_emit_load(ctx, type, def_reg, op2); } else { - ir_emit_load_mem(ctx, type, def_reg, ir_fuse_load(ctx, def, op2)); + ir_emit_load_ex(ctx, type, def_reg, op2, def); } | jmp >3 |2: @@ -6093,10 +6232,8 @@ static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_fp_mov(ctx, type, def_reg, op3_reg); } } - } else if (IR_IS_CONST_REF(op3) || !(ir_rule(ctx, op3) & IR_FUSED)) { - ir_emit_load(ctx, type, def_reg, op3); } else { - ir_emit_load_mem(ctx, type, def_reg, ir_fuse_load(ctx, def, op3)); + ir_emit_load_ex(ctx, type, def_reg, op3, def); } |3: @@ -6117,14 +6254,14 @@ static void ir_emit_cond_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg op3_reg = ctx->regs[def][3]; ir_op op; - if (op2_reg != IR_REG_NONE && (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2))) { + if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, op2); if (op3 == op2) { op3_reg = op2_reg; } } - if (op3_reg != IR_REG_NONE && op3 != op2 && (IR_REG_SPILLED(op3_reg) || IR_IS_CONST_REF(op3))) { + if (op3_reg != IR_REG_NONE && op3 != op2 && IR_REG_SPILLED(op3_reg)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, op3); } @@ -6150,21 +6287,17 @@ static void ir_emit_cond_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else if (IR_IS_CONST_REF(op2) && !IR_IS_SYM_CONST(ctx->ir_base[op2].op)) { /* prevent "xor" and flags clobbering */ ir_emit_mov_imm_int(ctx, type, def_reg, ctx->ir_base[op2].val.i64); - } else if (IR_IS_CONST_REF(op2) || !(ir_rule(ctx, op2) & IR_FUSED)) { - ir_emit_load(ctx, type, def_reg, op2); } else { - ir_emit_load_mem(ctx, type, def_reg, ir_fuse_load(ctx, def, op2)); + ir_emit_load_ex(ctx, type, def_reg, op2, def); } } } else { IR_ASSERT(op2_reg != IR_REG_NONE && op2_reg != def_reg); - if (IR_IS_CONST_REF(op3) && !IR_IS_SYM_CONST(ctx->ir_base[op3].op)) { + if (IR_IS_CONST_REF(op3) && !IR_IS_SYM_CONST(ctx->ir_base[op3].op)) { /* prevent "xor" and flags clobbering */ ir_emit_mov_imm_int(ctx, type, def_reg, ctx->ir_base[op3].val.i64); - } else if (IR_IS_CONST_REF(op3) || !(ir_rule(ctx, op3) & IR_FUSED)) { - ir_emit_load(ctx, type, def_reg, op3); } else { - ir_emit_load_mem(ctx, type, def_reg, ir_fuse_load(ctx, def, op3)); + ir_emit_load_ex(ctx, type, def_reg, op3, def); } op3_reg = op2_reg; op ^= 1; // reverse @@ -6249,10 +6382,8 @@ static void ir_emit_cond_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_fp_mov(ctx, type, def_reg, op2_reg); } } - } else if (IR_IS_CONST_REF(op2) || !(ir_rule(ctx, op2) & IR_FUSED)) { - ir_emit_load(ctx, type, def_reg, op2); } else { - ir_emit_load_mem(ctx, type, def_reg, ir_fuse_load(ctx, def, op2)); + ir_emit_load_ex(ctx, type, def_reg, op2, def); } | jmp >3 |2: @@ -6264,10 +6395,8 @@ static void ir_emit_cond_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_fp_mov(ctx, type, def_reg, op3_reg); } } - } else if (IR_IS_CONST_REF(op3) || !(ir_rule(ctx, op3) & IR_FUSED)) { - ir_emit_load(ctx, type, def_reg, op3); } else { - ir_emit_load_mem(ctx, type, def_reg, ir_fuse_load(ctx, def, op3)); + ir_emit_load_ex(ctx, type, def_reg, op3, def); } |3: } @@ -6289,14 +6418,14 @@ static void ir_emit_cond_cmp_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg op3_reg = ctx->regs[def][3]; ir_op op; - if (op2_reg != IR_REG_NONE && (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(op2))) { + if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, op2); if (op3 == op2) { op3_reg = op2_reg; } } - if (op3_reg != IR_REG_NONE && op3 != op2 && (IR_REG_SPILLED(op3_reg) || IR_IS_CONST_REF(op3))) { + if (op3_reg != IR_REG_NONE && op3 != op2 && IR_REG_SPILLED(op3_reg)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, op3); } @@ -6353,10 +6482,8 @@ static void ir_emit_cond_cmp_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_fp_mov(ctx, type, def_reg, op2_reg); } } - } else if (IR_IS_CONST_REF(op2) || !(ir_rule(ctx, op2) & IR_FUSED)) { - ir_emit_load(ctx, type, def_reg, op2); } else { - ir_emit_load_mem(ctx, type, def_reg, ir_fuse_load(ctx, def, op2)); + ir_emit_load_ex(ctx, type, def_reg, op2, def); } | jmp >3 |2: @@ -6368,10 +6495,8 @@ static void ir_emit_cond_cmp_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_fp_mov(ctx, type, def_reg, op3_reg); } } - } else if (IR_IS_CONST_REF(op3) || !(ir_rule(ctx, op3) & IR_FUSED)) { - ir_emit_load(ctx, type, def_reg, op3); } else { - ir_emit_load_mem(ctx, type, def_reg, ir_fuse_load(ctx, def, op3)); + ir_emit_load_ex(ctx, type, def_reg, op3, def); } |3: @@ -6664,25 +6789,16 @@ static void ir_emit_trunc(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(IR_IS_TYPE_INT(dst_type)); IR_ASSERT(ir_type_size[dst_type] < ir_type_size[src_type]); IR_ASSERT(def_reg != IR_REG_NONE); - if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { - op1_reg = IR_REG_NUM(op1_reg); - ir_emit_load(ctx, src_type, op1_reg, insn->op1); - } if (op1_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op1_reg)) { + op1_reg = IR_REG_NUM(op1_reg); + ir_emit_load(ctx, src_type, op1_reg, insn->op1); + } if (op1_reg != def_reg) { ir_emit_mov(ctx, dst_type, def_reg, op1_reg); } - } else if (IR_IS_CONST_REF(insn->op1)) { - ir_emit_load(ctx, dst_type, def_reg, insn->op1); } else { - ir_mem mem; - - if (ir_rule(ctx, insn->op1) & IR_FUSED) { - mem = ir_fuse_load(ctx, def, insn->op1); - } else { - mem = ir_ref_spill_slot(ctx, insn->op1); - } - ir_emit_load_mem(ctx, dst_type, def_reg, mem); + ir_emit_load_ex(ctx, dst_type, def_reg, insn->op1, def); } if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, dst_type, def, def_reg); @@ -6701,9 +6817,7 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(ir_type_size[dst_type] == ir_type_size[src_type]); IR_ASSERT(def_reg != IR_REG_NONE); if (IR_IS_TYPE_INT(src_type) && IR_IS_TYPE_INT(dst_type)) { - if (!IR_IS_CONST_REF(insn->op1) && (ir_rule(ctx, insn->op1) & IR_FUSED)) { - ir_emit_load_mem_int(ctx, dst_type, def_reg, ir_fuse_load(ctx, def, insn->op1)); - } else if (op1_reg != IR_REG_NONE) { + if (op1_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, src_type, op1_reg, insn->op1); @@ -6712,13 +6826,10 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_mov(ctx, dst_type, def_reg, op1_reg); } } else { - ir_emit_load(ctx, dst_type, def_reg, insn->op1); + ir_emit_load_ex(ctx, dst_type, def_reg, insn->op1, def); } } else if (IR_IS_TYPE_FP(src_type) && IR_IS_TYPE_FP(dst_type)) { - if (!IR_IS_CONST_REF(insn->op1) && (ir_rule(ctx, insn->op1) & IR_FUSED)) { - ir_mem mem = ir_fuse_load(ctx, def, insn->op1); - ir_emit_load_mem_fp(ctx, dst_type, def_reg, mem); - } else if (op1_reg != IR_REG_NONE) { + if (op1_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, src_type, op1_reg, insn->op1); @@ -6727,7 +6838,7 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_fp_mov(ctx, dst_type, def_reg, op1_reg); } } else { - ir_emit_load(ctx, dst_type, def_reg, insn->op1); + ir_emit_load_ex(ctx, dst_type, def_reg, insn->op1, def); } } else if (IR_IS_TYPE_FP(src_type)) { IR_ASSERT(IR_IS_TYPE_INT(dst_type)); @@ -6844,7 +6955,7 @@ static void ir_emit_int2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (op1_reg != IR_REG_NONE) { bool src64 = 0; - if (IR_REG_SPILLED(op1_reg) || IR_IS_CONST_REF(insn->op1)) { + if (IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, src_type, op1_reg, insn->op1); } @@ -7375,7 +7486,7 @@ static void ir_emit_load_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) } IR_ASSERT(def_reg != IR_REG_NONE); if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(insn->op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); @@ -7414,7 +7525,7 @@ static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) } IR_ASSERT(def_reg != IR_REG_NONE); if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(insn->op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); @@ -7449,7 +7560,7 @@ static void ir_emit_store_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) ir_mem mem; if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(insn->op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); @@ -7460,7 +7571,10 @@ static void ir_emit_store_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED); mem = ir_fuse_addr(ctx, ref, insn->op2); - if (!IR_IS_CONST_REF(insn->op3) && IR_REG_SPILLED(op3_reg) && ir_is_same_spill_slot(ctx, insn->op3, mem)) { + if (!IR_IS_CONST_REF(insn->op3) + && IR_REG_SPILLED(op3_reg) + && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA + && ir_is_same_spill_slot(ctx, insn->op3, mem)) { if (!ir_may_avoid_spill_load(ctx, insn->op3, ref)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, insn->op3); @@ -7495,7 +7609,7 @@ static void ir_emit_cmp_and_store_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) ir_reg op2_reg = ctx->regs[insn->op3][2]; if (addr_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(addr_reg) || IR_IS_CONST_REF(insn->op2)) { + if (IR_REG_SPILLED(addr_reg)) { addr_reg = IR_REG_NUM(addr_reg); IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); ir_emit_load(ctx, IR_ADDR, addr_reg, insn->op2); @@ -7508,11 +7622,11 @@ static void ir_emit_cmp_and_store_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn) mem = ir_fuse_addr(ctx, ref, insn->op2); } - if (op1_reg != IR_REG_NONE && (IR_IS_CONST_REF(op1) || IR_REG_SPILLED(op1_reg))) { + if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } - if (op2_reg != IR_REG_NONE && (IR_IS_CONST_REF(op2) || IR_REG_SPILLED(op2_reg))) { + if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -7532,7 +7646,7 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) IR_ASSERT(op3_reg != IR_REG_NONE); if (op2_reg != IR_REG_NONE) { - if (IR_REG_SPILLED(op2_reg) || IR_IS_CONST_REF(insn->op2)) { + if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR); ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); @@ -7543,7 +7657,10 @@ static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) } else { IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED); mem = ir_fuse_addr(ctx, ref, insn->op2); - if (!IR_IS_CONST_REF(insn->op3) && IR_REG_SPILLED(op3_reg) && ir_is_same_spill_slot(ctx, insn->op3, mem)) { + if (!IR_IS_CONST_REF(insn->op3) + && IR_REG_SPILLED(op3_reg) + && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA + && ir_is_same_spill_slot(ctx, insn->op3, mem)) { if (!ir_may_avoid_spill_load(ctx, insn->op3, ref)) { op3_reg = IR_REG_NUM(op3_reg); ir_emit_load(ctx, type, op3_reg, insn->op3); @@ -7609,9 +7726,7 @@ static void ir_emit_rstore(ir_ctx *ctx, ir_ref ref, ir_insn *insn) ir_reg op2_reg = ctx->regs[ref][2]; ir_reg dst_reg = insn->op3; - if (!IR_IS_CONST_REF(insn->op2) && (ir_rule(ctx, insn->op2) & IR_FUSED)) { - ir_emit_load_mem(ctx, type, dst_reg, ir_fuse_load(ctx, ref, insn->op2)); - } else if (op2_reg != IR_REG_NONE) { + if (op2_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, insn->op2); @@ -7625,7 +7740,7 @@ static void ir_emit_rstore(ir_ctx *ctx, ir_ref ref, ir_insn *insn) } } } else { - ir_emit_load(ctx, type, dst_reg, insn->op2); + ir_emit_load_ex(ctx, type, dst_reg, insn->op2, ref); } } @@ -7648,7 +7763,7 @@ static void ir_emit_alloca(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 >= 0); IR_ASSERT(IR_IS_SIGNED_32BIT(val->val.i64)); - if (ctx->flags2 & IR_HAS_CALLS) { + if (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) { /* Stack must be 16 byte aligned */ size = IR_ALIGNED_SIZE(size, 16); } else { @@ -7659,7 +7774,7 @@ static void ir_emit_alloca(ir_ctx *ctx, ir_ref def, ir_insn *insn) ctx->call_stack_size += size; } } else { - int32_t alignment = (ctx->flags2 & IR_HAS_CALLS) ? 16 : 8; + int32_t alignment = (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) ? 16 : 8; ir_reg op2_reg = ctx->regs[def][2]; ir_type type = ctx->ir_base[insn->op2].type; @@ -7706,7 +7821,7 @@ static void ir_emit_afree(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(IR_IS_TYPE_UNSIGNED(val->type) || val->val.i64 > 0); IR_ASSERT(IR_IS_SIGNED_32BIT(val->val.i64)); - if (ctx->flags2 & IR_HAS_CALLS) { + if (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) { /* Stack must be 16 byte aligned */ size = IR_ALIGNED_SIZE(size, 16); } else { @@ -7717,7 +7832,7 @@ static void ir_emit_afree(ir_ctx *ctx, ir_ref def, ir_insn *insn) ctx->call_stack_size -= size; } } else { -// int32_t alignment = (ctx->flags2 & IR_HAS_CALLS) ? 16 : 8; +// int32_t alignment = (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) ? 16 : 8; ir_reg op2_reg = ctx->regs[def][2]; ir_type type = ctx->ir_base[insn->op2].type; @@ -7786,11 +7901,19 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) int arg_area_offset; ir_reg op2_reg = ctx->regs[def][2]; ir_reg tmp_reg = ctx->regs[def][3]; + int32_t offset; IR_ASSERT(tmp_reg != IR_REG_NONE); - if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -7801,13 +7924,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) arg_area_offset = ctx->stack_frame_size + ctx->call_stack_size + sizeof(void*) + ctx->param_stack_size; } | lea Ra(tmp_reg), aword [Ra(fp)+arg_area_offset] - if (op2_reg != IR_REG_NONE) { - | mov aword [Ra(op2_reg)], Ra(tmp_reg) - } else { - int32_t offset = ir_ref_spill_slot_offset(ctx, insn->op2, &op2_reg); - - | mov aword [Ra(op2_reg)+offset], Ra(tmp_reg) - } + | mov aword [Ra(op2_reg)+offset], Ra(tmp_reg) #elif defined(IR_TARGET_X64) |.if X64 ir_backend_data *data = ctx->data; @@ -7818,11 +7935,19 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg op2_reg = ctx->regs[def][2]; ir_reg tmp_reg = ctx->regs[def][3]; bool have_reg_save_area = 0; + int32_t offset; - IR_ASSERT(op2_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + IR_ASSERT(tmp_reg != IR_REG_NONE); + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } if (ctx->flags & IR_USE_FRAME_POINTER) { @@ -7839,11 +7964,11 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) | lea Ra(tmp_reg), aword [Ra(fp)+reg_save_area_offset] have_reg_save_area = 1; /* Set va_list.gp_offset */ - | mov dword [Ra(op2_reg)+offsetof(ir_va_list, gp_offset)], sizeof(void*) * ctx->gp_reg_params + | mov dword [Ra(op2_reg)+(offset+offsetof(ir_va_list, gp_offset))], sizeof(void*) * ctx->gp_reg_params } else { reg_save_area_offset -= sizeof(void*) * IR_REG_INT_ARGS; /* Set va_list.gp_offset */ - | mov dword [Ra(op2_reg)+offsetof(ir_va_list, gp_offset)], sizeof(void*) * IR_REG_INT_ARGS + | mov dword [Ra(op2_reg)+(offset+offsetof(ir_va_list, gp_offset))], sizeof(void*) * IR_REG_INT_ARGS } if ((ctx->flags2 & (IR_HAS_VA_ARG_FP|IR_HAS_VA_COPY)) && ctx->fp_reg_params < IR_REG_FP_ARGS) { if (!have_reg_save_area) { @@ -7851,18 +7976,18 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) have_reg_save_area = 1; } /* Set va_list.fp_offset */ - | mov dword [Ra(op2_reg)+offsetof(ir_va_list, fp_offset)], sizeof(void*) * IR_REG_INT_ARGS + 16 * ctx->fp_reg_params + | mov dword [Ra(op2_reg)+(offset+offsetof(ir_va_list, fp_offset))], sizeof(void*) * IR_REG_INT_ARGS + 16 * ctx->fp_reg_params } else { /* Set va_list.fp_offset */ - | mov dword [Ra(op2_reg)+offsetof(ir_va_list, fp_offset)], sizeof(void*) * IR_REG_INT_ARGS + 16 * IR_REG_FP_ARGS + | mov dword [Ra(op2_reg)+(offset+offsetof(ir_va_list, fp_offset))], sizeof(void*) * IR_REG_INT_ARGS + 16 * IR_REG_FP_ARGS } if (have_reg_save_area) { /* Set va_list.reg_save_area */ - | mov qword [Ra(op2_reg)+offsetof(ir_va_list, reg_save_area)], Ra(tmp_reg) + | mov qword [Ra(op2_reg)+(offset+offsetof(ir_va_list, reg_save_area))], Ra(tmp_reg) } | lea Ra(tmp_reg), aword [Ra(fp)+overflow_arg_area_offset] /* Set va_list.overflow_arg_area */ - | mov qword [Ra(op2_reg)+offsetof(ir_va_list, overflow_arg_area)], Ra(tmp_reg) + | mov qword [Ra(op2_reg)+(offset+offsetof(ir_va_list, overflow_arg_area))], Ra(tmp_reg) |.endif #else IR_ASSERT(0 && "NIY va_start"); @@ -7877,18 +8002,33 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg tmp_reg = ctx->regs[def][1]; ir_reg op2_reg = ctx->regs[def][2]; ir_reg op3_reg = ctx->regs[def][3]; + int32_t op2_offset, op3_offset; - IR_ASSERT(tmp_reg != IR_REG_NONE && op2_reg != IR_REG_NONE && op3_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + IR_ASSERT(tmp_reg != IR_REG_NONE); + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + op2_offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + op2_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } - if (IR_REG_SPILLED(op3_reg)) { - op3_reg = IR_REG_NUM(op3_reg); - ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3); + if (op3_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op3_reg)) { + op3_reg = IR_REG_NUM(op3_reg); + ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3); + } + op3_offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); + op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + op3_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op3].op3); } - | mov Ra(tmp_reg), aword [Ra(op3_reg)] - | mov aword [Ra(op2_reg)], Ra(tmp_reg) + | mov Ra(tmp_reg), aword [Ra(op3_reg)+op3_offset] + | mov aword [Ra(op2_reg)+op2_offset], Ra(tmp_reg) #elif defined(IR_TARGET_X64) |.if X64 ir_backend_data *data = ctx->data; @@ -7896,24 +8036,39 @@ static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg tmp_reg = ctx->regs[def][1]; ir_reg op2_reg = ctx->regs[def][2]; ir_reg op3_reg = ctx->regs[def][3]; + int32_t op2_offset, op3_offset; - IR_ASSERT(tmp_reg != IR_REG_NONE && op2_reg != IR_REG_NONE && op3_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); - } - if (IR_REG_SPILLED(op3_reg)) { - op3_reg = IR_REG_NUM(op3_reg); - ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3); + IR_ASSERT(tmp_reg != IR_REG_NONE); + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + op2_offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + op2_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } - | mov Rd(tmp_reg), dword [Ra(op3_reg)+offsetof(ir_va_list, gp_offset)] - | mov dword [Ra(op2_reg)+offsetof(ir_va_list, gp_offset)], Rd(tmp_reg) - | mov Rd(tmp_reg), dword [Ra(op3_reg)+offsetof(ir_va_list, fp_offset)] - | mov aword [Ra(op2_reg)+offsetof(ir_va_list, fp_offset)], Ra(tmp_reg) - | mov Ra(tmp_reg), aword [Ra(op3_reg)+offsetof(ir_va_list, overflow_arg_area)] - | mov aword [Ra(op2_reg)+offsetof(ir_va_list, overflow_arg_area)], Ra(tmp_reg) - | mov Ra(tmp_reg), aword [Ra(op3_reg)+offsetof(ir_va_list, reg_save_area)] - | mov aword [Ra(op2_reg)+offsetof(ir_va_list, reg_save_area)], Ra(tmp_reg) + if (op3_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op3_reg)) { + op3_reg = IR_REG_NUM(op3_reg); + ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3); + } + op3_offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA); + op3_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + op3_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op3].op3); + } + | mov Rd(tmp_reg), dword [Ra(op3_reg)+(op3_offset+offsetof(ir_va_list, gp_offset))] + | mov dword [Ra(op2_reg)+(op2_offset+offsetof(ir_va_list, gp_offset))], Rd(tmp_reg) + | mov Rd(tmp_reg), dword [Ra(op3_reg)+(op3_offset+offsetof(ir_va_list, fp_offset))] + | mov aword [Ra(op2_reg)+(op2_offset+offsetof(ir_va_list, fp_offset))], Ra(tmp_reg) + | mov Ra(tmp_reg), aword [Ra(op3_reg)+(op3_offset+offsetof(ir_va_list, overflow_arg_area))] + | mov aword [Ra(op2_reg)+(op2_offset+offsetof(ir_va_list, overflow_arg_area))], Ra(tmp_reg) + | mov Ra(tmp_reg), aword [Ra(op3_reg)+(op3_offset+offsetof(ir_va_list, reg_save_area))] + | mov aword [Ra(op2_reg)+(op2_offset+offsetof(ir_va_list, reg_save_area))], Ra(tmp_reg) |.endif #else IR_ASSERT(0 && "NIY va_copy"); @@ -7929,22 +8084,24 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg def_reg = ctx->regs[def][0]; ir_reg op2_reg = ctx->regs[def][2]; ir_reg tmp_reg = ctx->regs[def][3]; + int32_t offset; IR_ASSERT(def_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); - if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); - } - | mov Ra(tmp_reg), aword [Ra(op2_reg)] - ir_emit_load_mem(ctx, type, def_reg, IR_MEM_B(tmp_reg)); - | add Ra(tmp_reg), IR_MAX(ir_type_size[type], sizeof(void*)) if (op2_reg != IR_REG_NONE) { - | mov aword [Ra(op2_reg)], Ra(tmp_reg) + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + offset = 0; } else { - int32_t offset = ir_ref_spill_slot_offset(ctx, insn->op2, &op2_reg); - - | mov aword [Ra(op2_reg)+offset], Ra(tmp_reg) + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } + | mov Ra(tmp_reg), aword [Ra(op2_reg)+offset] + ir_emit_load_mem(ctx, type, def_reg, IR_MEM_B(tmp_reg)); + | add Ra(tmp_reg), IR_MAX(ir_type_size[type], sizeof(void*)) + | mov aword [Ra(op2_reg)+offset], Ra(tmp_reg) if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, type, def, def_reg); } @@ -7956,40 +8113,48 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg def_reg = ctx->regs[def][0]; ir_reg op2_reg = ctx->regs[def][2]; ir_reg tmp_reg = ctx->regs[def][3]; + int32_t offset; - IR_ASSERT(def_reg != IR_REG_NONE && op2_reg != IR_REG_NONE && tmp_reg != IR_REG_NONE); - if (IR_REG_SPILLED(op2_reg)) { - op2_reg = IR_REG_NUM(op2_reg); - ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + IR_ASSERT(def_reg != IR_REG_NONE&& tmp_reg != IR_REG_NONE); + if (op2_reg != IR_REG_NONE) { + if (IR_REG_SPILLED(op2_reg)) { + op2_reg = IR_REG_NUM(op2_reg); + ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2); + } + offset = 0; + } else { + IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA); + op2_reg = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3); } if (IR_IS_TYPE_INT(type)) { - | mov Rd(tmp_reg), dword [Ra(op2_reg)+offsetof(ir_va_list, gp_offset)] + | mov Rd(tmp_reg), dword [Ra(op2_reg)+(offset+offsetof(ir_va_list, gp_offset))] | cmp Rd(tmp_reg), sizeof(void*)*IR_REG_INT_ARGS | jge >1 | add Rd(tmp_reg), sizeof(void*) - | mov dword [Ra(op2_reg)+offsetof(ir_va_list, gp_offset)], Rd(tmp_reg) - | add Ra(tmp_reg), aword [Ra(op2_reg)+offsetof(ir_va_list, reg_save_area)] + | mov dword [Ra(op2_reg)+(offset+offsetof(ir_va_list, gp_offset))], Rd(tmp_reg) + | add Ra(tmp_reg), aword [Ra(op2_reg)+(offset+offsetof(ir_va_list, reg_save_area))] | jmp >2 |1: - | mov Ra(tmp_reg), aword [Ra(op2_reg)+offsetof(ir_va_list, overflow_arg_area)] + | mov Ra(tmp_reg), aword [Ra(op2_reg)+(offset+offsetof(ir_va_list, overflow_arg_area))] | add Ra(tmp_reg), sizeof(void*) - | mov aword [Ra(op2_reg)+offsetof(ir_va_list, overflow_arg_area)], Ra(tmp_reg) + | mov aword [Ra(op2_reg)+(offset+offsetof(ir_va_list, overflow_arg_area))], Ra(tmp_reg) |2: | mov Ra(def_reg), aword [Ra(tmp_reg)-sizeof(void*)] } else { - | mov Rd(tmp_reg), dword [Ra(op2_reg)+offsetof(ir_va_list, fp_offset)] + | mov Rd(tmp_reg), dword [Ra(op2_reg)+(offset+offsetof(ir_va_list, fp_offset))] | cmp Rd(tmp_reg), sizeof(void*) * IR_REG_INT_ARGS + 16 * IR_REG_FP_ARGS | jge >1 | add Rd(tmp_reg), 16 - | mov dword [Ra(op2_reg)+offsetof(ir_va_list, fp_offset)], Rd(tmp_reg) - | add Ra(tmp_reg), aword [Ra(op2_reg)+offsetof(ir_va_list, reg_save_area)] + | mov dword [Ra(op2_reg)+(offset+offsetof(ir_va_list, fp_offset))], Rd(tmp_reg) + | add Ra(tmp_reg), aword [Ra(op2_reg)+(offset+offsetof(ir_va_list, reg_save_area))] ir_emit_load_mem_fp(ctx, type, def_reg, IR_MEM_BO(tmp_reg, -16)); | jmp >2 |1: - | mov Ra(tmp_reg), aword [Ra(op2_reg)+offsetof(ir_va_list, overflow_arg_area)] + | mov Ra(tmp_reg), aword [Ra(op2_reg)+(offset+offsetof(ir_va_list, overflow_arg_area))] ir_emit_load_mem_fp(ctx, type, def_reg, IR_MEM_BO(tmp_reg, 0)); | add Ra(tmp_reg), 8 - | mov aword [Ra(op2_reg)+offsetof(ir_va_list, overflow_arg_area)], Ra(tmp_reg) + | mov aword [Ra(op2_reg)+(offset+offsetof(ir_va_list, overflow_arg_area))], Ra(tmp_reg) |2: } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -8054,8 +8219,6 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); ir_emit_load(ctx, type, op2_reg, insn->op2); - } else if (IR_IS_CONST_REF(insn->op2)) { - ir_emit_load(ctx, type, op2_reg, insn->op2); } /* Generate a table jmp or a seqence of calls */ @@ -8380,7 +8543,7 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, ir_reg #endif } if (dst_reg != IR_REG_NONE) { - if (IR_IS_CONST_REF(arg) || src_reg == IR_REG_NONE) { + if (src_reg == IR_REG_NONE) { /* delay CONST->REG and MEM->REG moves to third pass */ do_pass3 = 1; } else { @@ -8448,7 +8611,7 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, ir_reg #endif } if (dst_reg != IR_REG_NONE) { - if (IR_IS_CONST_REF(arg) || src_reg == IR_REG_NONE) { + if (src_reg == IR_REG_NONE) { if (IR_IS_TYPE_INT(type)) { if (IR_IS_CONST_REF(arg)) { if (type == IR_I8 || type == IR_I16) { @@ -8457,7 +8620,7 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, ir_reg type = IR_U32; } ir_emit_load(ctx, type, dst_reg, arg); - } else { + } else if (ctx->vregs[arg]) { ir_mem mem = ir_ref_spill_slot(ctx, arg); if (ir_type_size[type] > 2) { @@ -8476,6 +8639,8 @@ static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, ir_reg | ASM_TXT_TMEM_OP movzx, Rd(dst_reg), byte, mem } } + } else { + ir_load_local_addr(ctx, dst_reg, arg); } } else { ir_emit_load(ctx, type, dst_reg, arg); @@ -9100,11 +9265,11 @@ static bool ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn * ir_reg op2_reg = ctx->regs[insn->op2][2]; void *addr; - if (op1_reg != IR_REG_NONE && (IR_IS_CONST_REF(op1) || IR_REG_SPILLED(op1_reg))) { + if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } - if (op2_reg != IR_REG_NONE && (IR_IS_CONST_REF(op2) || IR_REG_SPILLED(op2_reg))) { + if (op2_reg != IR_REG_NONE && IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); if (op1 != op2) { ir_emit_load(ctx, type, op2_reg, op2); @@ -9846,7 +10011,13 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) n--; if (constraints.tmp_regs[n].type) { ir_reg reg = ir_get_free_reg(constraints.tmp_regs[n].type, available); + ir_ref *ops = insn->ops; IR_REGSET_EXCL(available, reg); + if (constraints.tmp_regs[n].num > 0 + && IR_IS_CONST_REF(ops[constraints.tmp_regs[n].num])) { + /* rematerialization */ + reg |= IR_REG_SPILL_LOAD; + } ctx->regs[i][constraints.tmp_regs[n].num] = reg; } else if (constraints.tmp_regs[n].reg == IR_REG_SCRATCH) { available = IR_REGSET_DIFFERENCE(available, IR_REGSET_SCRATCH); @@ -9930,18 +10101,11 @@ void ir_fix_stack_frame(ir_ctx *ctx) ctx->locals_area_size = ctx->stack_frame_size; - if (ctx->used_preserved_regs) { - ir_regset used_preserved_regs = (ir_regset)ctx->used_preserved_regs; - ir_reg reg; - (void) reg; - - IR_REGSET_FOREACH(used_preserved_regs, reg) { - additional_size += sizeof(void*); - } IR_REGSET_FOREACH_END(); - } - #if defined(IR_TARGET_X64) && !defined(_WIN64) if ((ctx->flags & IR_VARARG_FUNC) && (ctx->flags2 & IR_HAS_VA_START)) { + ctx->flags2 |= IR_16B_FRAME_ALIGNMENT; + ctx->stack_frame_size = IR_ALIGNED_SIZE(ctx->stack_frame_size, 16); + ctx->locals_area_size = ctx->stack_frame_size; if ((ctx->flags2 & (IR_HAS_VA_ARG_GP|IR_HAS_VA_COPY)) && ctx->gp_reg_params < IR_REG_INT_ARGS) { additional_size += sizeof(void*) * IR_REG_INT_ARGS; } @@ -9951,12 +10115,22 @@ void ir_fix_stack_frame(ir_ctx *ctx) } #endif + if (ctx->used_preserved_regs) { + ir_regset used_preserved_regs = (ir_regset)ctx->used_preserved_regs; + ir_reg reg; + (void) reg; + + IR_REGSET_FOREACH(used_preserved_regs, reg) { + additional_size += sizeof(void*); + } IR_REGSET_FOREACH_END(); + } + ctx->stack_frame_size = IR_ALIGNED_SIZE(ctx->stack_frame_size, sizeof(void*)); ctx->stack_frame_size += additional_size; ctx->stack_frame_alignment = 0; ctx->call_stack_size = 0; - if (ctx->flags2 & IR_HAS_CALLS) { + if (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) { /* Stack must be 16 byte aligned */ if (!(ctx->flags & IR_FUNCTION)) { while (IR_ALIGNED_SIZE(ctx->stack_frame_size, 16) != ctx->stack_frame_size) {

« previous php.cvs (#128005) next »