diff --git a/gcc/config/i386/i386-features.cc b/gcc/config/i386/i386-features.cc index 6a2444eb6b68cc190436974423872d5bdd53e116..114aa7df8bbdd317dafdc6ec2b84ab572b867ffb 100644 --- a/gcc/config/i386/i386-features.cc +++ b/gcc/config/i386/i386-features.cc @@ -2163,6 +2163,203 @@ make_pass_insert_endbr_and_patchable_area (gcc::context *ctxt) return new pass_insert_endbr_and_patchable_area (ctxt); } +/* When a hot loop can be fit into one cacheline, + force align the loop without considering the max skip. */ +static void +ix86_align_loops () +{ + basic_block bb; + + /* Don't do this when we don't know cache line size. */ + if (ix86_cost->prefetch_block == 0) + return; + + loop_optimizer_init (AVOID_CFG_MODIFICATIONS); + profile_count count_threshold = cfun->cfg->count_max.apply_scale + (1, param_align_threshold); + FOR_EACH_BB_FN (bb, cfun) + { + rtx_insn *label = BB_HEAD (bb); + bool has_fallthru = 0; + edge e; + edge_iterator ei; + + if (!LABEL_P (label)) + continue; + + profile_count fallthru_count = profile_count::zero (); + profile_count branch_count = profile_count::zero (); + + FOR_EACH_EDGE (e, ei, bb->preds) + { + if (e->flags & EDGE_FALLTHRU) + has_fallthru = 1, fallthru_count += e->count (); + else + branch_count += e->count (); + } + + if (!fallthru_count.initialized_p () || !branch_count.initialized_p ()) + continue; + + if (bb->loop_father + && bb->loop_father->latch != EXIT_BLOCK_PTR_FOR_FN (cfun) + && (has_fallthru + ? (!(single_succ_p (bb) + && single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun)) + && optimize_bb_for_speed_p (bb) + && branch_count + fallthru_count > count_threshold + && (branch_count + > fallthru_count.apply_scale + (param_align_loop_iterations, 1))) + /* In case there'no fallthru for the loop. + Nops inserted won't be executed. */ + : (branch_count > count_threshold + || (bb->count > bb->prev_bb->count.apply_scale (10, 1) + && (bb->prev_bb->count + <= ENTRY_BLOCK_PTR_FOR_FN (cfun) + ->count.apply_scale (1, 2)))))) + { + rtx_insn* insn, *end_insn; + HOST_WIDE_INT size = 0; + bool padding_p = true; + basic_block tbb = bb; + unsigned cond_branch_num = 0; + bool detect_tight_loop_p = false; + + for (unsigned int i = 0; i != bb->loop_father->num_nodes; + i++, tbb = tbb->next_bb) + { + /* Only handle continuous cfg layout. */ + if (bb->loop_father != tbb->loop_father) + { + padding_p = false; + break; + } + + FOR_BB_INSNS (tbb, insn) + { + if (!NONDEBUG_INSN_P (insn)) + continue; + size += ix86_min_insn_size (insn); + + /* We don't know size of inline asm. + Don't align loop for call. */ + if (asm_noperands (PATTERN (insn)) >= 0 + || CALL_P (insn)) + { + size = -1; + break; + } + } + + if (size == -1 || size > ix86_cost->prefetch_block) + { + padding_p = false; + break; + } + + FOR_EACH_EDGE (e, ei, tbb->succs) + { + /* It could be part of the loop. */ + if (e->dest == bb) + { + detect_tight_loop_p = true; + break; + } + } + + if (detect_tight_loop_p) + break; + + end_insn = BB_END (tbb); + if (JUMP_P (end_insn)) + { + /* For decoded icache: + 1. Up to two branches are allowed per Way. + 2. A non-conditional branch is the last micro-op in a Way. + */ + if (onlyjump_p (end_insn) + && (any_uncondjump_p (end_insn) + || single_succ_p (tbb))) + { + padding_p = false; + break; + } + else if (++cond_branch_num >= 2) + { + padding_p = false; + break; + } + } + + } + + if (padding_p && detect_tight_loop_p) + { + emit_insn_before (gen_max_skip_align (GEN_INT (ceil_log2 (size)), + GEN_INT (0)), label); + /* End of function. */ + if (!tbb || tbb == EXIT_BLOCK_PTR_FOR_FN (cfun)) + break; + /* Skip bb which already fits into one cacheline. */ + bb = tbb; + } + } + } + + loop_optimizer_finalize (); + free_dominance_info (CDI_DOMINATORS); +} + +namespace { + +const pass_data pass_data_align_tight_loops = +{ + RTL_PASS, /* type */ + "align_tight_loops", /* name */ + OPTGROUP_NONE, /* optinfo_flags */ + TV_MACH_DEP, /* tv_id */ + 0, /* properties_required */ + 0, /* properties_provided */ + 0, /* properties_destroyed */ + 0, /* todo_flags_start */ + 0, /* todo_flags_finish */ +}; + +class pass_align_tight_loops : public rtl_opt_pass +{ +public: + pass_align_tight_loops (gcc::context *ctxt) + : rtl_opt_pass (pass_data_align_tight_loops, ctxt) + {} + + /* opt_pass methods: */ + bool gate (function *) final override + { + return TARGET_ALIGN_TIGHT_LOOPS + && optimize + && optimize_function_for_speed_p (cfun); + } + + unsigned int execute (function *) final override + { + timevar_push (TV_MACH_DEP); +#ifdef ASM_OUTPUT_MAX_SKIP_ALIGN + ix86_align_loops (); +#endif + timevar_pop (TV_MACH_DEP); + return 0; + } +}; // class pass_align_tight_loops + +} // anon namespace + +rtl_opt_pass * +make_pass_align_tight_loops (gcc::context *ctxt) +{ + return new pass_align_tight_loops (ctxt); +} + /* At entry of the nearest common dominator for basic blocks with conversions/rcp/sqrt/rsqrt/round, generate a single vxorps %xmmN, %xmmN, %xmmN @@ -2965,4 +3162,3 @@ ix86_generate_version_dispatcher_body (void *node_p) return resolver_decl; } - diff --git a/gcc/config/i386/i386-passes.def b/gcc/config/i386/i386-passes.def index ab49882457d45bfa26c4b2356c615d02df641c7f..bd6ea3878650e5f11e5d0c8d0f2bbfa14927db2a 100644 --- a/gcc/config/i386/i386-passes.def +++ b/gcc/config/i386/i386-passes.def @@ -31,5 +31,8 @@ along with GCC; see the file COPYING3. If not see INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */); INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_insert_endbr_and_patchable_area); + /* pass_align_tight_loops must be after pass_insert_endbr_and_patchable_area. + PR116174. */ + INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_align_tight_loops); INSERT_PASS_AFTER (pass_combine, 1, pass_remove_partial_avx_dependency); diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h index 5b09924c9299c89e0c174a00d09479b971499868..07f0d40407a3eb73c37b52d401a2ab7a8c9808b6 100644 --- a/gcc/config/i386/i386-protos.h +++ b/gcc/config/i386/i386-protos.h @@ -397,6 +397,7 @@ extern rtl_opt_pass *make_pass_insert_vzeroupper (gcc::context *); extern rtl_opt_pass *make_pass_stv (gcc::context *); extern rtl_opt_pass *make_pass_insert_endbr_and_patchable_area (gcc::context *); +extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *); extern rtl_opt_pass *make_pass_remove_partial_avx_dependency (gcc::context *); diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc index 169b2dd24b65a03636e00c14948d8bf53b7ce02f..291a191d8cbcb7d363f00c5cda69803b07428db3 100644 --- a/gcc/config/i386/i386.cc +++ b/gcc/config/i386/i386.cc @@ -21742,7 +21742,8 @@ ix86_avoid_jump_mispredicts (void) if (dump_file) fprintf (dump_file, "Padding insn %i by %i bytes!\n", INSN_UID (insn), padsize); - emit_insn_before (gen_pad (GEN_INT (padsize)), insn); + emit_insn_before (gen_max_skip_align (GEN_INT (4), + GEN_INT (padsize)), insn); } } } diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h index 34c9f2b0182b373eaebdad6176938eb73f677f16..0f187c50a5d358c853696b444f2dd90c40a46afa 100644 --- a/gcc/config/i386/i386.h +++ b/gcc/config/i386/i386.h @@ -364,6 +364,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST]; #define TARGET_INTER_UNIT_CONVERSIONS \ ix86_tune_features[X86_TUNE_INTER_UNIT_CONVERSIONS] #define TARGET_FOUR_JUMP_LIMIT ix86_tune_features[X86_TUNE_FOUR_JUMP_LIMIT] +#define TARGET_ALIGN_TIGHT_LOOPS \ + ix86_tune_features[X86_TUNE_ALIGN_TIGHT_LOOPS] #define TARGET_SCHEDULE ix86_tune_features[X86_TUNE_SCHEDULE] #define TARGET_USE_BT ix86_tune_features[X86_TUNE_USE_BT] #define TARGET_USE_INCDEC ix86_tune_features[X86_TUNE_USE_INCDEC] diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md index d23f3dac6a012a0dd72a14724571f7c09899af1c..788ea1f93cb4f36f4146050817a540e05d683347 100644 --- a/gcc/config/i386/i386.md +++ b/gcc/config/i386/i386.md @@ -15536,16 +15536,19 @@ (set_attr "length_immediate" "0") (set_attr "modrm" "0")]) -;; Pad to 16-byte boundary, max skip in op0. Used to avoid +;; Pad to 1 << op0 byte boundary, max skip in op1. Used to avoid ;; branch prediction penalty for the third jump in a 16-byte ;; block on K8. +;; Also it's used to align tight loops which can be fix into 1 cacheline. +;; It can help code prefetch and reduce DSB miss. -(define_insn "pad" - [(unspec_volatile [(match_operand 0)] UNSPECV_ALIGN)] +(define_insn "max_skip_align" + [(unspec_volatile [(match_operand 0) (match_operand 1)] UNSPECV_ALIGN)] "" { #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN - ASM_OUTPUT_MAX_SKIP_ALIGN (asm_out_file, 4, (int)INTVAL (operands[0])); + ASM_OUTPUT_MAX_SKIP_ALIGN (asm_out_file, (int)INTVAL (operands[0]), + (int)INTVAL (operands[1])); #else /* It is tempting to use ASM_OUTPUT_ALIGN here, but we don't want to do that. The align insn is used to avoid 3 jump instructions in the row to improve diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def index a7c3b25218575aa6feaef6ec22be0f9318ab092f..8522a9c5dc18d0177910a81bdbdebfd16fc60ed5 100644 --- a/gcc/config/i386/x86-tune.def +++ b/gcc/config/i386/x86-tune.def @@ -224,6 +224,9 @@ DEF_TUNE (X86_TUNE_FOUR_JUMP_LIMIT, "four_jump_limit", m_PPRO | m_P4_NOCONA | m_BONNELL | m_SILVERMONT | m_KNL | m_KNM | m_GOLDMONT | m_GOLDMONT_PLUS | m_INTEL | m_ATHLON_K8 | m_AMDFAM10) +/* X86_TUNE_ALIGN_TIGHT_LOOPS: if false, tight loops are not aligned. */ +DEF_TUNE (X86_TUNE_ALIGN_TIGHT_LOOPS, "align_tight_loops", m_C86_4G_M7) + /*****************************************************************************/ /* Integer instruction selection tuning */ /*****************************************************************************/ diff --git a/gcc/testsuite/gcc.target/i386/align-tight-loop-generic.c b/gcc/testsuite/gcc.target/i386/align-tight-loop-generic.c new file mode 100644 index 0000000000000000000000000000000000000000..f8000ce9aa2d9be08f0f8c95febdd987da653922 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/align-tight-loop-generic.c @@ -0,0 +1,11 @@ +/* { dg-do compile { target { *-*-linux* && lp64 } } } */ +/* { dg-options "-O2 -g0 -fcf-protection=branch -mtune=generic" } */ +/* { dg-final { scan-assembler-not "\\.p2align 5" } } */ + +char * +foo (char *dest, const char *src) +{ + while ((*dest++ = *src++) != '\0') + /* nothing */; + return --dest; +} diff --git a/gcc/testsuite/gcc.target/i386/pr116174.c b/gcc/testsuite/gcc.target/i386/pr116174.c new file mode 100644 index 0000000000000000000000000000000000000000..a8aaee87f8f27bdbd8e90e00804b64c5390ea1ed --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr116174.c @@ -0,0 +1,10 @@ +/* { dg-do compile { target { *-*-linux* && lp64 } } } */ +/* { dg-options "-O2 -g0 -fcf-protection=branch -mtune=c86-4g-m7" } */ +/* { dg-final { scan-assembler "endbr64\\n\\t\\.p2align 5" } } */ +char * +foo (char *dest, const char *src) +{ + while ((*dest++ = *src++) != '\0') + /* nothing */; + return --dest; +}