From f4aa7f9368fd5097d0ee0649160487ede97b1999 Mon Sep 17 00:00:00 2001 From: Xiaomei Xue Date: Fri, 16 Oct 2026 18:58:11 +0800 Subject: [PATCH 1/2] i386: Extend MOVNTDQ support to cover various packed integer types Reference: https://gcc.gnu.org/cgit/gcc/commit/?id=41d7bf31278cc166d9a3b4073da09afbaef78fc4 Prerequisite reference: https://gcc.gnu.org/cgit/gcc/commit/?id=2e51fff7ce02a6105aa1acff57cbbdd8a767a33f Backport the trunk MOVNTDQ and non-temporal store changes from 41d7bf31278 to the GCC 12-based TGCC branch. Extend the patterns to cover packed QI, HI and SI integer modes as well as the existing DI modes. This GCC 12 branch does not define VI_AVX_AVX512F, so import the mode iterator from trunk. gcc/ChangeLog: * config/i386/sse.md (VI_AVX_AVX512F): New mode iterator, partially backported from 2e51fff7ce02 with GCC 12 conditions. (sse2): Add V4SI/V8SI/V16SI/V8HI/V16HI/V32HI. (_movnt): Change mode iterator from VI8 to VI_AVX_AVX512F. (STORENT_MODE): Add V4SI/V8SI/V16SI/V8HI/V16HI/V32HI/V16QI/ V32QI/V64QI. Signed-off-by: Xiaomei Xue --- gcc/config/i386/sse.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md index 4d13cd75010..38d582f5a58 100644 --- a/gcc/config/i386/sse.md +++ b/gcc/config/i386/sse.md @@ -460,6 +460,12 @@ (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI]) +(define_mode_iterator VI_AVX_AVX512F + [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI + (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI + (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI + (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI]) + ;; All QImode vector integer modes (define_mode_iterator VI1 [(V32QI "TARGET_AVX") V16QI]) @@ -881,6 +887,8 @@ (define_mode_attr sse2 [(V16QI "sse2") (V32QI "avx") (V64QI "avx512f") + (V8HI "sse2") (V16HI "avx") (V32HI "avx512f") + (V4SI "sse2") (V8SI "avx") (V16SI "avx512f") (V2DI "sse2") (V4DI "avx") (V8DI "avx512f")]) (define_mode_attr sse3 @@ -1786,9 +1794,10 @@ (set_attr "mode" "")]) (define_insn "_movnt" - [(set (match_operand:VI8 0 "memory_operand" "=m") - (unspec:VI8 [(match_operand:VI8 1 "register_operand" "v")] - UNSPEC_MOVNT))] + [(set (match_operand:VI_AVX_AVX512F 0 "memory_operand" "=m") + (unspec:VI_AVX_AVX512F + [(match_operand:VI_AVX_AVX512F 1 "register_operand" "v")] + UNSPEC_MOVNT))] "TARGET_SSE2" "%vmovntdq\t{%1, %0|%0, %1}" [(set_attr "type" "ssecvt") @@ -1809,9 +1818,12 @@ (define_mode_iterator STORENT_MODE [(DI "TARGET_SSE2 && TARGET_64BIT") (SI "TARGET_SSE2") (SF "TARGET_SSE4A") (DF "TARGET_SSE4A") - (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") (V2DI "TARGET_SSE2") (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF - (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")]) + (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2") + (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") (V2DI "TARGET_SSE2") + (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") (V4SI "TARGET_SSE2") + (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") (V8HI "TARGET_SSE2") + (V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") (V16QI "TARGET_SSE2")]) (define_expand "storent" [(set (match_operand:STORENT_MODE 0 "memory_operand") -- Gitee From d6f41d0ef82cba87c8d92bb356e4a9c2fef9cf79 Mon Sep 17 00:00:00 2001 From: "xuexiaomei@hygon.cn" Date: Fri, 16 Oct 2026 18:55:00 +0800 Subject: [PATCH 2/2] Add non-temporal store optimization Add -fnon-temporal-store to enable non-temporal store optimization at -O3. Analyze candidate stores before vectorization so that the vectorizer can align their accesses. After vectorization, clear the preliminary marks and reanalyze the stores, inserting memory fences only in this second pass. For OpenMP static loops without an explicit chunk size, recover constant iteration counts from outlined functions to improve reuse-distance estimates when the per-thread trip count is unknown. Add serial and OpenMP tests covering vector and scalar stores and the optimization threshold. Update existing prefetch tests to use the aprefetch2 dump name. gcc/ChangeLog: * common.opt (fnon-temporal-store): New option. * passes.def: Add a pre-vectorization analysis instance of pass_loop_prefetch and parameterize the existing instance. * tree-ssa-loop-niter.cc (get_oldest_def_with_type_conv): New. (get_omp_orig_loop_niters): Likewise. (traverse_niters_expr_operands): Likewise. (get_const_niters_from_omp_outlined): Likewise. * tree-ssa-loop-niter.h (get_const_niters_from_omp_outlined): Declare. * tree-ssa-loop-prefetch.cc: Include tree-parloops.h. (nontemporal_store_p): Check the preferred SIMD mode during analysis. (mark_nontemporal_store, mark_nontemporal_stores): Parameterize. (determine_loop_nest_reuse): Recover OpenMP iteration counts. (loop_prefetch_arrays): Support analysis before vectorization. (unset_nontemporal_move): New. (tree_ssa_prefetch_arrays): Clear analysis marks before reanalysis. (pass_loop_prefetch): Clone and parameterize the pass; update gate. (pass_loop_prefetch::execute): Pass the analysis parameter. * tree-vect-data-refs.cc (vect_supportable_dr_alignment): Require aligned accesses for non-temporal stores. gcc/testsuite/ChangeLog: * gcc.target/i386/nontemporal-store-1.c: New test. * gcc.target/i386/nontemporal-store-2.c: Likewise. * gcc.dg/tree-ssa/prefetch-3.c: Scan aprefetch2 instead of aprefetch. * gcc.dg/tree-ssa/prefetch-5.c: Likewise. * gcc.dg/tree-ssa/prefetch-6.c: Likewise. * gcc.dg/tree-ssa/prefetch-7.c: Likewise. * gcc.dg/tree-ssa/prefetch-8.c: Likewise. * gcc.dg/tree-ssa/prefetch-9.c: Likewise. * gcc.dg/tree-ssa/loop-28.c: Likewise. * gcc.dg/tree-ssa/update-unroll-1.c: Likewise. * gcc.target/i386/pr53397-1.c: Likewise. * gcc.target/i386/pr53397-2.c: Likewise. libgomp/ChangeLog: * testsuite/libgomp.c/nontemporal-store-1.c: New test. * testsuite/libgomp.c/nontemporal-store-2.c: Likewise. * testsuite/libgomp.c/nontemporal-store-3.c: Likewise. * testsuite/libgomp.c/nontemporal-store-4.c: Likewise. Signed-off-by: Xiaomei Xue --- gcc/common.opt | 4 + gcc/passes.def | 3 +- gcc/testsuite/gcc.dg/tree-ssa/loop-28.c | 2 +- gcc/testsuite/gcc.dg/tree-ssa/prefetch-3.c | 2 +- gcc/testsuite/gcc.dg/tree-ssa/prefetch-5.c | 4 +- gcc/testsuite/gcc.dg/tree-ssa/prefetch-6.c | 4 +- gcc/testsuite/gcc.dg/tree-ssa/prefetch-7.c | 6 +- gcc/testsuite/gcc.dg/tree-ssa/prefetch-8.c | 2 +- gcc/testsuite/gcc.dg/tree-ssa/prefetch-9.c | 4 +- .../gcc.dg/tree-ssa/update-unroll-1.c | 4 +- .../gcc.target/i386/nontemporal-store-1.c | 47 ++++++ .../gcc.target/i386/nontemporal-store-2.c | 47 ++++++ gcc/testsuite/gcc.target/i386/pr53397-1.c | 2 +- gcc/testsuite/gcc.target/i386/pr53397-2.c | 4 +- gcc/tree-ssa-loop-niter.cc | 149 ++++++++++++++++++ gcc/tree-ssa-loop-niter.h | 1 + gcc/tree-ssa-loop-prefetch.cc | 113 +++++++++++-- gcc/tree-vect-data-refs.cc | 6 + .../testsuite/libgomp.c/nontemporal-store-1.c | 47 ++++++ .../testsuite/libgomp.c/nontemporal-store-2.c | 47 ++++++ .../testsuite/libgomp.c/nontemporal-store-3.c | 48 ++++++ .../testsuite/libgomp.c/nontemporal-store-4.c | 48 ++++++ 22 files changed, 560 insertions(+), 34 deletions(-) create mode 100644 gcc/testsuite/gcc.target/i386/nontemporal-store-1.c create mode 100644 gcc/testsuite/gcc.target/i386/nontemporal-store-2.c create mode 100644 libgomp/testsuite/libgomp.c/nontemporal-store-1.c create mode 100644 libgomp/testsuite/libgomp.c/nontemporal-store-2.c create mode 100644 libgomp/testsuite/libgomp.c/nontemporal-store-3.c create mode 100644 libgomp/testsuite/libgomp.c/nontemporal-store-4.c diff --git a/gcc/common.opt b/gcc/common.opt index bdf9b643664..708ed496d98 100644 --- a/gcc/common.opt +++ b/gcc/common.opt @@ -2257,6 +2257,10 @@ Enum(offload_abi) String(ilp32) Value(OFFLOAD_ABI_ILP32) EnumValue Enum(offload_abi) String(lp64) Value(OFFLOAD_ABI_LP64) +fnon-temporal-store +Common Var(flag_non_temporal_store) Init(0) Optimization +Generate nontemporal store instructions in loops if available. + fomit-frame-pointer Common Var(flag_omit_frame_pointer) Optimization When possible do not generate stack frames. diff --git a/gcc/passes.def b/gcc/passes.def index 63322c38b03..150a4012b95 100644 --- a/gcc/passes.def +++ b/gcc/passes.def @@ -314,6 +314,7 @@ along with GCC; see the file COPYING3. If not see NEXT_PASS (pass_parallelize_loops, false /* oacc_kernels_p */); NEXT_PASS (pass_expand_omp_ssa); NEXT_PASS (pass_ch_vect); + NEXT_PASS (pass_loop_prefetch, true /* nontemporal_store_analysis_p */); NEXT_PASS (pass_if_conversion); /* pass_vectorize must immediately follow pass_if_conversion. Please do not add any other passes in between. */ @@ -329,7 +330,7 @@ along with GCC; see the file COPYING3. If not see NEXT_PASS (pass_dse); POP_INSERT_PASSES () NEXT_PASS (pass_slp_vectorize); - NEXT_PASS (pass_loop_prefetch); + NEXT_PASS (pass_loop_prefetch, false); /* Run IVOPTs after the last pass that uses data-reference analysis as that doesn't handle TARGET_MEM_REFs. */ NEXT_PASS (pass_iv_optimize); diff --git a/gcc/testsuite/gcc.dg/tree-ssa/loop-28.c b/gcc/testsuite/gcc.dg/tree-ssa/loop-28.c index 3ca0ab8ea5f..ea897082637 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/loop-28.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/loop-28.c @@ -17,5 +17,5 @@ void foo(int n) /* { dg-final { scan-tree-dump-times "MEM" 65 "optimized" } } */ /* There should be no i_a = i_b assignments. */ -/* { dg-final { scan-tree-dump-times "i_.*= i_\[0-9\]*;" 0 "aprefetch" } } */ +/* { dg-final { scan-tree-dump-times "i_.*= i_\[0-9\]*;" 0 "aprefetch2" } } */ diff --git a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-3.c b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-3.c index 46f0d0c2317..319e6dd3271 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-3.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-3.c @@ -18,4 +18,4 @@ double test(void) return sum; } -/* { dg-final { scan-tree-dump-times "unroll factor 4" 1 "aprefetch" } } */ +/* { dg-final { scan-tree-dump-times "unroll factor 4" 1 "aprefetch2" } } */ diff --git a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-5.c b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-5.c index 7f343b66025..34d13acd286 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-5.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-5.c @@ -56,5 +56,5 @@ int loop5 (int n, struct tail5 *x) /* Until we are able to track likely upper bounds, we can't really work out that small trailing arrays should not be prefetched. */ -/* { dg-final { scan-tree-dump-times "Issued prefetch" 2 "aprefetch" } } */ -/* { dg-final { scan-tree-dump-times "Not prefetching" 1 "aprefetch" } } */ +/* { dg-final { scan-tree-dump-times "Issued prefetch" 2 "aprefetch2" } } */ +/* { dg-final { scan-tree-dump-times "Not prefetching" 1 "aprefetch2" } } */ diff --git a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-6.c b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-6.c index 3fa71685ace..a67e50c9eea 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-6.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-6.c @@ -44,5 +44,5 @@ double test(void) return sum; } -/* { dg-final { scan-tree-dump-times "Issued prefetch" 5 "aprefetch" } } */ -/* { dg-final { scan-tree-dump-times "Issued nontemporal prefetch" 3 "aprefetch" } } */ +/* { dg-final { scan-tree-dump-times "Issued prefetch" 5 "aprefetch2" } } */ +/* { dg-final { scan-tree-dump-times "Issued nontemporal prefetch" 3 "aprefetch2" } } */ diff --git a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-7.c b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-7.c index 26de8719dc2..185549c4d30 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-7.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-7.c @@ -32,8 +32,8 @@ void test(int *p) a[i] = 0; } -/* { dg-final { scan-tree-dump-times "Issued prefetch" 5 "aprefetch" } } */ -/* { dg-final { scan-tree-dump-times "Issued nontemporal prefetch" 2 "aprefetch" } } */ -/* { dg-final { scan-tree-dump-times "a nontemporal store" 0 "aprefetch" } } */ +/* { dg-final { scan-tree-dump-times "Issued prefetch" 5 "aprefetch2" } } */ +/* { dg-final { scan-tree-dump-times "Issued nontemporal prefetch" 2 "aprefetch2" } } */ +/* { dg-final { scan-tree-dump-times "a nontemporal store" 0 "aprefetch2" } } */ /* { dg-final { scan-tree-dump-times "builtin_prefetch" 7 "optimized" } } */ diff --git a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-8.c b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-8.c index 00f41c3e4f4..5315ba74d82 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-8.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-8.c @@ -13,7 +13,7 @@ void test() a[i] = 0; } -/* { dg-final { scan-tree-dump-times "a nontemporal store" 1 "aprefetch" } } */ +/* { dg-final { scan-tree-dump-times "a nontemporal store" 1 "aprefetch2" } } */ /* { dg-final { scan-tree-dump "=\\{nt\\}" "optimized" } } */ /* { dg-final { scan-tree-dump-times "__builtin_ia32_mfence" 1 "optimized" } } */ diff --git a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-9.c b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-9.c index e8848c30acc..b1c705354b2 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/prefetch-9.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/prefetch-9.c @@ -14,8 +14,8 @@ void test() } -/* { dg-final { scan-tree-dump-times "Issued nontemporal prefetch" 1 "aprefetch" } } */ -/* { dg-final { scan-tree-dump-times "a nontemporal store" 1 "aprefetch" } } */ +/* { dg-final { scan-tree-dump-times "Issued nontemporal prefetch" 1 "aprefetch2" } } */ +/* { dg-final { scan-tree-dump-times "a nontemporal store" 1 "aprefetch2" } } */ /* { dg-final { scan-tree-dump-times "builtin_prefetch" 1 "optimized" } } */ /* { dg-final { scan-tree-dump "=\\{nt\\}" "optimized" } } */ diff --git a/gcc/testsuite/gcc.dg/tree-ssa/update-unroll-1.c b/gcc/testsuite/gcc.dg/tree-ssa/update-unroll-1.c index 1028c8b06f5..5ca089bef1a 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/update-unroll-1.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/update-unroll-1.c @@ -16,5 +16,5 @@ int foo(unsigned n) /* We used to make the probability that the body of the loop (unrolled to enable prefetching) is entered 0, which is not correct. */ -/* { dg-final { scan-tree-dump-not "Invalid sum" "aprefetch"} } */ -/* { dg-final { scan-tree-dump-not "SUCC: 7 .100.0%" "aprefetch"} } */ +/* { dg-final { scan-tree-dump-not "Invalid sum" "aprefetch2"} } */ +/* { dg-final { scan-tree-dump-not "SUCC: 7 .100.0%" "aprefetch2"} } */ diff --git a/gcc/testsuite/gcc.target/i386/nontemporal-store-1.c b/gcc/testsuite/gcc.target/i386/nontemporal-store-1.c new file mode 100644 index 00000000000..e9aca00553b --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/nontemporal-store-1.c @@ -0,0 +1,47 @@ +/* Non-temporal vector stores to short arrays. */ +/* { dg-do run } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-O3 -fnon-temporal-store -fdump-tree-aprefetch-details -save-temps" } */ + +#include + +#define loop_num 1000000 +typedef unsigned long long ssize_t; +typedef short DP; + +DP a[loop_num + 10]; +DP b[loop_num + 10]; +DP c[loop_num + 10]; + +void +test () +{ + ssize_t j; + for (j = 0; j < loop_num; j++) + { + c[j] = a[j] + b[j]; + } +} + +int +main () +{ + for (ssize_t i = 0; i < loop_num; i++) + { + a[i] = (DP) i; + b[i] = (DP) i; + } + + test (); + + for (ssize_t i = 0; i < loop_num; i++) + { + assert (c[i] == (DP) (a[i] + b[i])); + } + + return 0; +} + +/* { dg-final { scan-tree-dump-times "Marked reference .*? as a nontemporal store" 4 "aprefetch1" } } */ +/* { dg-final { scan-tree-dump-times "Marked reference .*? as a nontemporal store" 4 "aprefetch2" } } */ +/* { dg-final { scan-assembler "movntdq" } } */ diff --git a/gcc/testsuite/gcc.target/i386/nontemporal-store-2.c b/gcc/testsuite/gcc.target/i386/nontemporal-store-2.c new file mode 100644 index 00000000000..93d14399b56 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/nontemporal-store-2.c @@ -0,0 +1,47 @@ +/* Scalar non-temporal stores with vectorization disabled. */ +/* { dg-do run } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-O3 -fnon-temporal-store -fdump-tree-aprefetch-details -save-temps -fno-tree-vectorize" } */ + +#include + +#define loop_num 100000 +typedef unsigned long long ssize_t; +typedef int DP; +typedef long DP2; +typedef long long DP3; + +DP a[loop_num + 10]; +DP b[loop_num + 10]; + +DP2 d[loop_num + 10]; +DP2 e[loop_num + 10]; + +DP3 q[loop_num + 10]; +DP3 w[loop_num + 10]; + +int +main () +{ + for (ssize_t i = 0; i < loop_num; i++) + { + a[i] = (DP) i; + b[i] = (DP) i; + } + for (ssize_t i = 0; i < loop_num; i++) + { + d[i] = (DP2) i; + e[i] = (DP2) i; + } + for (ssize_t i = 0; i < loop_num; i++) + { + q[i] = (DP3) i; + w[i] = (DP3) i; + } + + return 0; +} + +/* { dg-final { scan-tree-dump-times "Marked reference .*? as a nontemporal store" 6 "aprefetch2" } } */ +/* { dg-final { scan-assembler-not "movntdq" } } */ +/* { dg-final { scan-assembler "movnti" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr53397-1.c b/gcc/testsuite/gcc.target/i386/pr53397-1.c index 045ef386ed4..73267d06362 100644 --- a/gcc/testsuite/gcc.target/i386/pr53397-1.c +++ b/gcc/testsuite/gcc.target/i386/pr53397-1.c @@ -21,6 +21,6 @@ void prefetch_when_non_constant_step_is_invariant(int step, int n) } } -/* { dg-final { scan-tree-dump "Issued prefetch" "aprefetch" } } */ +/* { dg-final { scan-tree-dump "Issued prefetch" "aprefetch2" } } */ /* { dg-final { scan-assembler "prefetcht0" } } */ diff --git a/gcc/testsuite/gcc.target/i386/pr53397-2.c b/gcc/testsuite/gcc.target/i386/pr53397-2.c index 4c60308d86c..93db8a18708 100644 --- a/gcc/testsuite/gcc.target/i386/pr53397-2.c +++ b/gcc/testsuite/gcc.target/i386/pr53397-2.c @@ -20,7 +20,7 @@ void donot_prefetch_when_non_constant_step_is_variant(int step, int n) } } -/* { dg-final { scan-tree-dump "Not prefetching" "aprefetch" } } */ -/* { dg-final { scan-tree-dump "loop variant step" "aprefetch" } } */ +/* { dg-final { scan-tree-dump "Not prefetching" "aprefetch2" } } */ +/* { dg-final { scan-tree-dump "loop variant step" "aprefetch2" } } */ diff --git a/gcc/tree-ssa-loop-niter.cc b/gcc/tree-ssa-loop-niter.cc index 0353ffd3022..7d3f2c86d88 100644 --- a/gcc/tree-ssa-loop-niter.cc +++ b/gcc/tree-ssa-loop-niter.cc @@ -4587,6 +4587,155 @@ estimated_stmt_executions_int (class loop *loop) return snit < 0 ? -1 : snit; } +/* Extract x from type conversion. ie: + a=(uint64_t)b, we can use this function to get b from a. */ + +static tree +get_oldest_def_with_type_conv (tree t) +{ + auto ssa = t; + while (ssa && TREE_CODE (ssa) == SSA_NAME) + { + auto g = SSA_NAME_DEF_STMT (ssa); + if (!is_gimple_assign (g)) + break; + auto rhs_code = gimple_assign_rhs_code (g); + if (rhs_code != NOP_EXPR && rhs_code != SSA_NAME) + break; + ssa = gimple_assign_rhs1 (g); + } + return ssa; +} + +/* Recursively traces the SSA variable OP to obtain the original loop's + constant iteration count from an OpenMP outlined function. Only supports + cases where the trip count is an INTEGER_CST. The trip count can only be + determined when OpenMP static scheduling is used without a specified chunk + size. */ + +static HOST_WIDE_INT +get_omp_orig_loop_niters (tree op) +{ + HOST_WIDE_INT niters = -1; + tree rhs1 = NULL_TREE, rhs2 = NULL_TREE, fndecl, arg; + gimple *def; + gphi *phi; + unsigned num_args; + enum tree_code code; + + if (!op) + return niters; + + if (TREE_CODE (op) != SSA_NAME) + return niters; + + def = SSA_NAME_DEF_STMT (op); + if (!def) + return niters; + + if (gimple_code (def) == GIMPLE_PHI) + { + /* Only handle PHI nodes in the outermost loop. */ + if (gimple_bb (def)->loop_father != current_loops->tree_root) + return niters; + + phi = as_a (def); + num_args = gimple_phi_num_args (phi); + if (num_args > 2) + return niters; + + /* Recursively trace PHI arguments. */ + for (unsigned i = 0; i < num_args; i++) + { + arg = gimple_phi_arg_def (phi, i); + niters = get_omp_orig_loop_niters (arg); + if (niters != -1) + break; + } + return niters; + } + + if (!is_gimple_assign (def)) + return niters; + code = gimple_assign_rhs_code (def); + rhs1 = gimple_assign_rhs1 (def); + + if (code == COND_EXPR) + return get_omp_orig_loop_niters (gimple_assign_rhs2 (def)); + + /* Recursively trace rhs1 if not a division/modulo or not a constant. */ + if (code != TRUNC_MOD_EXPR && code != TRUNC_DIV_EXPR) + return get_omp_orig_loop_niters (rhs1); + else if (TREE_CODE (rhs1) != INTEGER_CST) + return get_omp_orig_loop_niters (rhs1); + + /* Find the expression q = n / nthreads or tt = n % nthreads. */ + rhs2 = gimple_assign_rhs2 (def); + rhs2 = get_oldest_def_with_type_conv (rhs2); + def = SSA_NAME_DEF_STMT (rhs2); + if (def && is_gimple_call (def)) + { + fndecl = gimple_call_fndecl (def); + if (fndecl + && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_OMP_GET_NUM_THREADS + || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_OMP_GET_THREAD_NUM)) + return tree_to_shwi (rhs1); + } + niters = get_omp_orig_loop_niters (rhs2); + + return niters; +} + +/* Recursively traverses all operands of NITERS_EXPR to determine if the + expression depends on the number of OpenMP threads, where NITERS_EXPR + is typically of the form (upper bound - lower bound + 1) for a thread's task. + If a dependency is found, NITERS is updated with the trip count of the + original loop corresponding to the OpenMP outlined function. */ + +static void +traverse_niters_expr_operands (HOST_WIDE_INT *niters, tree niters_expr) +{ + if (!niters_expr) + return; + + unsigned n = TREE_OPERAND_LENGTH (niters_expr); + if (n == 0) + { + *niters = get_omp_orig_loop_niters (niters_expr); + return; + } + + for (unsigned i = 0; i < n; i++) + { + tree op = TREE_OPERAND (niters_expr, i); + if (TREE_CODE (op) == INTEGER_CST) + continue; + + if (TREE_CODE (op) == NOP_EXPR) + op = TREE_OPERAND (op, 0); + traverse_niters_expr_operands (niters, op); + } +} + +/* Get the constant niters of the original loop from an OpenMP outlined + function. Only supports the case where niters is an INTEGER_CST. */ + +HOST_WIDE_INT +get_const_niters_from_omp_outlined (class loop *loop) +{ + HOST_WIDE_INT niters = -1; + tree niters_expr = number_of_latch_executions (loop); + if (TREE_CODE (niters_expr) == COND_EXPR) + niters_expr = TREE_OPERAND (niters_expr, 1); + + niters_expr = analyze_scalar_evolution (loop, niters_expr); + if (!niters_expr || chrec_contains_undetermined (niters_expr)) + return niters; + + traverse_niters_expr_operands (&niters, niters_expr); + return niters; +} + /* Sets NIT to the maximum number of executions of the latch of the LOOP, plus one. If we have no reliable estimate, the function returns false, otherwise returns true. */ diff --git a/gcc/tree-ssa-loop-niter.h b/gcc/tree-ssa-loop-niter.h index ceaf65e072b..c11b8fba5ea 100644 --- a/gcc/tree-ssa-loop-niter.h +++ b/gcc/tree-ssa-loop-niter.h @@ -45,6 +45,7 @@ extern HOST_WIDE_INT likely_max_loop_iterations_int (class loop *); extern HOST_WIDE_INT max_stmt_executions_int (class loop *); extern HOST_WIDE_INT likely_max_stmt_executions_int (class loop *); extern HOST_WIDE_INT estimated_stmt_executions_int (class loop *); +extern HOST_WIDE_INT get_const_niters_from_omp_outlined (class loop *); extern bool max_stmt_executions (class loop *, widest_int *); extern bool likely_max_stmt_executions (class loop *, widest_int *); extern bool estimated_stmt_executions (class loop *, widest_int *); diff --git a/gcc/tree-ssa-loop-prefetch.cc b/gcc/tree-ssa-loop-prefetch.cc index aebd7c9206f..fe157396c3c 100644 --- a/gcc/tree-ssa-loop-prefetch.cc +++ b/gcc/tree-ssa-loop-prefetch.cc @@ -48,6 +48,7 @@ along with GCC; see the file COPYING3. If not see #include "tree-data-ref.h" #include "diagnostic-core.h" #include "dbgcnt.h" +#include "tree-parloops.h" /* This pass inserts prefetch instructions to optimize cache usage during accesses to arrays in loops. It processes loops sequentially and: @@ -1243,7 +1244,7 @@ issue_prefetches (struct mem_ref_group *groups, can be used. */ static bool -nontemporal_store_p (struct mem_ref *ref) +nontemporal_store_p (struct mem_ref *ref, bool nontemporal_store_analysis_p) { machine_mode mode; enum insn_code code; @@ -1262,6 +1263,20 @@ nontemporal_store_p (struct mem_ref *ref) return false; code = optab_handler (storent_optab, mode); + + /* If there is no nontemporal store instruction for the current scalar mode, + then try to get the preferred SIMD (vector) mode for this scalar type + and check if there is a corresponding nontemporal store instruction. */ + if (nontemporal_store_analysis_p && code == CODE_FOR_nothing + && is_a (mode)) + { + scalar_mode smode = as_a (mode); + machine_mode simd_mode = targetm.vectorize.preferred_simd_mode (smode); + + if (VECTOR_MODE_P (simd_mode)) + code = optab_handler (storent_optab, simd_mode); + } + return code != CODE_FOR_nothing; } @@ -1269,9 +1284,9 @@ nontemporal_store_p (struct mem_ref *ref) and return true. Otherwise, we return false. */ static bool -mark_nontemporal_store (struct mem_ref *ref) +mark_nontemporal_store (struct mem_ref *ref, bool nontemporal_store_analysis_p) { - if (!nontemporal_store_p (ref)) + if (!nontemporal_store_p (ref, nontemporal_store_analysis_p)) return false; if (dump_file && (dump_flags & TDF_DETAILS)) @@ -1343,7 +1358,8 @@ may_use_storent_in_loop_p (class loop *loop) references in the loop. */ static void -mark_nontemporal_stores (class loop *loop, struct mem_ref_group *groups) +mark_nontemporal_stores (class loop *loop, struct mem_ref_group *groups, + bool nontemporal_store_analysis_p) { struct mem_ref *ref; bool any = false; @@ -1353,9 +1369,12 @@ mark_nontemporal_stores (class loop *loop, struct mem_ref_group *groups) for (; groups; groups = groups->next) for (ref = groups->refs; ref; ref = ref->next) - any |= mark_nontemporal_store (ref); + any |= mark_nontemporal_store (ref, nontemporal_store_analysis_p); - if (any && FENCE_FOLLOWING_MOVNT != NULL_TREE) + /* Avoid inserting excessive fence instructions, insert only in the aprefetch2 + pass. */ + if (!nontemporal_store_analysis_p && any + && FENCE_FOLLOWING_MOVNT != NULL_TREE) emit_mfence_after_loop (loop); } @@ -1645,6 +1664,10 @@ determine_loop_nest_reuse (class loop *loop, struct mem_ref_group *refs, aloop = vloops[i]; vol = estimated_stmt_executions_int (aloop); + + if (vol == -1 && flag_openmp && parallelized_function_p (cfun->decl)) + vol = get_const_niters_from_omp_outlined (aloop); + if (vol == -1) vol = expected_loop_iterations (aloop); volume *= vol; @@ -1877,7 +1900,7 @@ insn_to_prefetch_ratio_too_small_p (unsigned ninsns, unsigned prefetch_count, true if the LOOP was unrolled. */ static bool -loop_prefetch_arrays (class loop *loop) +loop_prefetch_arrays (class loop *loop, bool nontemporal_store_analysis_p) { struct mem_ref_group *refs; unsigned ahead, ninsns, time, unroll_factor; @@ -1918,7 +1941,8 @@ loop_prefetch_arrays (class loop *loop) /* Give up prefetching if the number of memory references in the loop is not reasonable based on profitablity and compilation time considerations. */ - if (!mem_ref_count_reasonable_p (ninsns, mem_ref_count)) + if (!nontemporal_store_analysis_p + && !mem_ref_count_reasonable_p (ninsns, mem_ref_count)) goto fail; /* Step 2: estimate the reuse effects. */ @@ -1948,11 +1972,17 @@ loop_prefetch_arrays (class loop *loop) /* Prefetching is not likely to be profitable if the instruction to prefetch ratio is too small. */ - if (insn_to_prefetch_ratio_too_small_p (ninsns, prefetch_count, - unroll_factor)) + if (!nontemporal_store_analysis_p + && insn_to_prefetch_ratio_too_small_p (ninsns, prefetch_count, + unroll_factor)) goto fail; - mark_nontemporal_stores (loop, refs); + mark_nontemporal_stores (loop, refs, nontemporal_store_analysis_p); + + /* With nontemporal store optimization enabled, only the analysis phase is + performed during the aprefetch1 pass. */ + if (nontemporal_store_analysis_p) + goto fail; /* Step 4: what to prefetch? */ if (!schedule_prefetches (refs, unroll_factor, ahead)) @@ -1974,10 +2004,32 @@ fail: return unrolled; } +/* Cancel the nontemporal store marked in aprefetch1 pass. */ +void +unset_nontemporal_move (void) +{ + basic_block bb; + gimple_stmt_iterator bsi; + gimple *stmt; + FOR_EACH_BB_FN (bb, cfun) + { + for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi)) + { + stmt = gsi_stmt (bsi); + if (!is_gimple_assign (stmt)) + continue; + + gassign *assign_stmt = as_a (stmt); + if (gimple_assign_nontemporal_move_p (assign_stmt)) + gimple_assign_set_nontemporal_move (assign_stmt, false); + } + } +} + /* Issue prefetch instructions for array references in loops. */ unsigned int -tree_ssa_prefetch_arrays (void) +tree_ssa_prefetch_arrays (bool nontemporal_store_analysis_p) { bool unrolled = false; int todo_flags = 0; @@ -2023,12 +2075,20 @@ tree_ssa_prefetch_arrays (void) set_builtin_decl (BUILT_IN_PREFETCH, decl, false); } + /* In the aprefetch1 pass, stmts that are successfully analyzed are marked + with nontemporal store, but no fence is inserted. The main purpose is + to complete alignment operations in the vect pass. In the aprefetch2 + pass, the nontemporal store flag is removed, and the stmts are + re-analyzed under the condition of alignment. */ + if (!nontemporal_store_analysis_p) + unset_nontemporal_move (); + for (auto loop : loops_list (cfun, LI_FROM_INNERMOST)) { if (dump_file && (dump_flags & TDF_DETAILS)) fprintf (dump_file, "Processing loop %d:\n", loop->num); - unrolled |= loop_prefetch_arrays (loop); + unrolled |= loop_prefetch_arrays (loop, nontemporal_store_analysis_p); if (dump_file && (dump_flags & TDF_DETAILS)) fprintf (dump_file, "\n\n"); @@ -2065,12 +2125,33 @@ class pass_loop_prefetch : public gimple_opt_pass { public: pass_loop_prefetch (gcc::context *ctxt) - : gimple_opt_pass (pass_data_loop_prefetch, ctxt) + : gimple_opt_pass (pass_data_loop_prefetch, ctxt), + nontemporal_store_analysis_p (false) {} /* opt_pass methods: */ - virtual bool gate (function *) { return flag_prefetch_loop_arrays > 0; } + opt_pass *clone () + { + return new pass_loop_prefetch (m_ctxt); + } + virtual bool gate (function *) + { + if (nontemporal_store_analysis_p) + return optimize >= 3 && flag_non_temporal_store + && (flag_tree_loop_vectorize || cfun->has_force_vectorize_loops); + else + return flag_prefetch_loop_arrays > 0 + || (optimize >= 3 && flag_non_temporal_store); + } virtual unsigned int execute (function *); + void set_pass_param (unsigned int n, bool param) + { + gcc_assert (n == 0); + nontemporal_store_analysis_p = param; + } + +private: + bool nontemporal_store_analysis_p; }; // class pass_loop_prefetch @@ -2094,7 +2175,7 @@ pass_loop_prefetch::execute (function *fun) return 0; } - return tree_ssa_prefetch_arrays (); + return tree_ssa_prefetch_arrays (nontemporal_store_analysis_p); } } // anon namespace diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc index 82e2ce74437..01839c1c6ae 100644 --- a/gcc/tree-vect-data-refs.cc +++ b/gcc/tree-vect-data-refs.cc @@ -6941,6 +6941,12 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info, if (misalignment == 0) return dr_aligned; + /* When using the vmovnt instruction, the data address must be aligned to the + vector width. */ + gassign *assign_stmt = dyn_cast (stmt_info->stmt); + if (assign_stmt && gimple_assign_nontemporal_move_p (assign_stmt)) + return dr_unaligned_unsupported; + /* For now assume all conditional loads/stores support unaligned access without any special code. */ if (gcall *stmt = dyn_cast (stmt_info->stmt)) diff --git a/libgomp/testsuite/libgomp.c/nontemporal-store-1.c b/libgomp/testsuite/libgomp.c/nontemporal-store-1.c new file mode 100644 index 00000000000..90f62f12f39 --- /dev/null +++ b/libgomp/testsuite/libgomp.c/nontemporal-store-1.c @@ -0,0 +1,47 @@ +/* Non-temporal stores to short arrays in an OpenMP loop. */ +/* { dg-do run { target { i?86-*-* x86_64-*-* } } } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-O3 -fnon-temporal-store -fdump-tree-aprefetch-details -save-temps -fopenmp" } */ + +#include + +#define loop_num 1000000 +typedef unsigned long long ssize_t; +typedef short DP; + +DP a[loop_num + 10]; +DP b[loop_num + 10]; +DP c[loop_num + 10]; + +void __attribute__ ((noinline)) test () +{ + ssize_t j; +#pragma omp parallel for + for (j = 0; j < loop_num; j++) + { + c[j] = a[j] + b[j]; + } +} + +int +main () +{ + for (ssize_t i = 0; i < loop_num; i++) + { + a[i] = (DP) i; + b[i] = (DP) i; + } + + test (); + + for (ssize_t i = 0; i < loop_num; i++) + { + assert (c[i] == (DP) (a[i] + b[i])); + } + + return 0; +} + +/* { dg-final { scan-tree-dump-times "Marked reference .*? as a nontemporal store" 3 "aprefetch1" } } */ +/* { dg-final { scan-tree-dump-times "Marked reference .*? as a nontemporal store" 3 "aprefetch2" } } */ +/* { dg-final { scan-assembler "movntdq" } } */ diff --git a/libgomp/testsuite/libgomp.c/nontemporal-store-2.c b/libgomp/testsuite/libgomp.c/nontemporal-store-2.c new file mode 100644 index 00000000000..89f292c6508 --- /dev/null +++ b/libgomp/testsuite/libgomp.c/nontemporal-store-2.c @@ -0,0 +1,47 @@ +/* Non-temporal stores to char arrays in an OpenMP loop. */ +/* { dg-do run { target { i?86-*-* x86_64-*-* } } } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-O3 -fnon-temporal-store -fdump-tree-aprefetch-details -save-temps -fopenmp" } */ + +#include + +#define loop_num 1000000 +typedef unsigned long long ssize_t; +typedef unsigned char DP; + +DP a[loop_num + 10]; +DP b[loop_num + 10]; +DP c[loop_num + 10]; + +void __attribute__ ((noinline)) test () +{ + ssize_t j; +#pragma omp parallel for + for (j = 0; j < loop_num; j++) + { + c[j] = a[j] + b[j]; + } +} + +int +main () +{ + for (ssize_t i = 0; i < loop_num; i++) + { + a[i] = (DP) i; + b[i] = (DP) i; + } + + test (); + + for (ssize_t i = 0; i < loop_num; i++) + { + assert (c[i] == (DP) (a[i] + b[i])); + } + + return 0; +} + +/* { dg-final { scan-tree-dump-times "Marked reference .*? as a nontemporal store" 3 "aprefetch1" } } */ +/* { dg-final { scan-tree-dump-times "Marked reference .*? as a nontemporal store" 3 "aprefetch2" } } */ +/* { dg-final { scan-assembler "movntdq" } } */ diff --git a/libgomp/testsuite/libgomp.c/nontemporal-store-3.c b/libgomp/testsuite/libgomp.c/nontemporal-store-3.c new file mode 100644 index 00000000000..4dee007cb16 --- /dev/null +++ b/libgomp/testsuite/libgomp.c/nontemporal-store-3.c @@ -0,0 +1,48 @@ +/* OpenMP loop just below the non-temporal store threshold. */ +/* { dg-do run { target { i?86-*-* x86_64-*-* } } } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-O3 -fnon-temporal-store --param=l2-cache-size=512 -fdump-tree-aprefetch-details -save-temps -fopenmp" } */ + +#include + +#define loop_num 21845 +typedef unsigned long long ssize_t; +typedef double DP; + +DP a[loop_num + 10]; +DP b[loop_num + 10]; +DP c[loop_num + 10]; + +void __attribute__ ((noinline)) test () +{ + ssize_t j; +#pragma omp parallel for + for (j = 0; j < loop_num; j++) + { + c[j] = a[j] + b[j]; + } +} + +int +main () +{ + for (ssize_t i = 0; i < loop_num; i++) + { + a[i] = (DP) i; + b[i] = (DP) i; + } + + test (); + + for (ssize_t i = 0; i < loop_num; i++) + { + assert (c[i] == a[i] + b[i]); + } + + return 0; +} + +/* { dg-final { scan-tree-dump-not "Marked reference .*? as a nontemporal store" "aprefetch1" } } */ +/* { dg-final { scan-tree-dump-times "Marked reference .*? as a nontemporal store" 1 "aprefetch2" } } */ +/* { dg-final { scan-assembler-not "movntpd" } } */ +/* { dg-final { scan-assembler-not "movntsd" } } */ diff --git a/libgomp/testsuite/libgomp.c/nontemporal-store-4.c b/libgomp/testsuite/libgomp.c/nontemporal-store-4.c new file mode 100644 index 00000000000..2ca30e1fed3 --- /dev/null +++ b/libgomp/testsuite/libgomp.c/nontemporal-store-4.c @@ -0,0 +1,48 @@ +/* OpenMP loop just above the non-temporal store threshold. */ +/* { dg-do run { target { i?86-*-* x86_64-*-* } } } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-O3 -fnon-temporal-store --param=l2-cache-size=512 -fdump-tree-aprefetch-details -save-temps -fopenmp" } */ + +#include + +#define loop_num 21846 +typedef unsigned long long ssize_t; +typedef double DP; + +DP a[loop_num + 10]; +DP b[loop_num + 10]; +DP c[loop_num + 10]; + +void __attribute__ ((noinline)) test () +{ + ssize_t j; +#pragma omp parallel for + for (j = 0; j < loop_num; j++) + { + c[j] = a[j] + b[j]; + } +} + +int +main () +{ + for (ssize_t i = 0; i < loop_num; i++) + { + a[i] = (DP) i; + b[i] = (DP) i; + } + + test (); + + for (ssize_t i = 0; i < loop_num; i++) + { + assert (c[i] == a[i] + b[i]); + } + + return 0; +} + +/* { dg-final { scan-tree-dump-times "Marked reference .*? as a nontemporal store" 1 "aprefetch1" } } */ +/* { dg-final { scan-tree-dump-times "Marked reference .*? as a nontemporal store" 1 "aprefetch2" } } */ +/* { dg-final { scan-assembler "movntpd" } } */ +/* { dg-final { scan-assembler-not "movntsd" } } */ -- Gitee