From 084ea8ad5845c9b69c1366fa6dfeaf1a6c5e423b Mon Sep 17 00:00:00 2001 From: Paul-Antoine Arras Date: Wed, 20 Nov 2024 15:28:57 +0100 Subject: [PATCH] OpenMP: middle-end support for dispatch + adjust_args This patch adds middle-end support for the `dispatch` construct and the `adjust_args` clause. The heavy lifting is done in `gimplify_omp_dispatch` and `gimplify_call_expr` respectively. For `adjust_args`, this mostly consists in emitting a call to `omp_get_mapped_ptr` for the adequate device. For dispatch, the following steps are performed: * Handle the device clause, if any: set the default-device ICV at the top of the dispatch region and restore its previous value at the end. * Handle novariants and nocontext clauses, if any. Evaluate compile-time constants and select a variant, if possible. Otherwise, emit code to handle all possible cases at run time. gcc/ChangeLog: * builtins.cc (builtin_fnspec): Handle BUILT_IN_OMP_GET_MAPPED_PTR. * gimple-low.cc (lower_stmt): Handle GIMPLE_OMP_DISPATCH. * gimple-pretty-print.cc (dump_gimple_omp_dispatch): New function. (pp_gimple_stmt_1): Handle GIMPLE_OMP_DISPATCH. * gimple-walk.cc (walk_gimple_stmt): Likewise. * gimple.cc (gimple_build_omp_dispatch): New function. (gimple_copy): Handle GIMPLE_OMP_DISPATCH. * gimple.def (GIMPLE_OMP_DISPATCH): Define. * gimple.h (gimple_build_omp_dispatch): Declare. (gimple_has_substatements): Handle GIMPLE_OMP_DISPATCH. (gimple_omp_dispatch_clauses): New function. (gimple_omp_dispatch_clauses_ptr): Likewise. (gimple_omp_dispatch_set_clauses): Likewise. (gimple_return_set_retval): Handle GIMPLE_OMP_DISPATCH. * gimplify.cc (enum omp_region_type): Add ORT_DISPATCH. (struct gimplify_omp_ctx): Add in_call_args. (gimplify_call_expr): Handle need_device_ptr arguments. (is_gimple_stmt): Handle OMP_DISPATCH. (gimplify_scan_omp_clauses): Handle OMP_CLAUSE_DEVICE in a dispatch construct. Handle OMP_CLAUSE_NOVARIANTS and OMP_CLAUSE_NOCONTEXT. (omp_has_novariants): New function. (omp_has_nocontext): Likewise. (omp_construct_selector_matches): Handle OMP_DISPATCH with nocontext clause. (find_ifn_gomp_dispatch): New function. (gimplify_omp_dispatch): Likewise. (gimplify_expr): Handle OMP_DISPATCH. * gimplify.h (omp_has_novariants): Declare. * internal-fn.cc (expand_GOMP_DISPATCH): New function. * internal-fn.def (GOMP_DISPATCH): Define. * omp-builtins.def (BUILT_IN_OMP_GET_MAPPED_PTR): Define. (BUILT_IN_OMP_GET_DEFAULT_DEVICE): Define. (BUILT_IN_OMP_SET_DEFAULT_DEVICE): Define. * omp-general.cc (omp_construct_traits_to_codes): Add OMP_DISPATCH. (struct omp_ts_info): Add dispatch. (omp_resolve_declare_variant): Handle novariants. Adjust DECL_ASSEMBLER_NAME. * omp-low.cc (scan_omp_1_stmt): Handle GIMPLE_OMP_DISPATCH. (lower_omp_dispatch): New function. (lower_omp_1): Call it. * tree-inline.cc (remap_gimple_stmt): Handle GIMPLE_OMP_DISPATCH. (estimate_num_insns): Handle GIMPLE_OMP_DISPATCH. --- gcc/builtins.cc | 2 + gcc/gimple-low.cc | 1 + gcc/gimple-pretty-print.cc | 33 +++ gcc/gimple-walk.cc | 1 + gcc/gimple.cc | 20 ++ gcc/gimple.def | 5 + gcc/gimple.h | 33 ++- gcc/gimplify.cc | 497 +++++++++++++++++++++++++++++++++++-- gcc/gimplify.h | 1 + gcc/internal-fn.cc | 8 + gcc/internal-fn.def | 1 + gcc/omp-builtins.def | 6 + gcc/omp-general.cc | 13 +- gcc/omp-low.cc | 35 +++ gcc/tree-inline.cc | 7 + 15 files changed, 641 insertions(+), 22 deletions(-) diff --git a/gcc/builtins.cc b/gcc/builtins.cc index 7243c408cf2..ce92b368a32 100644 --- a/gcc/builtins.cc +++ b/gcc/builtins.cc @@ -12583,6 +12583,8 @@ builtin_fnspec (tree callee) by its first argument. */ case BUILT_IN_POSIX_MEMALIGN: return ".cOt"; + case BUILT_IN_OMP_GET_MAPPED_PTR: + return ". R "; default: return ""; diff --git a/gcc/gimple-low.cc b/gcc/gimple-low.cc index 6475d7fadb4..55d9dbf8167 100644 --- a/gcc/gimple-low.cc +++ b/gcc/gimple-low.cc @@ -746,6 +746,7 @@ lower_stmt (gimple_stmt_iterator *gsi, struct lower_data *data) case GIMPLE_EH_MUST_NOT_THROW: case GIMPLE_OMP_FOR: case GIMPLE_OMP_SCOPE: + case GIMPLE_OMP_DISPATCH: case GIMPLE_OMP_SECTIONS: case GIMPLE_OMP_SECTIONS_SWITCH: case GIMPLE_OMP_SECTION: diff --git a/gcc/gimple-pretty-print.cc b/gcc/gimple-pretty-print.cc index 8bd1a6c45c5..dcd03f37bb9 100644 --- a/gcc/gimple-pretty-print.cc +++ b/gcc/gimple-pretty-print.cc @@ -1727,6 +1727,35 @@ dump_gimple_omp_scope (pretty_printer *pp, const gimple *gs, } } +/* Dump a GIMPLE_OMP_DISPATCH tuple on the pretty_printer BUFFER. */ + +static void +dump_gimple_omp_dispatch (pretty_printer *buffer, const gimple *gs, int spc, + dump_flags_t flags) +{ + if (flags & TDF_RAW) + { + dump_gimple_fmt (buffer, spc, flags, "%G <%+BODY <%S>%nCLAUSES <", gs, + gimple_omp_body (gs)); + dump_omp_clauses (buffer, gimple_omp_dispatch_clauses (gs), spc, flags); + dump_gimple_fmt (buffer, spc, flags, " >"); + } + else + { + pp_string (buffer, "#pragma omp dispatch"); + dump_omp_clauses (buffer, gimple_omp_dispatch_clauses (gs), spc, flags); + if (!gimple_seq_empty_p (gimple_omp_body (gs))) + { + newline_and_indent (buffer, spc + 2); + pp_left_brace (buffer); + pp_newline (buffer); + dump_gimple_seq (buffer, gimple_omp_body (gs), spc + 4, flags); + newline_and_indent (buffer, spc + 2); + pp_right_brace (buffer); + } + } +} + /* Dump a GIMPLE_OMP_TARGET tuple on the pretty_printer PP. */ static void @@ -2806,6 +2835,10 @@ pp_gimple_stmt_1 (pretty_printer *pp, const gimple *gs, int spc, dump_gimple_omp_scope (pp, gs, spc, flags); break; + case GIMPLE_OMP_DISPATCH: + dump_gimple_omp_dispatch(pp, gs, spc, flags); + break; + case GIMPLE_OMP_MASTER: case GIMPLE_OMP_SECTION: case GIMPLE_OMP_STRUCTURED_BLOCK: diff --git a/gcc/gimple-walk.cc b/gcc/gimple-walk.cc index 00520319aa9..73346df2830 100644 --- a/gcc/gimple-walk.cc +++ b/gcc/gimple-walk.cc @@ -707,6 +707,7 @@ walk_gimple_stmt (gimple_stmt_iterator *gsi, walk_stmt_fn callback_stmt, case GIMPLE_OMP_PARALLEL: case GIMPLE_OMP_TASK: case GIMPLE_OMP_SCOPE: + case GIMPLE_OMP_DISPATCH: case GIMPLE_OMP_SECTIONS: case GIMPLE_OMP_SINGLE: case GIMPLE_OMP_TARGET: diff --git a/gcc/gimple.cc b/gcc/gimple.cc index f7b313be40e..add4a645b45 100644 --- a/gcc/gimple.cc +++ b/gcc/gimple.cc @@ -1239,6 +1239,21 @@ gimple_build_omp_scope (gimple_seq body, tree clauses) return p; } +/* Build a GIMPLE_OMP_DISPATCH statement. + + BODY is the target function call to be dispatched. + CLAUSES are any of the OMP dispatch construct's clauses. */ + +gimple * +gimple_build_omp_dispatch (gimple_seq body, tree clauses) +{ + gimple *p = gimple_alloc (GIMPLE_OMP_DISPATCH, 0); + gimple_omp_dispatch_set_clauses (p, clauses); + if (body) + gimple_omp_set_body (p, body); + + return p; +} /* Build a GIMPLE_OMP_TARGET statement. @@ -2152,6 +2167,11 @@ gimple_copy (gimple *stmt) gimple_omp_scope_set_clauses (copy, t); goto copy_omp_body; + case GIMPLE_OMP_DISPATCH: + t = unshare_expr (gimple_omp_dispatch_clauses (stmt)); + gimple_omp_dispatch_set_clauses (copy, t); + goto copy_omp_body; + case GIMPLE_OMP_TARGET: { gomp_target *omp_target_stmt = as_a (stmt); diff --git a/gcc/gimple.def b/gcc/gimple.def index fbcd727f945..21c7405875d 100644 --- a/gcc/gimple.def +++ b/gcc/gimple.def @@ -350,6 +350,11 @@ DEFGSCODE(GIMPLE_OMP_SCAN, "gimple_omp_scan", GSS_OMP_SINGLE_LAYOUT) CLAUSES is an OMP_CLAUSE chain holding the associated clauses. */ DEFGSCODE(GIMPLE_OMP_SCOPE, "gimple_omp_scope", GSS_OMP_SINGLE_LAYOUT) +/* GIMPLE_OMP_DISPATCH represents #pragma omp dispatch + BODY is the target function call to be dispatched. + CLAUSES is an OMP_CLAUSE chain holding the associated clauses. */ +DEFGSCODE(GIMPLE_OMP_DISPATCH, "gimple_omp_dispatch", GSS_OMP_SINGLE_LAYOUT) + /* OMP_SECTION represents #pragma omp section. BODY is the sequence of statements in the section body. */ DEFGSCODE(GIMPLE_OMP_SECTION, "gimple_omp_section", GSS_OMP) diff --git a/gcc/gimple.h b/gcc/gimple.h index 4a6e0e97d1e..b6967e63de2 100644 --- a/gcc/gimple.h +++ b/gcc/gimple.h @@ -742,7 +742,7 @@ struct GTY((tag("GSS_OMP_CONTINUE"))) }; /* GIMPLE_OMP_SINGLE, GIMPLE_OMP_ORDERED, GIMPLE_OMP_TASKGROUP, - GIMPLE_OMP_SCAN, GIMPLE_OMP_MASKED, GIMPLE_OMP_SCOPE. */ + GIMPLE_OMP_SCAN, GIMPLE_OMP_MASKED, GIMPLE_OMP_SCOPE, GIMPLE_OMP_DISPATCH. */ struct GTY((tag("GSS_OMP_SINGLE_LAYOUT"))) gimple_statement_omp_single_layout : public gimple_statement_omp @@ -1591,6 +1591,7 @@ gomp_task *gimple_build_omp_task (gimple_seq, tree, tree, tree, tree, gimple *gimple_build_omp_section (gimple_seq); gimple *gimple_build_omp_structured_block (gimple_seq); gimple *gimple_build_omp_scope (gimple_seq, tree); +gimple *gimple_build_omp_dispatch (gimple_seq, tree); gimple *gimple_build_omp_master (gimple_seq); gimple *gimple_build_omp_masked (gimple_seq, tree); gimple *gimple_build_omp_taskgroup (gimple_seq, tree); @@ -1882,6 +1883,7 @@ gimple_has_substatements (gimple *g) case GIMPLE_OMP_PARALLEL: case GIMPLE_OMP_TASK: case GIMPLE_OMP_SCOPE: + case GIMPLE_OMP_DISPATCH: case GIMPLE_OMP_SECTIONS: case GIMPLE_OMP_SINGLE: case GIMPLE_OMP_TARGET: @@ -5434,6 +5436,34 @@ gimple_omp_scope_set_clauses (gimple *gs, tree clauses) = clauses; } +/* Return the clauses associated with OMP_DISPATCH statement GS. */ + +inline tree +gimple_omp_dispatch_clauses (const gimple *gs) +{ + GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH); + return static_cast (gs)->clauses; +} + +/* Return a pointer to the clauses associated with OMP dispatch statement + GS. */ + +inline tree * +gimple_omp_dispatch_clauses_ptr (gimple *gs) +{ + GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH); + return &static_cast (gs)->clauses; +} + +/* Set CLAUSES to be the clauses associated with OMP dispatch statement + GS. */ + +inline void +gimple_omp_dispatch_set_clauses (gimple *gs, tree clauses) +{ + GIMPLE_CHECK (gs, GIMPLE_OMP_DISPATCH); + static_cast (gs)->clauses = clauses; +} /* Return the kind of the OMP_FOR statemement G. */ @@ -6768,6 +6798,7 @@ gimple_return_set_retval (greturn *gs, tree retval) case GIMPLE_OMP_TARGET: \ case GIMPLE_OMP_TEAMS: \ case GIMPLE_OMP_SCOPE: \ + case GIMPLE_OMP_DISPATCH: \ case GIMPLE_OMP_SECTION: \ case GIMPLE_OMP_STRUCTURED_BLOCK: \ case GIMPLE_OMP_MASTER: \ diff --git a/gcc/gimplify.cc b/gcc/gimplify.cc index 827941b24db..0899b279513 100644 --- a/gcc/gimplify.cc +++ b/gcc/gimplify.cc @@ -162,7 +162,8 @@ enum omp_region_type { ORT_WORKSHARE = 0x00, ORT_TASKGROUP = 0x01, - ORT_SIMD = 0x04, + ORT_DISPATCH = 0x02, + ORT_SIMD = 0x04, ORT_PARALLEL = 0x08, ORT_COMBINED_PARALLEL = ORT_PARALLEL | 1, @@ -259,6 +260,7 @@ struct gimplify_omp_ctx bool order_concurrent; bool has_depend; bool in_for_exprs; + bool in_call_args; int defaultmap[5]; }; @@ -4072,23 +4074,137 @@ gimplify_call_expr (tree *expr_p, gimple_seq *pre_p, bool want_value) /* Gimplify the function arguments. */ if (nargs > 0) { + tree device_num = NULL_TREE; for (i = (PUSH_ARGS_REVERSED ? nargs - 1 : 0); - PUSH_ARGS_REVERSED ? i >= 0 : i < nargs; - PUSH_ARGS_REVERSED ? i-- : i++) - { - enum gimplify_status t; + PUSH_ARGS_REVERSED ? i >= 0 : i < nargs; + PUSH_ARGS_REVERSED ? i-- : i++) + { + enum gimplify_status t; - /* Avoid gimplifying the second argument to va_start, which needs to - be the plain PARM_DECL. */ - if ((i != 1) || !builtin_va_start_p) - { - t = gimplify_arg (&CALL_EXPR_ARG (*expr_p, i), pre_p, - EXPR_LOCATION (*expr_p), ! returns_twice); + /* Avoid gimplifying the second argument to va_start, which needs to + be the plain PARM_DECL. */ + if ((i != 1) || !builtin_va_start_p) + { + tree *arg_p = &CALL_EXPR_ARG (*expr_p, i); + tree adjust_args_list; + if (flag_openmp && gimplify_omp_ctxp != NULL + && gimplify_omp_ctxp->code == OMP_DISPATCH + && !gimplify_omp_ctxp->in_call_args + && !integer_zerop (*arg_p) + && EXPR_P (CALL_EXPR_FN (*expr_p)) + && DECL_P (TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0)) + && (adjust_args_list = lookup_attribute ( + "omp declare variant variant adjust_args", + DECL_ATTRIBUTES ( + TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0)))) + != NULL_TREE) + { + tree arg_types = TYPE_ARG_TYPES ( + TREE_TYPE (TREE_OPERAND (CALL_EXPR_FN (*expr_p), 0))); - if (t == GS_ERROR) - ret = GS_ERROR; - } - } + if (arg_types != NULL_TREE) + { + for (int param_idx = 0; param_idx < i; param_idx++) + arg_types = TREE_CHAIN (arg_types); + + bool need_device_ptr = false; + for (tree arg + = TREE_PURPOSE (TREE_VALUE (adjust_args_list)); + arg != NULL; arg = TREE_CHAIN (arg)) + { + if (TREE_VALUE (arg) + && TREE_CODE (TREE_VALUE (arg)) == INTEGER_CST + && wi::eq_p (i, wi::to_wide (TREE_VALUE (arg)))) + { + need_device_ptr = true; + break; + } + } + + if (need_device_ptr) + { + bool is_device_ptr = false; + for (tree c = gimplify_omp_ctxp->clauses; c; + c = TREE_CHAIN (c)) + { + if (OMP_CLAUSE_CODE (c) + == OMP_CLAUSE_IS_DEVICE_PTR) + { + tree decl1 = DECL_NAME (OMP_CLAUSE_DECL (c)); + tree decl2 + = tree_strip_nop_conversions (*arg_p); + if (TREE_CODE (decl2) == ADDR_EXPR) + decl2 = TREE_OPERAND (decl2, 0); + if (VAR_P (decl2) + || TREE_CODE (decl2) == PARM_DECL) + { + decl2 = DECL_NAME (decl2); + if (decl1 == decl2) + is_device_ptr = true; + } + } + else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE) + device_num = OMP_CLAUSE_OPERAND (c, 0); + } + + if (!is_device_ptr) + { + if (device_num == NULL_TREE) + { + // device_num = omp_get_default_device () + tree fn = builtin_decl_explicit ( + BUILT_IN_OMP_GET_DEFAULT_DEVICE); + gcall *call = gimple_build_call (fn, 0); + device_num = create_tmp_var ( + gimple_call_return_type (call)); + gimple_call_set_lhs (call, device_num); + gimplify_seq_add_stmt (pre_p, call); + } + + // mapped_arg = omp_get_mapped_ptr (arg, + // device_num) + tree fn = builtin_decl_explicit ( + BUILT_IN_OMP_GET_MAPPED_PTR); + gimplify_arg (arg_p, pre_p, loc); + gimplify_arg (&device_num, pre_p, loc); + call + = gimple_build_call (fn, 2, *arg_p, device_num); + tree mapped_arg = create_tmp_var ( + gimple_call_return_type (call)); + gimple_call_set_lhs (call, mapped_arg); + gimplify_seq_add_stmt (pre_p, call); + + *arg_p = mapped_arg; + + // gimplify_call_expr might be called several + // times on the same call, which would result in + // duplicated calls to omp_get_default_device and + // omp_get_mapped_ptr. To prevent that, we mark + // already mapped arguments as device pointers. + gcc_checking_assert (gimplify_omp_ctxp->code + == OMP_DISPATCH); + tree c + = build_omp_clause (input_location, + OMP_CLAUSE_IS_DEVICE_PTR); + OMP_CLAUSE_DECL (c) = *arg_p; + OMP_CLAUSE_CHAIN (c) = gimplify_omp_ctxp->clauses; + gimplify_omp_ctxp->clauses = c; + } + } + } + } + + if (gimplify_omp_ctxp && gimplify_omp_ctxp->code == OMP_DISPATCH) + gimplify_omp_ctxp->in_call_args = true; + t = gimplify_arg (arg_p, pre_p, EXPR_LOCATION (*expr_p), + !returns_twice); + if (gimplify_omp_ctxp && gimplify_omp_ctxp->code == OMP_DISPATCH) + gimplify_omp_ctxp->in_call_args = false; + + if (t == GS_ERROR) + ret = GS_ERROR; + } + } } /* Gimplify the static chain. */ @@ -6388,6 +6504,7 @@ is_gimple_stmt (tree t) case OACC_LOOP: case OMP_SCAN: case OMP_SCOPE: + case OMP_DISPATCH: case OMP_SECTIONS: case OMP_SECTION: case OMP_STRUCTURED_BLOCK: @@ -13208,6 +13325,21 @@ gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p, break; } } + else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE + && code == OMP_DISPATCH) + { + bool saved_into_ssa = gimplify_ctxp->into_ssa; + gimplify_ctxp->into_ssa = false; + if (gimplify_expr (&OMP_CLAUSE_DEVICE_ID (c), pre_p, NULL, + is_gimple_val, fb_rvalue) + == GS_ERROR) + remove = true; + else if (DECL_P (OMP_CLAUSE_DEVICE_ID (c))) + omp_add_variable (ctx, OMP_CLAUSE_DEVICE_ID (c), + GOVD_SHARED | GOVD_SEEN); + gimplify_ctxp->into_ssa = saved_into_ssa; + break; + } /* Fall through. */ case OMP_CLAUSE_PRIORITY: @@ -13448,6 +13580,14 @@ gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p, } break; + case OMP_CLAUSE_NOVARIANTS: + OMP_CLAUSE_NOVARIANTS_EXPR (c) + = gimple_boolify (OMP_CLAUSE_NOVARIANTS_EXPR (c)); + break; + case OMP_CLAUSE_NOCONTEXT: + OMP_CLAUSE_NOCONTEXT_EXPR (c) + = gimple_boolify (OMP_CLAUSE_NOCONTEXT_EXPR (c)); + break; case OMP_CLAUSE_NOHOST: default: gcc_unreachable (); @@ -14775,6 +14915,54 @@ gimplify_adjust_omp_clauses (gimple_seq *pre_p, gimple_seq body, tree *list_p, delete_omp_context (ctx); } +/* Try to evaluate a novariants clause. Return 1 if true, 0 if false or absent, + * -1 if run-time evaluation is needed. */ + +int +omp_has_novariants (void) +{ + struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp; + if (ctx != NULL && ctx->code == OMP_DISPATCH && !ctx->in_call_args) + { + tree c = omp_find_clause (ctx->clauses, OMP_CLAUSE_NOVARIANTS); + if (c != NULL_TREE) + { + if (integer_nonzerop (OMP_CLAUSE_NOVARIANTS_EXPR (c))) + return 1; + else if (integer_zerop (OMP_CLAUSE_NOVARIANTS_EXPR (c))) + return 0; + else + return -1; + } + return 0; + } + return 0; +} + +/* Try to evaluate a nocontext clause. Return 1 if true, 0 if false or absent, + * -1 if run-time evaluation is needed. */ + +static int +omp_has_nocontext (void) +{ + struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp; + if (ctx != NULL && ctx->code == OMP_DISPATCH) + { + tree c = omp_find_clause (ctx->clauses, OMP_CLAUSE_NOCONTEXT); + if (c != NULL_TREE) + { + if (integer_nonzerop (OMP_CLAUSE_NOCONTEXT_EXPR (c))) + return 1; + else if (integer_zerop (OMP_CLAUSE_NOCONTEXT_EXPR (c))) + return 0; + else + return -1; + } + return 0; + } + return 0; +} + /* Return 0 if CONSTRUCTS selectors don't match the OpenMP context, -1 if unknown yet (simd is involved, won't be known until vectorization) and 1 if they do. If SCORES is non-NULL, it should point to an array @@ -14802,9 +14990,9 @@ omp_construct_selector_matches (enum tree_code *constructs, int nconstructs, == ORT_TARGET && ctx->code == OMP_TARGET) || ((ctx->region_type & ORT_TEAMS) && ctx->code == OMP_TEAMS) || (ctx->region_type == ORT_WORKSHARE && ctx->code == OMP_FOR) - || (ctx->region_type == ORT_SIMD - && ctx->code == OMP_SIMD - && !omp_find_clause (ctx->clauses, OMP_CLAUSE_BIND))) + || (ctx->region_type == ORT_SIMD && ctx->code == OMP_SIMD + && !omp_find_clause (ctx->clauses, OMP_CLAUSE_BIND)) + || (ctx->code == OMP_DISPATCH && omp_has_nocontext () != 1)) { ++cnt; if (scores) @@ -17915,6 +18103,272 @@ gimplify_omp_ordered (tree expr, gimple_seq body) return gimple_build_omp_ordered (body, OMP_ORDERED_CLAUSES (expr)); } +/* Callback for walk_tree to find an IFN_GOMP_DISPATCH. */ + +static tree +find_ifn_gomp_dispatch (tree *tp, int *, void *modify) +{ + tree t = *tp; + + if (TREE_CODE (t) == CALL_EXPR && CALL_EXPR_IFN (t) == IFN_GOMP_DISPATCH) + { + *tp = CALL_EXPR_ARG (t, 0); + return *(tree *) modify ? *(tree *) modify : *tp; + } + + if (TREE_CODE (t) == MODIFY_EXPR) + *(tree *) modify = *tp; + + return NULL_TREE; +} + +/* Gimplify an OMP_DISPATCH construct. */ + +static enum gimplify_status +gimplify_omp_dispatch (tree *expr_p, gimple_seq *pre_p) +{ + tree expr = *expr_p; + gimple_seq body = NULL; + + gimplify_scan_omp_clauses (&OMP_DISPATCH_CLAUSES (expr), pre_p, ORT_DISPATCH, + OMP_DISPATCH); + push_gimplify_context (); + + // If device clause, adjust ICV + tree device + = omp_find_clause (OMP_DISPATCH_CLAUSES (expr), OMP_CLAUSE_DEVICE); + tree saved_device_icv = NULL_TREE; + if (device + && (TREE_CODE (OMP_CLAUSE_DEVICE_ID (device)) != INTEGER_CST + || !wi::eq_p (wi::to_wide (OMP_CLAUSE_DEVICE_ID (device)), + -1 /* omp_initial_device */))) + { + // Save current default-device-var ICV + saved_device_icv = create_tmp_var (integer_type_node); + tree fn = builtin_decl_explicit (BUILT_IN_OMP_GET_DEFAULT_DEVICE); + gcall *call = gimple_build_call (fn, 0); + gimple_call_set_lhs (call, saved_device_icv); + gimplify_seq_add_stmt (&body, call); + + // Set default device + fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE); + call = gimple_build_call (fn, 1, OMP_CLAUSE_DEVICE_ID (device)); + gimplify_seq_add_stmt (&body, call); + } + + // If the novariants and nocontext clauses are not compile-time constants, + // we need to generate code for all possible cases: + // if (novariants) // implies nocontext + // base() + // else if (nocontext) + // variant1() + // else + // variant2() + tree *dispatch_body_p = &OMP_DISPATCH_BODY (expr); + if (TREE_CODE (*dispatch_body_p) == BIND_EXPR) + dispatch_body_p = &BIND_EXPR_BODY (*dispatch_body_p); + tree dispatch_body = *dispatch_body_p; + + // Look for IFN_GOMP_DISPATCH and extract the base function call + tree base_call_expr = NULL_TREE; + if (TREE_CODE (dispatch_body) == STATEMENT_LIST) + for (tree_stmt_iterator tsi = tsi_start (dispatch_body); !tsi_end_p (tsi); + tsi_next (&tsi)) + { + tree modify = NULL_TREE; + tree stmt = tsi_stmt (tsi); + base_call_expr + = walk_tree (&stmt, find_ifn_gomp_dispatch, &modify, NULL); + if (base_call_expr != NULL_TREE) + { + tsi_link_before (&tsi, base_call_expr, TSI_CONTINUE_LINKING); + tsi_next (&tsi); + tsi_delink (&tsi); + break; + } + } + else + { + tree modify = NULL_TREE; + base_call_expr + = walk_tree (dispatch_body_p, find_ifn_gomp_dispatch, &modify, NULL); + } + gcc_assert (base_call_expr != NULL_TREE); + + tree dst = NULL_TREE; + if (TREE_CODE (base_call_expr) == MODIFY_EXPR) + { + dst = TREE_OPERAND (base_call_expr, 0); + base_call_expr = TREE_OPERAND (base_call_expr, 1); + } + while (TREE_CODE (base_call_expr) == FLOAT_EXPR + || TREE_CODE (base_call_expr) == CONVERT_EXPR + || TREE_CODE (base_call_expr) == COMPLEX_EXPR + || TREE_CODE (base_call_expr) == INDIRECT_REF + || TREE_CODE (base_call_expr) == NOP_EXPR) + base_call_expr = TREE_OPERAND (base_call_expr, 0); + + tree base_fndecl = get_callee_fndecl (base_call_expr); + if (base_fndecl != NULL_TREE) + { + if (DECL_VIRTUAL_P (base_fndecl)) + { + error_at ( + EXPR_LOCATION (base_call_expr), + "%qD is a virtual function but only a direct call is allowed " + "in a dispatch construct", + DECL_NAME (base_fndecl)); + } + + tree variant_fndecl = omp_resolve_declare_variant (base_fndecl); + if (base_fndecl != variant_fndecl + && (omp_has_novariants () == -1 || omp_has_nocontext () == -1)) + { + tree novariants_clause = NULL_TREE, nocontext_clause = NULL_TREE, + novariants_cond = NULL_TREE, nocontext_cond = NULL_TREE; + for (tree c = OMP_DISPATCH_CLAUSES (expr); c; c = TREE_CHAIN (c)) + { + if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOVARIANTS + && !integer_zerop (OMP_CLAUSE_NOVARIANTS_EXPR (c))) + { + gcc_assert (novariants_cond == NULL_TREE); + novariants_clause = c; + novariants_cond = OMP_CLAUSE_NOVARIANTS_EXPR (c); + } + else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOCONTEXT + && !integer_zerop (OMP_CLAUSE_NOCONTEXT_EXPR (c))) + { + gcc_assert (nocontext_cond == NULL_TREE); + nocontext_clause = c; + nocontext_cond = OMP_CLAUSE_NOCONTEXT_EXPR (c); + } + } + gcc_assert (novariants_cond != NULL_TREE + || nocontext_cond != NULL_TREE); + + enum gimplify_status ret + = gimplify_expr (&novariants_cond, &body, NULL, is_gimple_val, + fb_rvalue); + if (ret == GS_ERROR || ret == GS_UNHANDLED) + return ret; + ret = gimplify_expr (&nocontext_cond, &body, NULL, is_gimple_val, + fb_rvalue); + if (ret == GS_ERROR || ret == GS_UNHANDLED) + return ret; + + tree end_label = create_artificial_label (UNKNOWN_LOCATION); + + if (novariants_cond != NULL_TREE) + { + tree base_label = create_artificial_label (UNKNOWN_LOCATION); + tree cond_label = create_artificial_label (UNKNOWN_LOCATION); + gcond *novariants_cond_stmt + = gimple_build_cond_from_tree (novariants_cond, base_label, + cond_label); + gimplify_seq_add_stmt (&body, novariants_cond_stmt); + + gimplify_seq_add_stmt (&body, gimple_build_label (base_label)); + tree base_call_expr2 = copy_node (base_call_expr); + if (TREE_CODE (dispatch_body) == MODIFY_EXPR) + { + base_call_expr2 = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, + base_call_expr2); + } + OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause) + = boolean_true_node; + gimplify_and_add (base_call_expr2, &body); + gimplify_seq_add_stmt (&body, gimple_build_goto (end_label)); + + OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause) + = boolean_false_node; + gimplify_seq_add_stmt (&body, gimple_build_label (cond_label)); + } + + if (nocontext_cond != NULL_TREE) + { + tree variant1_label = create_artificial_label (UNKNOWN_LOCATION); + tree variant2_label = create_artificial_label (UNKNOWN_LOCATION); + gcond *nocontext_cond_stmt + = gimple_build_cond_from_tree (nocontext_cond, variant1_label, + variant2_label); + gimplify_seq_add_stmt (&body, nocontext_cond_stmt); + + gimplify_seq_add_stmt (&body, + gimple_build_label (variant1_label)); + tree variant_call_expr = copy_node (base_call_expr); + if (TREE_CODE (dispatch_body) == MODIFY_EXPR) + { + variant_call_expr = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, + variant_call_expr); + } + OMP_CLAUSE_NOCONTEXT_EXPR (nocontext_clause) = boolean_true_node; + gimplify_and_add (variant_call_expr, &body); + gimplify_seq_add_stmt (&body, gimple_build_goto (end_label)); + OMP_CLAUSE_NOCONTEXT_EXPR (nocontext_clause) = boolean_false_node; + gimplify_seq_add_stmt (&body, + gimple_build_label (variant2_label)); + } + + tree variant_call_expr = base_call_expr; + if (TREE_CODE (dispatch_body) == MODIFY_EXPR) + { + variant_call_expr + = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, variant_call_expr); + } + gimplify_and_add (variant_call_expr, &body); + gimplify_seq_add_stmt (&body, gimple_build_label (end_label)); + } + else + gimplify_and_add (OMP_DISPATCH_BODY (expr), &body); + } + else + gimplify_and_add (OMP_DISPATCH_BODY (expr), &body); + + // Restore default-device-var ICV + if (saved_device_icv != NULL_TREE) + { + tree fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE); + gcall *call = gimple_build_call (fn, 1, saved_device_icv); + gimplify_seq_add_stmt (&body, call); + } + + // Wrap dispatch body into a bind + gimple *bind = gimple_build_bind (NULL_TREE, body, NULL_TREE); + pop_gimplify_context (bind); + + // Manually tear down context created by gimplify_scan_omp_clauses to avoid a + // call to gimplify_adjust_omp_clauses + gimplify_omp_ctx *ctx = gimplify_omp_ctxp; + if (ctx != NULL) + { + gcc_assert (ctx->code == OMP_DISPATCH); + gimplify_omp_ctxp = ctx->outer_context; + delete_omp_context (ctx); + } + + // Remove nowait as it has no effect on dispatch (OpenMP 5.2), device as it + // has been handled above, and depend as the front end handled it by inserting + // taskwait. + tree *dispatch_clauses_ptr = &OMP_DISPATCH_CLAUSES (expr); + for (tree c = *dispatch_clauses_ptr; c; c = *dispatch_clauses_ptr) + { + if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOWAIT + || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND + || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE) + { + *dispatch_clauses_ptr = OMP_CLAUSE_CHAIN (c); + break; + } + else + dispatch_clauses_ptr = &OMP_CLAUSE_CHAIN (c); + } + + gimple *stmt = gimple_build_omp_dispatch (bind, OMP_DISPATCH_CLAUSES (expr)); + gimplify_seq_add_stmt (pre_p, stmt); + *expr_p = NULL_TREE; + return GS_ALL_DONE; +} + /* Convert the GENERIC expression tree *EXPR_P to GIMPLE. If the expression produces a value to be used as an operand inside a GIMPLE statement, the value will be stored back in *EXPR_P. This value will @@ -18853,6 +19307,10 @@ gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p, ret = gimplify_omp_atomic (expr_p, pre_p); break; + case OMP_DISPATCH: + ret = gimplify_omp_dispatch (expr_p, pre_p); + break; + case TRANSACTION_EXPR: ret = gimplify_transaction (expr_p, pre_p); break; @@ -19178,7 +19636,8 @@ gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p, && code != OMP_SECTION && code != OMP_STRUCTURED_BLOCK && code != OMP_SINGLE - && code != OMP_SCOPE); + && code != OMP_SCOPE + && code != OMP_DISPATCH); } #endif diff --git a/gcc/gimplify.h b/gcc/gimplify.h index ac3cc8eb552..2e912677022 100644 --- a/gcc/gimplify.h +++ b/gcc/gimplify.h @@ -77,6 +77,7 @@ extern enum gimplify_status gimplify_expr (tree *, gimple_seq *, gimple_seq *, bool (*) (tree), fallback_t); int omp_construct_selector_matches (enum tree_code *, int, int *); +int omp_has_novariants (void); extern void gimplify_type_sizes (tree, gimple_seq *); extern void gimplify_one_sizepos (tree *, gimple_seq *); diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc index c522a53a433..a45b3291800 100644 --- a/gcc/internal-fn.cc +++ b/gcc/internal-fn.cc @@ -689,6 +689,14 @@ expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *) gcc_unreachable (); } +/* This should get expanded in gimplify_omp_dispatch. */ + +static void +expand_GOMP_DISPATCH (internal_fn, gcall *) +{ + gcc_unreachable (); +} + /* This should get expanded in the sanopt pass. */ static void diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def index 09b7844d094..e993c99c558 100644 --- a/gcc/internal-fn.def +++ b/gcc/internal-fn.def @@ -476,6 +476,7 @@ DEF_INTERNAL_FN (GOMP_SIMD_VF, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMD_LAST_LANE, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMD_ORDERED_START, ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMD_ORDERED_END, ECF_LEAF | ECF_NOTHROW, NULL) +DEF_INTERNAL_FN (GOMP_DISPATCH, ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (LOOP_VECTORIZED, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (LOOP_DIST_ALIAS, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (ANNOTATE, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL) diff --git a/gcc/omp-builtins.def b/gcc/omp-builtins.def index 7b49ef1c0e5..c70e077063b 100644 --- a/gcc/omp-builtins.def +++ b/gcc/omp-builtins.def @@ -80,6 +80,12 @@ DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_TEAM_NUM, "omp_get_team_num", BT_FN_INT, ATTR_CONST_NOTHROW_LEAF_LIST) DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_NUM_TEAMS, "omp_get_num_teams", BT_FN_INT, ATTR_CONST_NOTHROW_LEAF_LIST) +DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_MAPPED_PTR, "omp_get_mapped_ptr", + BT_FN_PTR_CONST_PTR_INT, ATTR_NOTHROW_LEAF_LIST) +DEF_GOMP_BUILTIN (BUILT_IN_OMP_GET_DEFAULT_DEVICE, "omp_get_default_device", + BT_FN_INT, ATTR_NOTHROW_LEAF_LIST) +DEF_GOMP_BUILTIN (BUILT_IN_OMP_SET_DEFAULT_DEVICE, "omp_set_default_device", + BT_FN_INT, ATTR_NOTHROW_LEAF_LIST) DEF_GOMP_BUILTIN (BUILT_IN_GOMP_ATOMIC_START, "GOMP_atomic_start", BT_FN_VOID, ATTR_NOTHROW_LEAF_LIST) diff --git a/gcc/omp-general.cc b/gcc/omp-general.cc index 72fb7f92ff7..adf622954d7 100644 --- a/gcc/omp-general.cc +++ b/gcc/omp-general.cc @@ -1063,7 +1063,7 @@ omp_construct_traits_to_codes (tree ctx, int nconstructs, /* Order must match the OMP_TRAIT_CONSTRUCT_* enumerators in enum omp_ts_code. */ static enum tree_code code_map[] - = { OMP_TARGET, OMP_TEAMS, OMP_PARALLEL, OMP_FOR, OMP_SIMD }; + = { OMP_TARGET, OMP_TEAMS, OMP_PARALLEL, OMP_FOR, OMP_SIMD, OMP_DISPATCH }; for (tree ts = ctx; ts; ts = TREE_CHAIN (ts), i--) { @@ -1267,10 +1267,14 @@ struct omp_ts_info omp_ts_map[] = OMP_TRAIT_PROPERTY_CLAUSE_LIST, false, NULL }, + { "dispatch", + (1 << OMP_TRAIT_SET_CONSTRUCT), + OMP_TRAIT_PROPERTY_NONE, false, + NULL + }, { NULL, 0, OMP_TRAIT_PROPERTY_NONE, false, NULL } /* OMP_TRAIT_LAST */ }; - /* Return a name from PROP, a property in selectors accepting name lists. */ @@ -2593,6 +2597,9 @@ omp_resolve_declare_variant (tree base) if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0) return omp_resolve_late_declare_variant (base); + if (omp_has_novariants () == 1) + return base; + auto_vec variants; auto_vec defer; bool any_deferred = false; @@ -2739,6 +2746,8 @@ omp_resolve_declare_variant (tree base) (*slot)->variants = entry.variants; tree alt = build_decl (DECL_SOURCE_LOCATION (base), FUNCTION_DECL, DECL_NAME (base), TREE_TYPE (base)); + if (DECL_ASSEMBLER_NAME_SET_P (base)) + SET_DECL_ASSEMBLER_NAME (alt, DECL_ASSEMBLER_NAME (base)); DECL_ARTIFICIAL (alt) = 1; DECL_IGNORED_P (alt) = 1; TREE_STATIC (alt) = 1; diff --git a/gcc/omp-low.cc b/gcc/omp-low.cc index 70a2c108fbc..32d252c6b39 100644 --- a/gcc/omp-low.cc +++ b/gcc/omp-low.cc @@ -4206,6 +4206,11 @@ scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p, scan_omp (gimple_omp_body_ptr (stmt), ctx); break; + case GIMPLE_OMP_DISPATCH: + ctx = new_omp_context (stmt, ctx); + scan_omp (gimple_omp_body_ptr (stmt), ctx); + break; + case GIMPLE_OMP_SECTIONS: scan_omp_sections (as_a (stmt), ctx); break; @@ -8948,6 +8953,31 @@ lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx) if (BLOCK_VARS (block)) TREE_USED (block) = 1; } + +/* Lower code for an OMP dispatch directive. */ + +static void +lower_omp_dispatch (gimple_stmt_iterator *gsi_p, omp_context *ctx) +{ + tree block; + gimple *stmt = gsi_stmt (*gsi_p); + gbind *bind; + + push_gimplify_context (); + + block = make_node (BLOCK); + bind = gimple_build_bind (NULL, NULL, block); + gsi_replace (gsi_p, bind, true); + + lower_omp (gimple_omp_body_ptr (stmt), ctx); + gimple_bind_set_body (bind, maybe_catch_exception (gimple_omp_body (stmt))); + + pop_gimplify_context (bind); + + gimple_bind_append_vars (bind, ctx->block_vars); + BLOCK_VARS (block) = ctx->block_vars; +} + /* Expand code for an OpenMP master or masked directive. */ static void @@ -14579,6 +14609,11 @@ lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx) gcc_assert (ctx); lower_omp_scope (gsi_p, ctx); break; + case GIMPLE_OMP_DISPATCH: + ctx = maybe_lookup_ctx (stmt); + gcc_assert (ctx); + lower_omp_dispatch (gsi_p, ctx); + break; case GIMPLE_OMP_SINGLE: ctx = maybe_lookup_ctx (stmt); gcc_assert (ctx); diff --git a/gcc/tree-inline.cc b/gcc/tree-inline.cc index d16ad6a28de..d89b7ce74aa 100644 --- a/gcc/tree-inline.cc +++ b/gcc/tree-inline.cc @@ -1680,6 +1680,12 @@ remap_gimple_stmt (gimple *stmt, copy_body_data *id) (s1, gimple_omp_scope_clauses (stmt)); break; + case GIMPLE_OMP_DISPATCH: + s1 = remap_gimple_seq (gimple_omp_body (stmt), id); + copy = gimple_build_omp_dispatch (s1, + gimple_omp_dispatch_clauses (stmt)); + break; + case GIMPLE_OMP_TASKGROUP: s1 = remap_gimple_seq (gimple_omp_body (stmt), id); copy = gimple_build_omp_taskgroup @@ -4610,6 +4616,7 @@ estimate_num_insns (gimple *stmt, eni_weights *weights) case GIMPLE_OMP_MASTER: case GIMPLE_OMP_MASKED: case GIMPLE_OMP_SCOPE: + case GIMPLE_OMP_DISPATCH: case GIMPLE_OMP_TASKGROUP: case GIMPLE_OMP_ORDERED: case GIMPLE_OMP_SCAN: