mirror/gcc
1
0
Fork 0
mirror of https://github.com/gcc-mirror/gcc.git synced 2024-11-21 13:40:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add suggested_epilogue_mode to vector costs

Browse Source 
The following enables targets to suggest the vector mode to be used
preferably for the epilogue of a vectorized loop.  The patch also
enables more than one vectorized epilogue in case the target suggests
a vector mode for the epilogue of a vector epilogue.

	* tree-vectorizer.h (vector_costs::suggested_epilogue_mode): New.
	(vector_costs::m_suggested_epilogue_mode): Likewise.
	(vector_costs::vector_costs): Initialize m_suggested_epilogue_mode.
	* tree-vect-loop.cc (vect_analyze_loop): Honor the target
	suggested prefered epilogue mode and support vector epilogues
	of vector epilogues if requested.
This commit is contained in:
Richard Biener 2024-11-08 11:12:06 +01:00 committed by Richard Biener
parent 61cd1c43b8
commit 82d955b0a8
2 changed files with 93 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
gcc/tree-vect-loop.cc
View File

@ -3714,72 +3714,102 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
array may contain length-agnostic and length-specific modes. Their
ordering is not guaranteed, so we could end up picking a mode for the main
loop that is after the epilogue's optimal mode. */
vector_modes[0] = autodetected_vector_mode;
if (!unlimited_cost_model (loop)
&& first_loop_vinfo->vector_costs->suggested_epilogue_mode () != VOIDmode)
{
vector_modes[0]
= first_loop_vinfo->vector_costs->suggested_epilogue_mode ();
cached_vf_per_mode[0] = 0;
}
else
vector_modes[0] = autodetected_vector_mode;
mode_i = 0;
bool supports_partial_vectors =
partial_vectors_supported_p () && param_vect_partial_vector_usage != 0;
poly_uint64 first_vinfo_vf = LOOP_VINFO_VECT_FACTOR (first_loop_vinfo);
while (1)
loop_vec_info orig_loop_vinfo = first_loop_vinfo;
do
{
/* If the target does not support partial vectors we can shorten the
number of modes to analyze for the epilogue as we know we can't pick a
mode that would lead to a VF at least as big as the
FIRST_VINFO_VF. */
if (!supports_partial_vectors
&& maybe_ge (cached_vf_per_mode[mode_i], first_vinfo_vf))
while (1)
{
mode_i++;
/* If the target does not support partial vectors we can shorten the
number of modes to analyze for the epilogue as we know we can't
pick a mode that would lead to a VF at least as big as the
FIRST_VINFO_VF. */
if (!supports_partial_vectors
&& maybe_ge (cached_vf_per_mode[mode_i], first_vinfo_vf))
{
mode_i++;
if (mode_i == vector_modes.length ())
break;
continue;
}
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"***** Re-trying epilogue analysis with vector "
"mode %s\n", GET_MODE_NAME (vector_modes[mode_i]));
bool fatal;
opt_loop_vec_info loop_vinfo
= vect_analyze_loop_1 (loop, shared, &loop_form_info,
orig_loop_vinfo,
vector_modes, mode_i,
autodetected_vector_mode, fatal);
if (fatal)
break;
if (loop_vinfo)
{
if (pick_lowest_cost_p
&& orig_loop_vinfo->epilogue_vinfo
&& vect_joust_loop_vinfos (loop_vinfo,
orig_loop_vinfo->epilogue_vinfo))
{
gcc_assert (vect_epilogues);
delete orig_loop_vinfo->epilogue_vinfo;
orig_loop_vinfo->epilogue_vinfo = nullptr;
}
if (!orig_loop_vinfo->epilogue_vinfo)
orig_loop_vinfo->epilogue_vinfo = loop_vinfo;
else
{
delete loop_vinfo;
loop_vinfo = opt_loop_vec_info::success (NULL);
}
/* For now only allow one epilogue loop, but allow
pick_lowest_cost_p to replace it, so commit to the
first epilogue if we have no reason to try alternatives. */
if (!pick_lowest_cost_p)
break;
}
if (mode_i == vector_modes.length ())
break;
continue;
}
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"***** Re-trying epilogue analysis with vector "
"mode %s\n", GET_MODE_NAME (vector_modes[mode_i]));
bool fatal;
opt_loop_vec_info loop_vinfo
= vect_analyze_loop_1 (loop, shared, &loop_form_info,
first_loop_vinfo,
vector_modes, mode_i,
autodetected_vector_mode, fatal);
if (fatal)
orig_loop_vinfo = orig_loop_vinfo->epilogue_vinfo;
if (!orig_loop_vinfo)
break;
if (loop_vinfo)
/* When we selected a first vectorized epilogue, see if the target
suggests to have another one. */
if (!unlimited_cost_model (loop)
&& (orig_loop_vinfo->vector_costs->suggested_epilogue_mode ()
!= VOIDmode))
{
if (pick_lowest_cost_p
&& first_loop_vinfo->epilogue_vinfo
&& vect_joust_loop_vinfos (loop_vinfo,
first_loop_vinfo->epilogue_vinfo))
{
gcc_assert (vect_epilogues);
delete first_loop_vinfo->epilogue_vinfo;
first_loop_vinfo->epilogue_vinfo = nullptr;
}
if (!first_loop_vinfo->epilogue_vinfo)
first_loop_vinfo->epilogue_vinfo = loop_vinfo;
else
{
delete loop_vinfo;
loop_vinfo = opt_loop_vec_info::success (NULL);
}
/* For now only allow one epilogue loop, but allow
pick_lowest_cost_p to replace it, so commit to the
first epilogue if we have no reason to try alternatives. */
if (!pick_lowest_cost_p)
break;
vector_modes[0]
= orig_loop_vinfo->vector_costs->suggested_epilogue_mode ();
cached_vf_per_mode[0] = 0;
mode_i = 0;
}
if (mode_i == vector_modes.length ())
else
break;
}
while (1);
if (first_loop_vinfo->epilogue_vinfo)
{

15
gcc/tree-vectorizer.h
View File

@ -1669,6 +1669,7 @@ public:
unsigned int outside_cost () const;
unsigned int total_cost () const;
unsigned int suggested_unroll_factor () const;
machine_mode suggested_epilogue_mode () const;
protected:
unsigned int record_stmt_cost (stmt_vec_info, vect_cost_model_location,
@ -1691,6 +1692,10 @@ protected:
/* The suggested unrolling factor determined at finish_cost. */
unsigned int m_suggested_unroll_factor;
/* The suggested mode to be used for a vectorized epilogue or VOIDmode,
determined at finish_cost. */
machine_mode m_suggested_epilogue_mode;
/* True if finish_cost has been called. */
bool m_finished;
};
@ -1704,6 +1709,7 @@ vector_costs::vector_costs (vec_info *vinfo, bool costing_for_scalar)
m_costing_for_scalar (costing_for_scalar),
m_costs (),
m_suggested_unroll_factor(1),
m_suggested_epilogue_mode(VOIDmode),
m_finished (false)
{
}
@ -1761,6 +1767,15 @@ vector_costs::suggested_unroll_factor () const
return m_suggested_unroll_factor;
}
/* Return the suggested epilogue mode. */
inline machine_mode
vector_costs::suggested_epilogue_mode () const
{
gcc_checking_assert (m_finished);
return m_suggested_epilogue_mode;
}
#define VECT_MAX_COST 1000
/* The maximum number of intermediate steps required in multi-step type