Skip to content

Commit

Permalink
api: eltwise: remove bounded_relu
Browse files Browse the repository at this point in the history
  • Loading branch information
@dzarukin @densamoilov
dzarukin authored and densamoilov committed Oct 3, 2022
1 parent 22b5364 commit 7de83da
Show file tree
Hide file tree
Showing 50 changed files with 119 additions and 278 deletions.
2 changes: 1 addition & 1 deletion doc/primitives/eltwise.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ The following operations are supported:
| Operation | oneDNN algorithm kind | Forward formula | Backward formula (from src) | Backward formula (from dst) |
| :-- | :-- | :-- | :-- | :-- |
| abs | #dnnl_eltwise_abs | \f$ d = \begin{cases} s & \text{if}\ s > 0 \\ -s & \text{if}\ s \leq 0 \end{cases} \f$ | \f$ ds = \begin{cases} dd & \text{if}\ s > 0 \\ -dd & \text{if}\ s < 0 \\ 0 & \text{if}\ s = 0 \end{cases} \f$ | -- |
| bounded_relu | #dnnl_eltwise_bounded_relu | \f$ d = \begin{cases} \alpha & \text{if}\ s > \alpha \geq 0 \\ s & \text{if}\ 0 < s \leq \alpha \\ 0 & \text{if}\ s \leq 0 \end{cases} \f$ | \f$ ds = \begin{cases} dd & \text{if}\ 0 < s \leq \alpha, \\ 0 & \text{otherwise}\ \end{cases} \f$ | -- |
| clip | #dnnl_eltwise_clip | \f$ d = \begin{cases} \beta & \text{if}\ s > \beta \geq \alpha \\ s & \text{if}\ \alpha < s \leq \beta \\ \alpha & \text{if}\ s \leq \alpha \end{cases} \f$ | \f$ ds = \begin{cases} dd & \text{if}\ \alpha < s \leq \beta \\ 0 & \text{otherwise}\ \end{cases} \f$ | -- |
| clip_v2 | #dnnl_eltwise_clip_v2 <br> #dnnl_eltwise_clip_v2_use_dst_for_bwd | \f$ d = \begin{cases} \beta & \text{if}\ s \geq \beta \geq \alpha \\ s & \text{if}\ \alpha < s < \beta \\ \alpha & \text{if}\ s \leq \alpha \end{cases} \f$ | \f$ ds = \begin{cases} dd & \text{if}\ \alpha < s < \beta \\ 0 & \text{otherwise}\ \end{cases} \f$ | \f$ ds = \begin{cases} dd & \text{if}\ \alpha < d < \beta \\ 0 & \text{otherwise}\ \end{cases} \f$ |
| elu | #dnnl_eltwise_elu <br> #dnnl_eltwise_elu_use_dst_for_bwd | \f$ d = \begin{cases} s & \text{if}\ s > 0 \\ \alpha (e^s - 1) & \text{if}\ s \leq 0 \end{cases} \f$ | \f$ ds = \begin{cases} dd & \text{if}\ s > 0 \\ dd \cdot \alpha e^s & \text{if}\ s \leq 0 \end{cases} \f$ | \f$ ds = \begin{cases} dd & \text{if}\ d > 0 \\ dd \cdot (d + \alpha) & \text{if}\ d \leq 0 \end{cases}. See\ (2). \f$ |
Expand Down Expand Up @@ -54,6 +53,7 @@ The following operations are supported:
\f$ (3)\ \text{where, } \omega = e^{3s} + 4 \cdot e^{2s} + e^{s} \cdot (4 \cdot s + 6) + 4 \cdot (s + 1) \text{ and } \delta = e^{2s} + 2 \cdot e^{s} + 2. \f$

Note that following equations hold:
* \f$ bounded\_relu(s, alpha) = clip(s, 0, alpha) \f$
* \f$ logsigmoid(s) = soft\_relu(s, -1) \f$

#### Difference Between Forward Training and Forward Inference
Expand Down
2 changes: 0 additions & 2 deletions include/oneapi/dnnl/dnnl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -543,8 +543,6 @@ enum class algorithm {
eltwise_swish = dnnl_eltwise_swish,
/// Elementwise: linear
eltwise_linear = dnnl_eltwise_linear,
/// Elementwise: bounded_relu
eltwise_bounded_relu = dnnl_eltwise_bounded_relu,
/// Elementwise: soft_relu
eltwise_soft_relu = dnnl_eltwise_soft_relu,
/// Elementwise: mish
Expand Down
2 changes: 0 additions & 2 deletions include/oneapi/dnnl/dnnl_types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1454,8 +1454,6 @@ typedef enum {
dnnl_eltwise_sqrt = 0x6f,
/// Eltwise: linear
dnnl_eltwise_linear = 0x7f,
/// Eltwise: bounded_relu
dnnl_eltwise_bounded_relu = 0x8f,
/// Eltwise: soft_relu
dnnl_eltwise_soft_relu = 0xa0,
/// Eltwise: hardsigmoid
Expand Down
1 change: 0 additions & 1 deletion src/common/c_types_map.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ const alg_kind_t eltwise_abs = dnnl_eltwise_abs;
const alg_kind_t eltwise_sqrt = dnnl_eltwise_sqrt;
const alg_kind_t eltwise_swish = dnnl_eltwise_swish;
const alg_kind_t eltwise_linear = dnnl_eltwise_linear;
const alg_kind_t eltwise_bounded_relu = dnnl_eltwise_bounded_relu;
const alg_kind_t eltwise_soft_relu = dnnl_eltwise_soft_relu;
const alg_kind_t eltwise_logistic = dnnl_eltwise_logistic;
const alg_kind_t eltwise_mish = dnnl_eltwise_mish;
Expand Down
1 change: 0 additions & 1 deletion src/common/dnnl_debug_autogenerated.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1130,7 +1130,6 @@ const char *dnnl_alg_kind2str(dnnl_alg_kind_t v) {
if (v == dnnl_eltwise_abs) return "eltwise_abs";
if (v == dnnl_eltwise_sqrt) return "eltwise_sqrt";
if (v == dnnl_eltwise_linear) return "eltwise_linear";
if (v == dnnl_eltwise_bounded_relu) return "eltwise_bounded_relu";
if (v == dnnl_eltwise_soft_relu) return "eltwise_soft_relu";
if (v == dnnl_eltwise_hardsigmoid) return "eltwise_hardsigmoid";
if (v == dnnl_eltwise_logistic) return "eltwise_logistic";
Expand Down
16 changes: 8 additions & 8 deletions src/common/eltwise_pd.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,8 @@ struct eltwise_fwd_pd_t : public eltwise_pd_t {
using namespace utils;
return one_of(alg, eltwise_relu, eltwise_tanh, eltwise_elu,
eltwise_square, eltwise_abs, eltwise_sqrt, eltwise_swish,
eltwise_bounded_relu, eltwise_gelu_tanh,
eltwise_gelu_erf, eltwise_round, eltwise_hardswish)
eltwise_gelu_tanh, eltwise_gelu_erf, eltwise_round,
eltwise_hardswish)
|| one_of(alg, eltwise_relu_use_dst_for_bwd,
eltwise_tanh_use_dst_for_bwd,
eltwise_elu_use_dst_for_bwd,
Expand Down Expand Up @@ -221,12 +221,12 @@ struct eltwise_bwd_pd_t : public eltwise_pd_t {
// zero, so excluding all those algs from here.
using namespace alg_kind;
using namespace utils;
return one_of(alg, eltwise_abs, eltwise_bounded_relu, eltwise_clip,
eltwise_clip_v2, eltwise_elu, eltwise_exp,
eltwise_gelu_erf, eltwise_gelu_tanh, eltwise_hardsigmoid,
eltwise_linear, eltwise_logistic, eltwise_mish,
eltwise_relu, eltwise_soft_relu, eltwise_square,
eltwise_swish, eltwise_tanh)
return one_of(alg, eltwise_abs, eltwise_clip, eltwise_clip_v2,
eltwise_elu, eltwise_exp, eltwise_gelu_erf,
eltwise_gelu_tanh, eltwise_hardsigmoid, eltwise_linear,
eltwise_logistic, eltwise_mish, eltwise_relu,
eltwise_soft_relu, eltwise_square, eltwise_swish,
eltwise_tanh)
|| one_of(alg, eltwise_elu_use_dst_for_bwd,
eltwise_exp_use_dst_for_bwd,
eltwise_logistic_use_dst_for_bwd,
Expand Down
23 changes: 5 additions & 18 deletions src/common/math_utils.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,18 +214,6 @@ inline U linear_bwd(T dd, T s, A alpha, A beta) {
return (U)(dd * alpha);
}

template <typename T, typename A,
typename U = typename utils::remove_reference<T>::type>
inline U bounded_relu_fwd(T s, A alpha) {
s = s > 0 ? s : (U)0;
return s > alpha ? (U)(alpha) : s;
}
template <typename T, typename A,
typename U = typename utils::remove_reference<T>::type>
inline U bounded_relu_bwd(T dd, T s, A alpha) {
return dd * (0 < s && s <= alpha ? 1 : 0);
}

template <typename T, typename U = typename utils::remove_reference<T>::type>
inline U logistic_fwd(T s) {
// Here we avoid division/inverse by infinity as some architectures have
Expand Down Expand Up @@ -395,7 +383,7 @@ inline U hardsigmoid_bwd(T dd, T s, A alpha, A beta) {

template <typename T, typename U = typename utils::remove_reference<T>::type>
inline U hardswish_fwd(T s) {
return (s / 6.f) * bounded_relu_fwd(s + 3.f, 6.f);
return (s / 6.f) * clip_fwd(s + 3.f, 0.f, 6.f);
}
template <typename T, typename U = typename utils::remove_reference<T>::type>
inline U hardswish_bwd(T dd, T s) {
Expand All @@ -411,12 +399,11 @@ inline bool is_eltwise_ok(
const bool eltwise_use_src
= one_of(alg, eltwise_relu, eltwise_tanh, eltwise_elu,
eltwise_square, eltwise_abs, eltwise_sqrt, eltwise_linear,
eltwise_bounded_relu, eltwise_soft_relu, eltwise_mish,
eltwise_logistic, eltwise_exp, eltwise_gelu_tanh,
eltwise_hardsigmoid, eltwise_hardswish, eltwise_swish,
eltwise_log, eltwise_clip, eltwise_clip_v2, eltwise_pow,
eltwise_soft_relu, eltwise_mish, eltwise_logistic,
eltwise_exp, eltwise_gelu_tanh, eltwise_hardsigmoid,
eltwise_hardswish, eltwise_swish, eltwise_log,
eltwise_clip, eltwise_clip_v2, eltwise_pow,
eltwise_gelu_erf, eltwise_round)
&& IMPLICATION(alg == eltwise_bounded_relu, alpha >= 0)
&& IMPLICATION(
one_of(alg, eltwise_clip, eltwise_clip_v2), beta >= alpha)
&& IMPLICATION(alg == eltwise_round, dt == dnnl_f32)
Expand Down
2 changes: 0 additions & 2 deletions src/common/opdesc.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,6 @@ struct eltwise_desc_t {
// The kind of eltwise algorithm. Possible values: #dnnl_eltwise_relu,
// #dnnl_eltwise_tanh, #dnnl_eltwise_elu, #dnnl_eltwise_square,
// #dnnl_eltwise_abs, #dnnl_eltwise_sqrt, #dnnl_eltwise_linear,
// #dnnl_eltwise_bounded_relu,
// #dnnl_eltwise_soft_relu, #dnnl_eltwise_logistic, #dnnl_eltwise_exp,
// #dnnl_eltwise_gelu_tanh, #dnnl_eltwise_swish, #dnnl_eltwise_log,
// #dnnl_eltwise_clip, #dnnl_eltwise_clip_v2, #dnnl_eltwise_pow,
Expand All @@ -221,7 +220,6 @@ struct eltwise_desc_t {
// - #dnnl_eltwise_abs: @p alpha and @p beta ignored
// - #dnnl_eltwise_sqrt: @p alpha and @p beta ignored
// - #dnnl_eltwise_linear: @p alpha -- scale, @p beta -- shift
// - #dnnl_eltwise_bounded_relu: @p alpha -- upper bound, @p beta ignored
// - #dnnl_eltwise_soft_relu: @p alpha -- soft_relu arg scaling, @p beta ignored
// - #dnnl_eltwise_logistic: @p alpha and @p beta ignored
// - #dnnl_eltwise_exp: @p alpha and @p beta ignored
Expand Down
3 changes: 0 additions & 3 deletions src/cpu/aarch64/acl_utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,6 @@ status_t convert_to_acl_act(alg_kind_t eltwise_alg, float alpha, float beta,
case eltwise_linear:
act_info = ActivationLayerInfo(act_func::LINEAR, alpha, beta);
break;
case eltwise_bounded_relu:
act_info = ActivationLayerInfo(act_func::BOUNDED_RELU, alpha, beta);
break;
case eltwise_soft_relu:
act_info = ActivationLayerInfo(act_func::SOFT_RELU, alpha, beta);
break;
Expand Down
34 changes: 1 addition & 33 deletions src/cpu/aarch64/injectors/jit_uni_eltwise_injector.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ bool is_alg_supported(alg_kind_t alg) {
using namespace alg_kind;
return utils::one_of(alg, eltwise_relu, eltwise_tanh, eltwise_elu,
eltwise_square, eltwise_abs, eltwise_sqrt, eltwise_linear,
eltwise_bounded_relu, /*eltwise_soft_relu,*/
/*eltwise_soft_relu,*/
eltwise_logistic, /*eltwise_mish,*/ eltwise_exp, eltwise_gelu_tanh,
/*eltwise_hardswish,*/ eltwise_swish, eltwise_log, eltwise_clip,
/*eltwise_clip_v2, eltwise_pow,*/ eltwise_gelu_erf, eltwise_round,
Expand Down Expand Up @@ -256,7 +256,6 @@ void jit_uni_eltwise_injector_f32<isa>::set_coef_to_regs() {
table_val(alpha, z_tmp);
table_val(beta, vmm_aux0);
break;
case eltwise_bounded_relu: table_val(alpha, z_tmp); break;
// case eltwise_soft_relu: // TODO: enable me.
case eltwise_logistic_use_dst_for_bwd:
case eltwise_logistic:
Expand Down Expand Up @@ -286,7 +285,6 @@ void jit_uni_eltwise_injector_f32<isa>::set_coef_to_regs() {
case eltwise_sqrt_use_dst_for_bwd:
case eltwise_sqrt:
case eltwise_linear:
case eltwise_bounded_relu:
// case eltwise_soft_relu:
case eltwise_logistic_use_dst_for_bwd:
case eltwise_logistic:
Expand Down Expand Up @@ -630,13 +628,6 @@ void jit_uni_eltwise_injector_f32<isa>::linear_compute_vector_fwd(
h->fmad(vmm_src, p_all / T_m, z_tmp, vmm_aux0);
}

template <cpu_isa_t isa>
void jit_uni_eltwise_injector_f32<isa>::bounded_relu_compute_vector_fwd(
const TRegS &vmm_src) {
h->fmaxnm(vmm_src, p_all, 0.f);
h->fminnm(vmm_src, p_all, z_tmp);
}

template <cpu_isa_t isa>
void jit_uni_eltwise_injector_f32<isa>::clip_compute_vector_fwd(
const TRegS &vmm_src) {
Expand Down Expand Up @@ -1090,21 +1081,6 @@ void jit_uni_eltwise_injector_f32<isa>::linear_compute_vector_bwd(
h->mov(ZRegD(IDX(vmm_src)), ZRegD(IDX(table_val(alpha, z_tmp))));
}

template <cpu_isa_t isa>
void jit_uni_eltwise_injector_f32<isa>::bounded_relu_compute_vector_bwd(
const TRegS &vmm_src) {
// get mask of values > alpha and blend with 0.f
compute_cmp_mask(vmm_src, table_val(alpha, z_tmp), _cmp_gt_os);
blend_with_mask(vmm_src, table_val(zero, z_tmp));
// make all negative values zeros
h->fmov(z_tmp, 0.f);
h->fmaxnm(vmm_src, p_all, z_tmp);

// everything bigger than 0.f should be 1.f
compute_cmp_mask(vmm_src, table_val(zero, z_tmp), _cmp_gt_os);
blend_with_mask(vmm_src, table_val(one, z_tmp));
}

template <cpu_isa_t isa>
void jit_uni_eltwise_injector_f32<isa>::soft_relu_compute_vector_bwd(
const TRegS &vmm_src) {
Expand Down Expand Up @@ -1296,7 +1272,6 @@ size_t jit_uni_eltwise_injector_f32<isa>::aux_vecs_count() {
case eltwise_sqrt_use_dst_for_bwd:
case eltwise_sqrt: return 0;
case eltwise_linear: return 2;
case eltwise_bounded_relu: return 1;
// case eltwise_soft_relu: return 5;
case eltwise_logistic_use_dst_for_bwd:
case eltwise_logistic: return 5; /* = exp + 1 */
Expand All @@ -1323,7 +1298,6 @@ size_t jit_uni_eltwise_injector_f32<isa>::aux_vecs_count() {
case eltwise_sqrt_use_dst_for_bwd:
case eltwise_sqrt: return 2;
case eltwise_linear: return 1;
case eltwise_bounded_relu: return 1;
// case eltwise_soft_relu: return 5; /* = logistic */
case eltwise_logistic_use_dst_for_bwd: return 2;
case eltwise_logistic: return 5; /* = logistic */
Expand Down Expand Up @@ -1370,9 +1344,6 @@ void jit_uni_eltwise_injector_f32<isa>::compute_body(
case eltwise_linear:
linear_compute_vector_fwd(TRegS(idx));
break;
case eltwise_bounded_relu:
bounded_relu_compute_vector_fwd(TRegS(idx));
break;
// case eltwise_soft_relu:
// soft_relu_compute_vector_fwd(TRegS(idx));
// break;
Expand Down Expand Up @@ -1410,9 +1381,6 @@ void jit_uni_eltwise_injector_f32<isa>::compute_body(
case eltwise_linear:
linear_compute_vector_bwd(TRegS(idx));
break;
case eltwise_bounded_relu:
bounded_relu_compute_vector_bwd(TRegS(idx));
break;
// case eltwise_soft_relu:
// soft_relu_compute_vector_bwd(TRegS(idx));
// break;
Expand Down
15 changes: 6 additions & 9 deletions src/cpu/aarch64/injectors/jit_uni_eltwise_injector.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,13 +120,12 @@ struct jit_uni_eltwise_injector_f32 {
assert(is_superset(isa, sve_128));
assert(utils::one_of(alg_, eltwise_relu, eltwise_tanh, eltwise_elu,
eltwise_square, eltwise_abs, eltwise_sqrt, eltwise_linear,
eltwise_bounded_relu, eltwise_logistic, eltwise_exp,
eltwise_gelu_tanh, eltwise_swish, eltwise_log, eltwise_clip,
eltwise_clip_v2, eltwise_gelu_erf, eltwise_round,
eltwise_relu_use_dst_for_bwd, eltwise_tanh_use_dst_for_bwd,
eltwise_elu_use_dst_for_bwd, eltwise_sqrt_use_dst_for_bwd,
eltwise_logistic_use_dst_for_bwd, eltwise_exp_use_dst_for_bwd,
eltwise_clip_v2_use_dst_for_bwd));
eltwise_logistic, eltwise_exp, eltwise_gelu_tanh, eltwise_swish,
eltwise_log, eltwise_clip, eltwise_clip_v2, eltwise_gelu_erf,
eltwise_round, eltwise_relu_use_dst_for_bwd,
eltwise_tanh_use_dst_for_bwd, eltwise_elu_use_dst_for_bwd,
eltwise_sqrt_use_dst_for_bwd, eltwise_logistic_use_dst_for_bwd,
eltwise_exp_use_dst_for_bwd, eltwise_clip_v2_use_dst_for_bwd));
register_table_entries();
}

Expand Down Expand Up @@ -227,7 +226,6 @@ struct jit_uni_eltwise_injector_f32 {
void abs_compute_vector_fwd(const TRegS &vmm_src);
void sqrt_compute_vector_fwd(const TRegS &vmm_src);
void linear_compute_vector_fwd(const TRegS &vmm_src);
void bounded_relu_compute_vector_fwd(const TRegS &vmm_src);
void soft_relu_compute_vector_fwd(const TRegS &vmm_src);
void logistic_compute_vector_fwd(const TRegS &vmm_src);
void gelu_tanh_compute_vector_fwd(const TRegS &vmm_src);
Expand All @@ -245,7 +243,6 @@ struct jit_uni_eltwise_injector_f32 {
void abs_compute_vector_bwd(const TRegS &vmm_src);
void sqrt_compute_vector_bwd(const TRegS &vmm_src);
void linear_compute_vector_bwd(const TRegS &vmm_src);
void bounded_relu_compute_vector_bwd(const TRegS &vmm_src);
void soft_relu_compute_vector_bwd(const TRegS &vmm_src);
void logistic_compute_vector_bwd(const TRegS &vmm_src);
void gelu_tanh_compute_vector_bwd(const TRegS &vmm_src);
Expand Down
17 changes: 8 additions & 9 deletions src/cpu/aarch64/jit_uni_eltwise.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,11 +203,10 @@ status_t jit_uni_eltwise_fwd_t<isa, d_type>::pd_t::init(engine_t *engine) {
eltwise_relu, eltwise_elu_use_dst_for_bwd, eltwise_elu,
eltwise_tanh_use_dst_for_bwd, eltwise_tanh, eltwise_square,
eltwise_abs, eltwise_sqrt_use_dst_for_bwd, eltwise_sqrt,
eltwise_linear, eltwise_bounded_relu, eltwise_soft_relu,
eltwise_logistic_use_dst_for_bwd, eltwise_logistic,
eltwise_exp_use_dst_for_bwd, eltwise_exp, eltwise_gelu_tanh,
eltwise_swish, eltwise_log, eltwise_clip, eltwise_gelu_erf,
eltwise_round);
eltwise_linear, eltwise_soft_relu, eltwise_logistic_use_dst_for_bwd,
eltwise_logistic, eltwise_exp_use_dst_for_bwd, eltwise_exp,
eltwise_gelu_tanh, eltwise_swish, eltwise_log, eltwise_clip,
eltwise_gelu_erf, eltwise_round);

return ok ? status::success : status::unimplemented;
}
Expand Down Expand Up @@ -279,10 +278,10 @@ status_t jit_uni_eltwise_bwd_t<isa, d_type>::pd_t::init(engine_t *engine) {
eltwise_relu, eltwise_elu_use_dst_for_bwd, eltwise_elu,
eltwise_tanh_use_dst_for_bwd, eltwise_tanh, eltwise_square,
eltwise_abs, eltwise_sqrt_use_dst_for_bwd, eltwise_sqrt,
eltwise_linear, eltwise_bounded_relu, eltwise_soft_relu,
eltwise_logistic_use_dst_for_bwd, eltwise_logistic,
eltwise_exp_use_dst_for_bwd, eltwise_exp, eltwise_gelu_tanh,
eltwise_swish, eltwise_log, eltwise_clip, eltwise_gelu_erf);
eltwise_linear, eltwise_soft_relu, eltwise_logistic_use_dst_for_bwd,
eltwise_logistic, eltwise_exp_use_dst_for_bwd, eltwise_exp,
eltwise_gelu_tanh, eltwise_swish, eltwise_log, eltwise_clip,
eltwise_gelu_erf);

return ok ? status::success : status::unimplemented;
}
Expand Down
Loading

0 comments on commit 7de83da

Please sign in to comment.