[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PULL 21/45] target/arm: Implement bfloat16 dot product (vector)
From: |
Peter Maydell |
Subject: |
[PULL 21/45] target/arm: Implement bfloat16 dot product (vector) |
Date: |
Thu, 3 Jun 2021 16:58:40 +0100 |
From: Richard Henderson <richard.henderson@linaro.org>
This is BFDOT for both AArch64 AdvSIMD and SVE,
and VDOT.BF16 for AArch32 NEON.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Message-id: 20210525225817.400336-7-richard.henderson@linaro.org
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
---
target/arm/helper.h | 3 +++
target/arm/neon-shared.decode | 2 ++
target/arm/sve.decode | 3 +++
target/arm/translate-a64.c | 20 ++++++++++++++++++
target/arm/translate-neon.c | 9 ++++++++
target/arm/translate-sve.c | 12 +++++++++++
target/arm/vec_helper.c | 40 +++++++++++++++++++++++++++++++++++
7 files changed, 89 insertions(+)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 8b4b7d92f36..de2f5331dcc 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -1002,6 +1002,9 @@ DEF_HELPER_FLAGS_5(gvec_ummla_b, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(gvec_usmmla_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_bfdot, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
+
#ifdef TARGET_AARCH64
#include "helper-a64.h"
#include "helper-sve.h"
diff --git a/target/arm/neon-shared.decode b/target/arm/neon-shared.decode
index cc9f4cdd85b..31a0839bbb0 100644
--- a/target/arm/neon-shared.decode
+++ b/target/arm/neon-shared.decode
@@ -52,6 +52,8 @@ VUDOT 1111 110 00 . 10 .... .... 1101 . q:1 . 1 ....
\
vm=%vm_dp vn=%vn_dp vd=%vd_dp
VUSDOT 1111 110 01 . 10 .... .... 1101 . q:1 . 0 .... \
vm=%vm_dp vn=%vn_dp vd=%vd_dp
+VDOT_b16 1111 110 00 . 00 .... .... 1101 . q:1 . 0 .... \
+ vm=%vm_dp vn=%vn_dp vd=%vd_dp
# VFM[AS]L
VFML 1111 110 0 s:1 . 10 .... .... 1000 . 0 . 1 .... \
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 18d1a0eecc5..a7429b293fe 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1625,6 +1625,9 @@ FMLALT_zzzw 01100100 10 1 ..... 10 0 00 1 ..... .....
@rda_rn_rm_e0
FMLSLB_zzzw 01100100 10 1 ..... 10 1 00 0 ..... ..... @rda_rn_rm_e0
FMLSLT_zzzw 01100100 10 1 ..... 10 1 00 1 ..... ..... @rda_rn_rm_e0
+### SVE2 floating-point bfloat16 dot-product
+BFDOT_zzzz 01100100 01 1 ..... 10 0 00 0 ..... ..... @rda_rn_rm_e0
+
### SVE2 floating-point multiply-add long (indexed)
FMLALB_zzxw 01100100 10 1 ..... 0100.0 ..... ..... @rrxr_3a esz=2
FMLALT_zzxw 01100100 10 1 ..... 0100.1 ..... ..... @rrxr_3a esz=2
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index 0f15fa42fc3..3c36de3df20 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -12235,6 +12235,16 @@ static void
disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
}
feature = dc_isar_feature(aa64_fcma, s);
break;
+ case 0x1f: /* BFDOT */
+ switch (size) {
+ case 1:
+ feature = dc_isar_feature(aa64_bf16, s);
+ break;
+ default:
+ unallocated_encoding(s);
+ return;
+ }
+ break;
default:
unallocated_encoding(s);
return;
@@ -12318,6 +12328,16 @@ static void
disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
}
return;
+ case 0xf: /* BFDOT */
+ switch (size) {
+ case 1:
+ gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
gen_helper_gvec_bfdot);
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ return;
+
default:
g_assert_not_reached();
}
diff --git a/target/arm/translate-neon.c b/target/arm/translate-neon.c
index 6d94229c691..9460857b2ad 100644
--- a/target/arm/translate-neon.c
+++ b/target/arm/translate-neon.c
@@ -296,6 +296,15 @@ static bool trans_VUSDOT(DisasContext *s, arg_VUSDOT *a)
gen_helper_gvec_usdot_b);
}
+static bool trans_VDOT_b16(DisasContext *s, arg_VDOT_b16 *a)
+{
+ if (!dc_isar_feature(aa32_bf16, s)) {
+ return false;
+ }
+ return do_neon_ddda(s, a->q * 7, a->vd, a->vn, a->vm, 0,
+ gen_helper_gvec_bfdot);
+}
+
static bool trans_VFML(DisasContext *s, arg_VFML *a)
{
int opr_sz;
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index fb692a18351..ed290827ad2 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8653,3 +8653,15 @@ static bool trans_UMMLA(DisasContext *s, arg_rrrr_esz *a)
{
return do_i8mm_zzzz_ool(s, a, gen_helper_gvec_ummla_b, 0);
}
+
+static bool trans_BFDOT_zzzz(DisasContext *s, arg_rrrr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve_bf16, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzzz(s, gen_helper_gvec_bfdot,
+ a->rd, a->rn, a->rm, a->ra, 0);
+ }
+ return true;
+}
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index e84b438340e..7eefcd06eae 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -2412,3 +2412,43 @@ static void do_mmla_b(void *vd, void *vn, void *vm, void
*va, uint32_t desc,
DO_MMLA_B(gvec_smmla_b, do_smmla_b)
DO_MMLA_B(gvec_ummla_b, do_ummla_b)
DO_MMLA_B(gvec_usmmla_b, do_usmmla_b)
+
+/*
+ * BFloat16 Dot Product
+ */
+
+static float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2)
+{
+ /* FPCR is ignored for BFDOT and BFMMLA. */
+ float_status bf_status = {
+ .tininess_before_rounding = float_tininess_before_rounding,
+ .float_rounding_mode = float_round_to_odd_inf,
+ .flush_to_zero = true,
+ .flush_inputs_to_zero = true,
+ .default_nan_mode = true,
+ };
+ float32 t1, t2;
+
+ /*
+ * Extract each BFloat16 from the element pair, and shift
+ * them such that they become float32.
+ */
+ t1 = float32_mul(e1 << 16, e2 << 16, &bf_status);
+ t2 = float32_mul(e1 & 0xffff0000u, e2 & 0xffff0000u, &bf_status);
+ t1 = float32_add(t1, t2, &bf_status);
+ t1 = float32_add(sum, t1, &bf_status);
+
+ return t1;
+}
+
+void HELPER(gvec_bfdot)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
+ float32 *d = vd, *a = va;
+ uint32_t *n = vn, *m = vm;
+
+ for (i = 0; i < opr_sz / 4; ++i) {
+ d[i] = bfdotadd(a[i], n[i], m[i]);
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
--
2.20.1
- [PULL 11/45] target/arm: Mark LDS{MIN,MAX} as signed operations, (continued)
- [PULL 11/45] target/arm: Mark LDS{MIN,MAX} as signed operations, Peter Maydell, 2021/06/03
- [PULL 12/45] target/arm: fix missing exception class, Peter Maydell, 2021/06/03
- [PULL 16/45] target/arm: Add isar_feature_{aa32, aa64, aa64_sve}_bf16, Peter Maydell, 2021/06/03
- [PULL 13/45] target/arm: fold do_raise_exception into raise_exception, Peter Maydell, 2021/06/03
- [PULL 14/45] target/arm: use raise_exception_ra for MTE check failure, Peter Maydell, 2021/06/03
- [PULL 20/45] softfpu: Add float_round_to_odd_inf, Peter Maydell, 2021/06/03
- [PULL 18/45] target/arm: Implement scalar float32 to bfloat16 conversion, Peter Maydell, 2021/06/03
- [PULL 15/45] target/arm: use raise_exception_ra for stack limit exception, Peter Maydell, 2021/06/03
- [PULL 17/45] target/arm: Unify unallocated path in disas_fp_1src, Peter Maydell, 2021/06/03
- [PULL 19/45] target/arm: Implement vector float32 to bfloat16 conversion, Peter Maydell, 2021/06/03
- [PULL 21/45] target/arm: Implement bfloat16 dot product (vector),
Peter Maydell <=
- [PULL 22/45] target/arm: Implement bfloat16 dot product (indexed), Peter Maydell, 2021/06/03
- [PULL 23/45] target/arm: Implement bfloat16 matrix multiply accumulate, Peter Maydell, 2021/06/03
- [PULL 24/45] target/arm: Implement bfloat widening fma (vector), Peter Maydell, 2021/06/03
- [PULL 26/45] linux-user/aarch64: Enable hwcap bits for bfloat16, Peter Maydell, 2021/06/03
- [PULL 25/45] target/arm: Implement bfloat widening fma (indexed), Peter Maydell, 2021/06/03
- [PULL 27/45] target/arm: Enable BFloat16 extensions, Peter Maydell, 2021/06/03
- [PULL 28/45] hvf: Move assert_hvf_ok() into common directory, Peter Maydell, 2021/06/03
- [PULL 29/45] hvf: Move vcpu thread functions into common directory, Peter Maydell, 2021/06/03
- [PULL 30/45] hvf: Move cpu functions into common directory, Peter Maydell, 2021/06/03
- [PULL 31/45] hvf: Move hvf internal definitions into common header, Peter Maydell, 2021/06/03