[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH v2 19/30] accel/tcg: Add have_lse2 support in ldst_atomicity
From: |
Richard Henderson |
Subject: |
[PATCH v2 19/30] accel/tcg: Add have_lse2 support in ldst_atomicity |
Date: |
Wed, 15 Feb 2023 16:57:28 -1000 |
Add fast paths for FEAT_LSE2, using the detection in tcg.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
accel/tcg/ldst_atomicity.c.inc | 37 ++++++++++++++++++++++++++++++----
1 file changed, 33 insertions(+), 4 deletions(-)
diff --git a/accel/tcg/ldst_atomicity.c.inc b/accel/tcg/ldst_atomicity.c.inc
index 9a95ac327d..277629f241 100644
--- a/accel/tcg/ldst_atomicity.c.inc
+++ b/accel/tcg/ldst_atomicity.c.inc
@@ -41,6 +41,8 @@
* but we're using tcg/tci/ instead.
*/
# define HAVE_al16_fast false
+#elif defined(__aarch64__)
+# define HAVE_al16_fast likely(have_lse2)
#elif defined(__x86_64__)
# define HAVE_al16_fast likely(have_atomic16)
#else
@@ -48,6 +50,8 @@
#endif
#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128)
# define HAVE_al16 true
+#elif defined(__aarch64__)
+# define HAVE_al16 true
#else
# define HAVE_al16 false
#endif
@@ -170,6 +174,14 @@ load_atomic16(void *pv)
r.u = qatomic_read__nocheck(p);
return r.s;
+#elif defined(__aarch64__)
+ uint64_t l, h;
+
+ /* Via HAVE_al16_fast, FEAT_LSE2 is present: LDP becomes atomic. */
+ asm("ldp %0, %1, %2" : "=r"(l), "=r"(h) : "m"(*(__uint128_t *)pv));
+
+ qemu_build_assert(!HOST_BIG_ENDIAN);
+ return int128_make128(l, h);
#elif defined(__x86_64__)
Int128Alias r;
@@ -409,6 +421,18 @@ load_atom_extract_al16_or_al8(void *pv, int s)
r = qatomic_read__nocheck(p16);
}
return r >> shr;
+#elif defined(__aarch64__)
+ /*
+ * Via HAVE_al16_fast, FEAT_LSE2 is present.
+ * LDP becomes single-copy atomic if 16-byte aligned, and
+ * single-copy atomic on the parts if 8-byte aligned.
+ */
+ uintptr_t pi = (uintptr_t)pv;
+ int shr = (pi & 7) * 8;
+ uint64_t l, h;
+
+ asm("ldp %0, %1, %2" : "=r"(l), "=r"(h) : "m"(*(__uint128_t *)(pi & ~7)));
+ return (l >> shr) | (h << (-shr & 63));
#elif defined(__x86_64__)
uintptr_t pi = (uintptr_t)pv;
int shr = (pi & 7) * 8;
@@ -764,10 +788,15 @@ store_atomic16(void *pv, Int128Alias val)
l = int128_getlo(val.s);
h = int128_gethi(val.s);
- asm("0: ldxp %0, xzr, %1\n\t"
- "stxp %w0, %2, %3, %1\n\t"
- "cbnz %w0, 0b"
- : "=&r"(t), "=Q"(*(__uint128_t *)pv) : "r"(l), "r"(h));
+ if (HAVE_al16_fast) {
+ /* Via HAVE_al16_fast, FEAT_LSE2 is present: STP becomes atomic. */
+ asm("stp %1, %2, %0" : "=Q"(*(__uint128_t *)pv) : "r"(l), "r"(h));
+ } else {
+ asm("0: ldxp %0, xzr, %1\n\t"
+ "stxp %w0, %2, %3, %1\n\t"
+ "cbnz %w0, 0b"
+ : "=&r"(t), "=Q"(*(__uint128_t *)pv) : "r"(l), "r"(h));
+ }
return;
}
#elif defined(CONFIG_CMPXCHG128)
--
2.34.1
- [PATCH v2 08/30] accel/tcg: Honor atomicity of stores, (continued)
- [PATCH v2 08/30] accel/tcg: Honor atomicity of stores, Richard Henderson, 2023/02/15
- [PATCH v2 09/30] tcg/tci: Use cpu_{ld,st}_mmu, Richard Henderson, 2023/02/15
- [PATCH v2 10/30] tcg: Unify helper_{be,le}_{ld,st}*, Richard Henderson, 2023/02/15
- [PATCH v2 13/30] meson: Detect atomic128 support with optimization, Richard Henderson, 2023/02/15
- [PATCH v2 11/30] accel/tcg: Implement helper_{ld, st}*_mmu for user-only, Richard Henderson, 2023/02/15
- [PATCH v2 12/30] tcg: Add 128-bit guest memory primitives, Richard Henderson, 2023/02/15
- [PATCH v2 16/30] accel/tcg: Add aarch64 specific support in ldst_atomicity, Richard Henderson, 2023/02/15
- [PATCH v2 14/30] tcg/i386: Add have_atomic16, Richard Henderson, 2023/02/15
- [PATCH v2 15/30] accel/tcg: Use have_atomic16 in ldst_atomicity.c.inc, Richard Henderson, 2023/02/15
- [PATCH v2 17/30] tcg/aarch64: Detect have_lse, have_lse2 for linux, Richard Henderson, 2023/02/15
- [PATCH v2 19/30] accel/tcg: Add have_lse2 support in ldst_atomicity,
Richard Henderson <=
- [PATCH v2 18/30] tcg/aarch64: Detect have_lse, have_lse2 for darwin, Richard Henderson, 2023/02/15
- [PATCH v2 20/30] tcg: Introduce TCG_OPF_TYPE_MASK, Richard Henderson, 2023/02/15
- [PATCH v2 22/30] tcg/i386: Introduce tcg_out_mov2, Richard Henderson, 2023/02/15
- [PATCH v2 21/30] tcg: Add INDEX_op_qemu_{ld,st}_i128, Richard Henderson, 2023/02/15
- [PATCH v2 23/30] tcg/i386: Introduce tcg_out_testi, Richard Henderson, 2023/02/15
- [PATCH v2 24/30] tcg/i386: Use full load/store helpers in user-only mode, Richard Henderson, 2023/02/15
- [PATCH v2 25/30] tcg/i386: Replace is64 with type in qemu_ld/st routines, Richard Henderson, 2023/02/15
- [PATCH v2 26/30] tcg/i386: Mark Win64 call-saved vector regs as reserved, Richard Henderson, 2023/02/15
- [PATCH v2 27/30] tcg/i386: Examine MemOp for atomicity and alignment, Richard Henderson, 2023/02/15
- [PATCH v2 28/30] tcg/i386: Support 128-bit load/store with have_atomic16, Richard Henderson, 2023/02/15