lkml.org 
[lkml]   [2022]   [Dec]   [27]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[tip: sched/core] selftests/rseq: s390: Template memory ordering and percpu access mode
    The following commit has been merged into the sched/core branch of tip:

    Commit-ID: a94af3c58462d23c45879e1bbe86ed5702d5bd86
    Gitweb: https://git.kernel.org/tip/a94af3c58462d23c45879e1bbe86ed5702d5bd86
    Author: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
    AuthorDate: Tue, 22 Nov 2022 15:39:18 -05:00
    Committer: Peter Zijlstra <peterz@infradead.org>
    CommitterDate: Tue, 27 Dec 2022 12:52:14 +01:00

    selftests/rseq: s390: Template memory ordering and percpu access mode

    Introduce a rseq-s390-bits.h template header which is internally included
    to generate the static inline functions covering:

    - relaxed and release memory ordering,
    - per-cpu-id and per-mm-cid per-cpu data access.

    Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
    Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    Link: https://lore.kernel.org/r/20221122203932.231377-17-mathieu.desnoyers@efficios.com
    ---
    tools/testing/selftests/rseq/rseq-s390-bits.h | 474 ++++++++++++++++-
    tools/testing/selftests/rseq/rseq-s390.h | 490 +-----------------
    2 files changed, 498 insertions(+), 466 deletions(-)
    create mode 100644 tools/testing/selftests/rseq/rseq-s390-bits.h

    diff --git a/tools/testing/selftests/rseq/rseq-s390-bits.h b/tools/testing/selftests/rseq/rseq-s390-bits.h
    new file mode 100644
    index 0000000..0cf17d9
    --- /dev/null
    +++ b/tools/testing/selftests/rseq/rseq-s390-bits.h
    @@ -0,0 +1,474 @@
    +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
    +
    +#include "rseq-bits-template.h"
    +
    +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
    + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
    +
    +static inline __attribute__((always_inline))
    +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
    +{
    + RSEQ_INJECT_C(9)
    +
    + __asm__ __volatile__ goto (
    + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    +#endif
    + /* Start rseq by storing table entry pointer into rseq_cs. */
    + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    + RSEQ_INJECT_ASM(3)
    + LONG_CMP " %[expect], %[v]\n\t"
    + "jnz %l[cmpfail]\n\t"
    + RSEQ_INJECT_ASM(4)
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    + LONG_CMP " %[expect], %[v]\n\t"
    + "jnz %l[error2]\n\t"
    +#endif
    + /* final store */
    + LONG_S " %[newv], %[v]\n\t"
    + "2:\n\t"
    + RSEQ_INJECT_ASM(5)
    + RSEQ_ASM_DEFINE_ABORT(4, "", abort)
    + : /* gcc asm goto does not allow outputs */
    + : [cpu_id] "r" (cpu),
    + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
    + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    + [v] "m" (*v),
    + [expect] "r" (expect),
    + [newv] "r" (newv)
    + RSEQ_INJECT_INPUT
    + : "memory", "cc", "r0"
    + RSEQ_INJECT_CLOBBER
    + : abort, cmpfail
    +#ifdef RSEQ_COMPARE_TWICE
    + , error1, error2
    +#endif
    + );
    + rseq_after_asm_goto();
    + return 0;
    +abort:
    + rseq_after_asm_goto();
    + RSEQ_INJECT_FAILED
    + return -1;
    +cmpfail:
    + rseq_after_asm_goto();
    + return 1;
    +#ifdef RSEQ_COMPARE_TWICE
    +error1:
    + rseq_after_asm_goto();
    + rseq_bug("cpu_id comparison failed");
    +error2:
    + rseq_after_asm_goto();
    + rseq_bug("expected value comparison failed");
    +#endif
    +}
    +
    +/*
    + * Compare @v against @expectnot. When it does _not_ match, load @v
    + * into @load, and store the content of *@v + voffp into @v.
    + */
    +static inline __attribute__((always_inline))
    +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
    + long voffp, intptr_t *load, int cpu)
    +{
    + RSEQ_INJECT_C(9)
    +
    + __asm__ __volatile__ goto (
    + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    +#endif
    + /* Start rseq by storing table entry pointer into rseq_cs. */
    + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    + RSEQ_INJECT_ASM(3)
    + LONG_L " %%r1, %[v]\n\t"
    + LONG_CMP_R " %%r1, %[expectnot]\n\t"
    + "je %l[cmpfail]\n\t"
    + RSEQ_INJECT_ASM(4)
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    + LONG_L " %%r1, %[v]\n\t"
    + LONG_CMP_R " %%r1, %[expectnot]\n\t"
    + "je %l[error2]\n\t"
    +#endif
    + LONG_S " %%r1, %[load]\n\t"
    + LONG_ADD_R " %%r1, %[voffp]\n\t"
    + LONG_L " %%r1, 0(%%r1)\n\t"
    + /* final store */
    + LONG_S " %%r1, %[v]\n\t"
    + "2:\n\t"
    + RSEQ_INJECT_ASM(5)
    + RSEQ_ASM_DEFINE_ABORT(4, "", abort)
    + : /* gcc asm goto does not allow outputs */
    + : [cpu_id] "r" (cpu),
    + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
    + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    + /* final store input */
    + [v] "m" (*v),
    + [expectnot] "r" (expectnot),
    + [voffp] "r" (voffp),
    + [load] "m" (*load)
    + RSEQ_INJECT_INPUT
    + : "memory", "cc", "r0", "r1"
    + RSEQ_INJECT_CLOBBER
    + : abort, cmpfail
    +#ifdef RSEQ_COMPARE_TWICE
    + , error1, error2
    +#endif
    + );
    + rseq_after_asm_goto();
    + return 0;
    +abort:
    + rseq_after_asm_goto();
    + RSEQ_INJECT_FAILED
    + return -1;
    +cmpfail:
    + rseq_after_asm_goto();
    + return 1;
    +#ifdef RSEQ_COMPARE_TWICE
    +error1:
    + rseq_after_asm_goto();
    + rseq_bug("cpu_id comparison failed");
    +error2:
    + rseq_after_asm_goto();
    + rseq_bug("expected value comparison failed");
    +#endif
    +}
    +
    +static inline __attribute__((always_inline))
    +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
    +{
    + RSEQ_INJECT_C(9)
    +
    + __asm__ __volatile__ goto (
    + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    +#endif
    + /* Start rseq by storing table entry pointer into rseq_cs. */
    + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    + RSEQ_INJECT_ASM(3)
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    +#endif
    + LONG_L " %%r0, %[v]\n\t"
    + LONG_ADD_R " %%r0, %[count]\n\t"
    + /* final store */
    + LONG_S " %%r0, %[v]\n\t"
    + "2:\n\t"
    + RSEQ_INJECT_ASM(4)
    + RSEQ_ASM_DEFINE_ABORT(4, "", abort)
    + : /* gcc asm goto does not allow outputs */
    + : [cpu_id] "r" (cpu),
    + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
    + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    + /* final store input */
    + [v] "m" (*v),
    + [count] "r" (count)
    + RSEQ_INJECT_INPUT
    + : "memory", "cc", "r0"
    + RSEQ_INJECT_CLOBBER
    + : abort
    +#ifdef RSEQ_COMPARE_TWICE
    + , error1
    +#endif
    + );
    + rseq_after_asm_goto();
    + return 0;
    +abort:
    + rseq_after_asm_goto();
    + RSEQ_INJECT_FAILED
    + return -1;
    +#ifdef RSEQ_COMPARE_TWICE
    +error1:
    + rseq_after_asm_goto();
    + rseq_bug("cpu_id comparison failed");
    +#endif
    +}
    +
    +static inline __attribute__((always_inline))
    +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
    + intptr_t *v2, intptr_t expect2,
    + intptr_t newv, int cpu)
    +{
    + RSEQ_INJECT_C(9)
    +
    + __asm__ __volatile__ goto (
    + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
    +#endif
    + /* Start rseq by storing table entry pointer into rseq_cs. */
    + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    + RSEQ_INJECT_ASM(3)
    + LONG_CMP " %[expect], %[v]\n\t"
    + "jnz %l[cmpfail]\n\t"
    + RSEQ_INJECT_ASM(4)
    + LONG_CMP " %[expect2], %[v2]\n\t"
    + "jnz %l[cmpfail]\n\t"
    + RSEQ_INJECT_ASM(5)
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    + LONG_CMP " %[expect], %[v]\n\t"
    + "jnz %l[error2]\n\t"
    + LONG_CMP " %[expect2], %[v2]\n\t"
    + "jnz %l[error3]\n\t"
    +#endif
    + /* final store */
    + LONG_S " %[newv], %[v]\n\t"
    + "2:\n\t"
    + RSEQ_INJECT_ASM(6)
    + RSEQ_ASM_DEFINE_ABORT(4, "", abort)
    + : /* gcc asm goto does not allow outputs */
    + : [cpu_id] "r" (cpu),
    + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
    + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    + /* cmp2 input */
    + [v2] "m" (*v2),
    + [expect2] "r" (expect2),
    + /* final store input */
    + [v] "m" (*v),
    + [expect] "r" (expect),
    + [newv] "r" (newv)
    + RSEQ_INJECT_INPUT
    + : "memory", "cc", "r0"
    + RSEQ_INJECT_CLOBBER
    + : abort, cmpfail
    +#ifdef RSEQ_COMPARE_TWICE
    + , error1, error2, error3
    +#endif
    + );
    + rseq_after_asm_goto();
    + return 0;
    +abort:
    + rseq_after_asm_goto();
    + RSEQ_INJECT_FAILED
    + return -1;
    +cmpfail:
    + rseq_after_asm_goto();
    + return 1;
    +#ifdef RSEQ_COMPARE_TWICE
    +error1:
    + rseq_after_asm_goto();
    + rseq_bug("cpu_id comparison failed");
    +error2:
    + rseq_after_asm_goto();
    + rseq_bug("1st expected value comparison failed");
    +error3:
    + rseq_after_asm_goto();
    + rseq_bug("2nd expected value comparison failed");
    +#endif
    +}
    +
    +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
    + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
    +
    +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
    + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
    +
    +/* s390 is TSO. */
    +static inline __attribute__((always_inline))
    +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
    + intptr_t *v2, intptr_t newv2,
    + intptr_t newv, int cpu)
    +{
    + RSEQ_INJECT_C(9)
    +
    + __asm__ __volatile__ goto (
    + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    +#endif
    + /* Start rseq by storing table entry pointer into rseq_cs. */
    + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    + RSEQ_INJECT_ASM(3)
    + LONG_CMP " %[expect], %[v]\n\t"
    + "jnz %l[cmpfail]\n\t"
    + RSEQ_INJECT_ASM(4)
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    + LONG_CMP " %[expect], %[v]\n\t"
    + "jnz %l[error2]\n\t"
    +#endif
    + /* try store */
    + LONG_S " %[newv2], %[v2]\n\t"
    + RSEQ_INJECT_ASM(5)
    + /* final store */
    + LONG_S " %[newv], %[v]\n\t"
    + "2:\n\t"
    + RSEQ_INJECT_ASM(6)
    + RSEQ_ASM_DEFINE_ABORT(4, "", abort)
    + : /* gcc asm goto does not allow outputs */
    + : [cpu_id] "r" (cpu),
    + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
    + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    + /* try store input */
    + [v2] "m" (*v2),
    + [newv2] "r" (newv2),
    + /* final store input */
    + [v] "m" (*v),
    + [expect] "r" (expect),
    + [newv] "r" (newv)
    + RSEQ_INJECT_INPUT
    + : "memory", "cc", "r0"
    + RSEQ_INJECT_CLOBBER
    + : abort, cmpfail
    +#ifdef RSEQ_COMPARE_TWICE
    + , error1, error2
    +#endif
    + );
    + rseq_after_asm_goto();
    + return 0;
    +abort:
    + rseq_after_asm_goto();
    + RSEQ_INJECT_FAILED
    + return -1;
    +cmpfail:
    + rseq_after_asm_goto();
    + return 1;
    +#ifdef RSEQ_COMPARE_TWICE
    +error1:
    + rseq_after_asm_goto();
    + rseq_bug("cpu_id comparison failed");
    +error2:
    + rseq_after_asm_goto();
    + rseq_bug("expected value comparison failed");
    +#endif
    +}
    +
    +/* s390 is TSO. */
    +static inline __attribute__((always_inline))
    +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
    + void *dst, void *src, size_t len,
    + intptr_t newv, int cpu)
    +{
    + uint64_t rseq_scratch[3];
    +
    + RSEQ_INJECT_C(9)
    +
    + __asm__ __volatile__ goto (
    + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    +#endif
    + LONG_S " %[src], %[rseq_scratch0]\n\t"
    + LONG_S " %[dst], %[rseq_scratch1]\n\t"
    + LONG_S " %[len], %[rseq_scratch2]\n\t"
    + /* Start rseq by storing table entry pointer into rseq_cs. */
    + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    + RSEQ_INJECT_ASM(3)
    + LONG_CMP " %[expect], %[v]\n\t"
    + "jnz 5f\n\t"
    + RSEQ_INJECT_ASM(4)
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
    + LONG_CMP " %[expect], %[v]\n\t"
    + "jnz 7f\n\t"
    +#endif
    + /* try memcpy */
    + LONG_LT_R " %[len], %[len]\n\t"
    + "jz 333f\n\t"
    + "222:\n\t"
    + "ic %%r0,0(%[src])\n\t"
    + "stc %%r0,0(%[dst])\n\t"
    + LONG_ADDI " %[src], 1\n\t"
    + LONG_ADDI " %[dst], 1\n\t"
    + LONG_ADDI " %[len], -1\n\t"
    + "jnz 222b\n\t"
    + "333:\n\t"
    + RSEQ_INJECT_ASM(5)
    + /* final store */
    + LONG_S " %[newv], %[v]\n\t"
    + "2:\n\t"
    + RSEQ_INJECT_ASM(6)
    + /* teardown */
    + LONG_L " %[len], %[rseq_scratch2]\n\t"
    + LONG_L " %[dst], %[rseq_scratch1]\n\t"
    + LONG_L " %[src], %[rseq_scratch0]\n\t"
    + RSEQ_ASM_DEFINE_ABORT(4,
    + LONG_L " %[len], %[rseq_scratch2]\n\t"
    + LONG_L " %[dst], %[rseq_scratch1]\n\t"
    + LONG_L " %[src], %[rseq_scratch0]\n\t",
    + abort)
    + RSEQ_ASM_DEFINE_CMPFAIL(5,
    + LONG_L " %[len], %[rseq_scratch2]\n\t"
    + LONG_L " %[dst], %[rseq_scratch1]\n\t"
    + LONG_L " %[src], %[rseq_scratch0]\n\t",
    + cmpfail)
    +#ifdef RSEQ_COMPARE_TWICE
    + RSEQ_ASM_DEFINE_CMPFAIL(6,
    + LONG_L " %[len], %[rseq_scratch2]\n\t"
    + LONG_L " %[dst], %[rseq_scratch1]\n\t"
    + LONG_L " %[src], %[rseq_scratch0]\n\t",
    + error1)
    + RSEQ_ASM_DEFINE_CMPFAIL(7,
    + LONG_L " %[len], %[rseq_scratch2]\n\t"
    + LONG_L " %[dst], %[rseq_scratch1]\n\t"
    + LONG_L " %[src], %[rseq_scratch0]\n\t",
    + error2)
    +#endif
    + : /* gcc asm goto does not allow outputs */
    + : [cpu_id] "r" (cpu),
    + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
    + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    + /* final store input */
    + [v] "m" (*v),
    + [expect] "r" (expect),
    + [newv] "r" (newv),
    + /* try memcpy input */
    + [dst] "r" (dst),
    + [src] "r" (src),
    + [len] "r" (len),
    + [rseq_scratch0] "m" (rseq_scratch[0]),
    + [rseq_scratch1] "m" (rseq_scratch[1]),
    + [rseq_scratch2] "m" (rseq_scratch[2])
    + RSEQ_INJECT_INPUT
    + : "memory", "cc", "r0"
    + RSEQ_INJECT_CLOBBER
    + : abort, cmpfail
    +#ifdef RSEQ_COMPARE_TWICE
    + , error1, error2
    +#endif
    + );
    + rseq_after_asm_goto();
    + return 0;
    +abort:
    + rseq_after_asm_goto();
    + RSEQ_INJECT_FAILED
    + return -1;
    +cmpfail:
    + rseq_after_asm_goto();
    + return 1;
    +#ifdef RSEQ_COMPARE_TWICE
    +error1:
    + rseq_after_asm_goto();
    + rseq_bug("cpu_id comparison failed");
    +error2:
    + rseq_after_asm_goto();
    + rseq_bug("expected value comparison failed");
    +#endif
    +}
    +
    +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
    + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
    +
    +#include "rseq-bits-reset.h"
    diff --git a/tools/testing/selftests/rseq/rseq-s390.h b/tools/testing/selftests/rseq/rseq-s390.h
    index 4d32864..46c9259 100644
    --- a/tools/testing/selftests/rseq/rseq-s390.h
    +++ b/tools/testing/selftests/rseq/rseq-s390.h
    @@ -130,476 +130,34 @@ do { \
    "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \
    ".popsection\n\t"

    -static inline __attribute__((always_inline))
    -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
    -{
    - RSEQ_INJECT_C(9)
    +/* Per-cpu-id indexing. */

    - __asm__ __volatile__ goto (
    - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    -#endif
    - /* Start rseq by storing table entry pointer into rseq_cs. */
    - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    - RSEQ_INJECT_ASM(3)
    - LONG_CMP " %[expect], %[v]\n\t"
    - "jnz %l[cmpfail]\n\t"
    - RSEQ_INJECT_ASM(4)
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    - LONG_CMP " %[expect], %[v]\n\t"
    - "jnz %l[error2]\n\t"
    -#endif
    - /* final store */
    - LONG_S " %[newv], %[v]\n\t"
    - "2:\n\t"
    - RSEQ_INJECT_ASM(5)
    - RSEQ_ASM_DEFINE_ABORT(4, "", abort)
    - : /* gcc asm goto does not allow outputs */
    - : [cpu_id] "r" (cpu),
    - [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
    - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    - [v] "m" (*v),
    - [expect] "r" (expect),
    - [newv] "r" (newv)
    - RSEQ_INJECT_INPUT
    - : "memory", "cc", "r0"
    - RSEQ_INJECT_CLOBBER
    - : abort, cmpfail
    -#ifdef RSEQ_COMPARE_TWICE
    - , error1, error2
    -#endif
    - );
    - rseq_after_asm_goto();
    - return 0;
    -abort:
    - rseq_after_asm_goto();
    - RSEQ_INJECT_FAILED
    - return -1;
    -cmpfail:
    - rseq_after_asm_goto();
    - return 1;
    -#ifdef RSEQ_COMPARE_TWICE
    -error1:
    - rseq_after_asm_goto();
    - rseq_bug("cpu_id comparison failed");
    -error2:
    - rseq_after_asm_goto();
    - rseq_bug("expected value comparison failed");
    -#endif
    -}
    -
    -/*
    - * Compare @v against @expectnot. When it does _not_ match, load @v
    - * into @load, and store the content of *@v + voffp into @v.
    - */
    -static inline __attribute__((always_inline))
    -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
    - long voffp, intptr_t *load, int cpu)
    -{
    - RSEQ_INJECT_C(9)
    -
    - __asm__ __volatile__ goto (
    - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    -#endif
    - /* Start rseq by storing table entry pointer into rseq_cs. */
    - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    - RSEQ_INJECT_ASM(3)
    - LONG_L " %%r1, %[v]\n\t"
    - LONG_CMP_R " %%r1, %[expectnot]\n\t"
    - "je %l[cmpfail]\n\t"
    - RSEQ_INJECT_ASM(4)
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    - LONG_L " %%r1, %[v]\n\t"
    - LONG_CMP_R " %%r1, %[expectnot]\n\t"
    - "je %l[error2]\n\t"
    -#endif
    - LONG_S " %%r1, %[load]\n\t"
    - LONG_ADD_R " %%r1, %[voffp]\n\t"
    - LONG_L " %%r1, 0(%%r1)\n\t"
    - /* final store */
    - LONG_S " %%r1, %[v]\n\t"
    - "2:\n\t"
    - RSEQ_INJECT_ASM(5)
    - RSEQ_ASM_DEFINE_ABORT(4, "", abort)
    - : /* gcc asm goto does not allow outputs */
    - : [cpu_id] "r" (cpu),
    - [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
    - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    - /* final store input */
    - [v] "m" (*v),
    - [expectnot] "r" (expectnot),
    - [voffp] "r" (voffp),
    - [load] "m" (*load)
    - RSEQ_INJECT_INPUT
    - : "memory", "cc", "r0", "r1"
    - RSEQ_INJECT_CLOBBER
    - : abort, cmpfail
    -#ifdef RSEQ_COMPARE_TWICE
    - , error1, error2
    -#endif
    - );
    - rseq_after_asm_goto();
    - return 0;
    -abort:
    - rseq_after_asm_goto();
    - RSEQ_INJECT_FAILED
    - return -1;
    -cmpfail:
    - rseq_after_asm_goto();
    - return 1;
    -#ifdef RSEQ_COMPARE_TWICE
    -error1:
    - rseq_after_asm_goto();
    - rseq_bug("cpu_id comparison failed");
    -error2:
    - rseq_after_asm_goto();
    - rseq_bug("expected value comparison failed");
    -#endif
    -}
    -
    -static inline __attribute__((always_inline))
    -int rseq_addv(intptr_t *v, intptr_t count, int cpu)
    -{
    - RSEQ_INJECT_C(9)
    -
    - __asm__ __volatile__ goto (
    - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    -#endif
    - /* Start rseq by storing table entry pointer into rseq_cs. */
    - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    - RSEQ_INJECT_ASM(3)
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    -#endif
    - LONG_L " %%r0, %[v]\n\t"
    - LONG_ADD_R " %%r0, %[count]\n\t"
    - /* final store */
    - LONG_S " %%r0, %[v]\n\t"
    - "2:\n\t"
    - RSEQ_INJECT_ASM(4)
    - RSEQ_ASM_DEFINE_ABORT(4, "", abort)
    - : /* gcc asm goto does not allow outputs */
    - : [cpu_id] "r" (cpu),
    - [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
    - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    - /* final store input */
    - [v] "m" (*v),
    - [count] "r" (count)
    - RSEQ_INJECT_INPUT
    - : "memory", "cc", "r0"
    - RSEQ_INJECT_CLOBBER
    - : abort
    -#ifdef RSEQ_COMPARE_TWICE
    - , error1
    -#endif
    - );
    - rseq_after_asm_goto();
    - return 0;
    -abort:
    - rseq_after_asm_goto();
    - RSEQ_INJECT_FAILED
    - return -1;
    -#ifdef RSEQ_COMPARE_TWICE
    -error1:
    - rseq_after_asm_goto();
    - rseq_bug("cpu_id comparison failed");
    -#endif
    -}
    +#define RSEQ_TEMPLATE_CPU_ID
    +#define RSEQ_TEMPLATE_MO_RELAXED
    +#include "rseq-s390-bits.h"
    +#undef RSEQ_TEMPLATE_MO_RELAXED

    -static inline __attribute__((always_inline))
    -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
    - intptr_t *v2, intptr_t newv2,
    - intptr_t newv, int cpu)
    -{
    - RSEQ_INJECT_C(9)
    +#define RSEQ_TEMPLATE_MO_RELEASE
    +#include "rseq-s390-bits.h"
    +#undef RSEQ_TEMPLATE_MO_RELEASE
    +#undef RSEQ_TEMPLATE_CPU_ID

    - __asm__ __volatile__ goto (
    - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    -#endif
    - /* Start rseq by storing table entry pointer into rseq_cs. */
    - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    - RSEQ_INJECT_ASM(3)
    - LONG_CMP " %[expect], %[v]\n\t"
    - "jnz %l[cmpfail]\n\t"
    - RSEQ_INJECT_ASM(4)
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    - LONG_CMP " %[expect], %[v]\n\t"
    - "jnz %l[error2]\n\t"
    -#endif
    - /* try store */
    - LONG_S " %[newv2], %[v2]\n\t"
    - RSEQ_INJECT_ASM(5)
    - /* final store */
    - LONG_S " %[newv], %[v]\n\t"
    - "2:\n\t"
    - RSEQ_INJECT_ASM(6)
    - RSEQ_ASM_DEFINE_ABORT(4, "", abort)
    - : /* gcc asm goto does not allow outputs */
    - : [cpu_id] "r" (cpu),
    - [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
    - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    - /* try store input */
    - [v2] "m" (*v2),
    - [newv2] "r" (newv2),
    - /* final store input */
    - [v] "m" (*v),
    - [expect] "r" (expect),
    - [newv] "r" (newv)
    - RSEQ_INJECT_INPUT
    - : "memory", "cc", "r0"
    - RSEQ_INJECT_CLOBBER
    - : abort, cmpfail
    -#ifdef RSEQ_COMPARE_TWICE
    - , error1, error2
    -#endif
    - );
    - rseq_after_asm_goto();
    - return 0;
    -abort:
    - rseq_after_asm_goto();
    - RSEQ_INJECT_FAILED
    - return -1;
    -cmpfail:
    - rseq_after_asm_goto();
    - return 1;
    -#ifdef RSEQ_COMPARE_TWICE
    -error1:
    - rseq_after_asm_goto();
    - rseq_bug("cpu_id comparison failed");
    -error2:
    - rseq_after_asm_goto();
    - rseq_bug("expected value comparison failed");
    -#endif
    -}
    +/* Per-mm-cid indexing. */

    -/* s390 is TSO. */
    -static inline __attribute__((always_inline))
    -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
    - intptr_t *v2, intptr_t newv2,
    - intptr_t newv, int cpu)
    -{
    - return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
    -}
    +#define RSEQ_TEMPLATE_MM_CID
    +#define RSEQ_TEMPLATE_MO_RELAXED
    +#include "rseq-s390-bits.h"
    +#undef RSEQ_TEMPLATE_MO_RELAXED

    -static inline __attribute__((always_inline))
    -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
    - intptr_t *v2, intptr_t expect2,
    - intptr_t newv, int cpu)
    -{
    - RSEQ_INJECT_C(9)
    +#define RSEQ_TEMPLATE_MO_RELEASE
    +#include "rseq-s390-bits.h"
    +#undef RSEQ_TEMPLATE_MO_RELEASE
    +#undef RSEQ_TEMPLATE_MM_CID

    - __asm__ __volatile__ goto (
    - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
    -#endif
    - /* Start rseq by storing table entry pointer into rseq_cs. */
    - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    - RSEQ_INJECT_ASM(3)
    - LONG_CMP " %[expect], %[v]\n\t"
    - "jnz %l[cmpfail]\n\t"
    - RSEQ_INJECT_ASM(4)
    - LONG_CMP " %[expect2], %[v2]\n\t"
    - "jnz %l[cmpfail]\n\t"
    - RSEQ_INJECT_ASM(5)
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    - LONG_CMP " %[expect], %[v]\n\t"
    - "jnz %l[error2]\n\t"
    - LONG_CMP " %[expect2], %[v2]\n\t"
    - "jnz %l[error3]\n\t"
    -#endif
    - /* final store */
    - LONG_S " %[newv], %[v]\n\t"
    - "2:\n\t"
    - RSEQ_INJECT_ASM(6)
    - RSEQ_ASM_DEFINE_ABORT(4, "", abort)
    - : /* gcc asm goto does not allow outputs */
    - : [cpu_id] "r" (cpu),
    - [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
    - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    - /* cmp2 input */
    - [v2] "m" (*v2),
    - [expect2] "r" (expect2),
    - /* final store input */
    - [v] "m" (*v),
    - [expect] "r" (expect),
    - [newv] "r" (newv)
    - RSEQ_INJECT_INPUT
    - : "memory", "cc", "r0"
    - RSEQ_INJECT_CLOBBER
    - : abort, cmpfail
    -#ifdef RSEQ_COMPARE_TWICE
    - , error1, error2, error3
    -#endif
    - );
    - rseq_after_asm_goto();
    - return 0;
    -abort:
    - rseq_after_asm_goto();
    - RSEQ_INJECT_FAILED
    - return -1;
    -cmpfail:
    - rseq_after_asm_goto();
    - return 1;
    -#ifdef RSEQ_COMPARE_TWICE
    -error1:
    - rseq_after_asm_goto();
    - rseq_bug("cpu_id comparison failed");
    -error2:
    - rseq_after_asm_goto();
    - rseq_bug("1st expected value comparison failed");
    -error3:
    - rseq_after_asm_goto();
    - rseq_bug("2nd expected value comparison failed");
    -#endif
    -}
    -
    -static inline __attribute__((always_inline))
    -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
    - void *dst, void *src, size_t len,
    - intptr_t newv, int cpu)
    -{
    - uint64_t rseq_scratch[3];
    -
    - RSEQ_INJECT_C(9)
    -
    - __asm__ __volatile__ goto (
    - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    -#endif
    - LONG_S " %[src], %[rseq_scratch0]\n\t"
    - LONG_S " %[dst], %[rseq_scratch1]\n\t"
    - LONG_S " %[len], %[rseq_scratch2]\n\t"
    - /* Start rseq by storing table entry pointer into rseq_cs. */
    - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    - RSEQ_INJECT_ASM(3)
    - LONG_CMP " %[expect], %[v]\n\t"
    - "jnz 5f\n\t"
    - RSEQ_INJECT_ASM(4)
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
    - LONG_CMP " %[expect], %[v]\n\t"
    - "jnz 7f\n\t"
    -#endif
    - /* try memcpy */
    - LONG_LT_R " %[len], %[len]\n\t"
    - "jz 333f\n\t"
    - "222:\n\t"
    - "ic %%r0,0(%[src])\n\t"
    - "stc %%r0,0(%[dst])\n\t"
    - LONG_ADDI " %[src], 1\n\t"
    - LONG_ADDI " %[dst], 1\n\t"
    - LONG_ADDI " %[len], -1\n\t"
    - "jnz 222b\n\t"
    - "333:\n\t"
    - RSEQ_INJECT_ASM(5)
    - /* final store */
    - LONG_S " %[newv], %[v]\n\t"
    - "2:\n\t"
    - RSEQ_INJECT_ASM(6)
    - /* teardown */
    - LONG_L " %[len], %[rseq_scratch2]\n\t"
    - LONG_L " %[dst], %[rseq_scratch1]\n\t"
    - LONG_L " %[src], %[rseq_scratch0]\n\t"
    - RSEQ_ASM_DEFINE_ABORT(4,
    - LONG_L " %[len], %[rseq_scratch2]\n\t"
    - LONG_L " %[dst], %[rseq_scratch1]\n\t"
    - LONG_L " %[src], %[rseq_scratch0]\n\t",
    - abort)
    - RSEQ_ASM_DEFINE_CMPFAIL(5,
    - LONG_L " %[len], %[rseq_scratch2]\n\t"
    - LONG_L " %[dst], %[rseq_scratch1]\n\t"
    - LONG_L " %[src], %[rseq_scratch0]\n\t",
    - cmpfail)
    -#ifdef RSEQ_COMPARE_TWICE
    - RSEQ_ASM_DEFINE_CMPFAIL(6,
    - LONG_L " %[len], %[rseq_scratch2]\n\t"
    - LONG_L " %[dst], %[rseq_scratch1]\n\t"
    - LONG_L " %[src], %[rseq_scratch0]\n\t",
    - error1)
    - RSEQ_ASM_DEFINE_CMPFAIL(7,
    - LONG_L " %[len], %[rseq_scratch2]\n\t"
    - LONG_L " %[dst], %[rseq_scratch1]\n\t"
    - LONG_L " %[src], %[rseq_scratch0]\n\t",
    - error2)
    -#endif
    - : /* gcc asm goto does not allow outputs */
    - : [cpu_id] "r" (cpu),
    - [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
    - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
    - /* final store input */
    - [v] "m" (*v),
    - [expect] "r" (expect),
    - [newv] "r" (newv),
    - /* try memcpy input */
    - [dst] "r" (dst),
    - [src] "r" (src),
    - [len] "r" (len),
    - [rseq_scratch0] "m" (rseq_scratch[0]),
    - [rseq_scratch1] "m" (rseq_scratch[1]),
    - [rseq_scratch2] "m" (rseq_scratch[2])
    - RSEQ_INJECT_INPUT
    - : "memory", "cc", "r0"
    - RSEQ_INJECT_CLOBBER
    - : abort, cmpfail
    -#ifdef RSEQ_COMPARE_TWICE
    - , error1, error2
    -#endif
    - );
    - rseq_after_asm_goto();
    - return 0;
    -abort:
    - rseq_after_asm_goto();
    - RSEQ_INJECT_FAILED
    - return -1;
    -cmpfail:
    - rseq_after_asm_goto();
    - return 1;
    -#ifdef RSEQ_COMPARE_TWICE
    -error1:
    - rseq_after_asm_goto();
    - rseq_bug("cpu_id comparison failed");
    -error2:
    - rseq_after_asm_goto();
    - rseq_bug("expected value comparison failed");
    -#endif
    -}
    +/* APIs which are not based on cpu ids. */

    -/* s390 is TSO. */
    -static inline __attribute__((always_inline))
    -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
    - void *dst, void *src, size_t len,
    - intptr_t newv, int cpu)
    -{
    - return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
    - newv, cpu);
    -}
    +#define RSEQ_TEMPLATE_CPU_ID_NONE
    +#define RSEQ_TEMPLATE_MO_RELAXED
    +#include "rseq-s390-bits.h"
    +#undef RSEQ_TEMPLATE_MO_RELAXED
    +#undef RSEQ_TEMPLATE_CPU_ID_NONE
    \
     
     \ /
      Last update: 2023-03-26 23:21    [W:4.142 / U:0.100 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site