lkml.org 
[lkml]   [2021]   [Dec]   [10]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    /
    From
    Date
    SubjectRe: [PATCH v2 1/9] atomic: Prepare scripts for macro ops
    On Fri, 10 Dec 2021 at 17:27, Peter Zijlstra <peterz@infradead.org> wrote:
    >
    > Due to the limited usability of the GCC 'Labels as Values' extention,
    > specifically it refuses to have a goto in an __always_inline function,
    > even with a compile time constant label pointer, some new atomic ops
    > will need to be introduced as macros.
    >
    > The xchg/cmpxchg family is already macros so extend that code to the
    > regular atomic ops.
    >
    > Specifically introduce meta-'M', meta-'m' and meta-'n' to signify the
    > op is a macro and add arg-'L', arg-'V' and arg-'P' for Label, VarArg
    > and Pair arguments respectively.
    >
    > This unconvered some latent bugs in the instrumentation wrappery,
    > specifically the try_cmpxchg() @oldp argument was instrumented as an
    > atomic_read_write, while it's a regular read_write and many of the
    > gen_xchg() ops had atomic_write instrumentation while
    > atomic_read_write seems more appropriate.

    Very nice, thanks for that!

    > Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

    Acked-by: Marco Elver <elver@google.com>

    > ---
    > include/linux/atomic/atomic-instrumented.h | 78 +++++++--------
    > scripts/atomic/atomic-tbl.sh | 8 +
    > scripts/atomic/atomics.tbl | 5 +
    > scripts/atomic/gen-atomic-fallback.sh | 4
    > scripts/atomic/gen-atomic-instrumented.sh | 142 +++++++++++++++++++----------
    > scripts/atomic/gen-atomic-long.sh | 32 +++++-
    > 6 files changed, 177 insertions(+), 92 deletions(-)
    >
    > --- a/include/linux/atomic/atomic-instrumented.h
    > +++ b/include/linux/atomic/atomic-instrumented.h
    > @@ -501,7 +501,7 @@ static __always_inline bool
    > atomic_try_cmpxchg(atomic_t *v, int *old, int new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic_try_cmpxchg(v, old, new);
    > }
    >
    > @@ -509,7 +509,7 @@ static __always_inline bool
    > atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic_try_cmpxchg_acquire(v, old, new);
    > }
    >
    > @@ -517,7 +517,7 @@ static __always_inline bool
    > atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic_try_cmpxchg_release(v, old, new);
    > }
    >
    > @@ -525,7 +525,7 @@ static __always_inline bool
    > atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic_try_cmpxchg_relaxed(v, old, new);
    > }
    >
    > @@ -1079,7 +1079,7 @@ static __always_inline bool
    > atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic64_try_cmpxchg(v, old, new);
    > }
    >
    > @@ -1087,7 +1087,7 @@ static __always_inline bool
    > atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic64_try_cmpxchg_acquire(v, old, new);
    > }
    >
    > @@ -1095,7 +1095,7 @@ static __always_inline bool
    > atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic64_try_cmpxchg_release(v, old, new);
    > }
    >
    > @@ -1103,7 +1103,7 @@ static __always_inline bool
    > atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
    > }
    >
    > @@ -1657,7 +1657,7 @@ static __always_inline bool
    > atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic_long_try_cmpxchg(v, old, new);
    > }
    >
    > @@ -1665,7 +1665,7 @@ static __always_inline bool
    > atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
    > }
    >
    > @@ -1673,7 +1673,7 @@ static __always_inline bool
    > atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic_long_try_cmpxchg_release(v, old, new);
    > }
    >
    > @@ -1681,7 +1681,7 @@ static __always_inline bool
    > atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
    > {
    > instrument_atomic_read_write(v, sizeof(*v));
    > - instrument_atomic_read_write(old, sizeof(*old));
    > + instrument_read_write(old, sizeof(*old));
    > return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
    > }
    >
    > @@ -1758,84 +1758,84 @@ atomic_long_dec_if_positive(atomic_long_
    > #define xchg(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_xchg(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define xchg_acquire(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define xchg_release(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_xchg_release(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define xchg_relaxed(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg_acquire(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg_release(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg_relaxed(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg64(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg64_acquire(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg64_release(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg64_relaxed(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > @@ -1843,8 +1843,8 @@ atomic_long_dec_if_positive(atomic_long_
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > typeof(oldp) __ai_oldp = (oldp); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
    > arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
    > })
    >
    > @@ -1852,8 +1852,8 @@ atomic_long_dec_if_positive(atomic_long_
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > typeof(oldp) __ai_oldp = (oldp); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
    > arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
    > })
    >
    > @@ -1861,8 +1861,8 @@ atomic_long_dec_if_positive(atomic_long_
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > typeof(oldp) __ai_oldp = (oldp); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
    > arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
    > })
    >
    > @@ -1870,46 +1870,46 @@ atomic_long_dec_if_positive(atomic_long_
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > typeof(oldp) __ai_oldp = (oldp); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
    > arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg_local(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg64_local(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define sync_cmpxchg(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
    > arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > #define cmpxchg_double(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
    > arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > -
    > #define cmpxchg_double_local(ptr, ...) \
    > ({ \
    > typeof(ptr) __ai_ptr = (ptr); \
    > - instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
    > + instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
    > arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
    > })
    >
    > +
    > #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
    > -// 2a9553f0a9d5619f19151092df5cabbbf16ce835
    > +// 66cdf9a0e0a995cba29c61baf018f7ef35974ae5
    > --- a/scripts/atomic/atomic-tbl.sh
    > +++ b/scripts/atomic/atomic-tbl.sh
    > @@ -15,7 +15,7 @@ meta_in()
    > #meta_has_ret(meta)
    > meta_has_ret()
    > {
    > - meta_in "$1" "bBiIfFlR"
    > + meta_in "$1" "bBiIfFlRm"
    > }
    >
    > #meta_has_acquire(meta)
    > @@ -36,6 +36,12 @@ meta_has_relaxed()
    > meta_in "$1" "BFIR"
    > }
    >
    > +#meta_has_macro(meta)
    > +meta_has_macro()
    > +{
    > + meta_in "$1" "mn"
    > +}
    > +
    > #find_fallback_template(pfx, name, sfx, order)
    > find_fallback_template()
    > {
    > --- a/scripts/atomic/atomics.tbl
    > +++ b/scripts/atomic/atomics.tbl
    > @@ -10,12 +10,17 @@
    > # * F/f - fetch: returns base type (has fetch_ variants)
    > # * l - load: returns base type (has _acquire order variant)
    > # * s - store: returns void (has _release order variant)
    > +# * m - macro: with return value
    > +# * n - macro: with No return value
    > #
    > # Where args contains list of type[:name], where type is:
    > # * cv - const pointer to atomic base type (atomic_t/atomic64_t/atomic_long_t)
    > # * v - pointer to atomic base type (atomic_t/atomic64_t/atomic_long_t)
    > +# * P - pointer to pair of atomic base type
    > # * i - base type (int/s64/long)
    > # * p - pointer to base type (int/s64/long)
    > +# * L - label for exception case
    > +# * V:... - vararg
    > #
    > read l cv
    > set s v i
    > --- a/scripts/atomic/gen-atomic-fallback.sh
    > +++ b/scripts/atomic/gen-atomic-fallback.sh
    > @@ -27,7 +27,9 @@ gen_template_fallback()
    > if [ ! -z "${template}" ]; then
    > printf "#ifndef ${atomicname}\n"
    > . ${template}
    > - printf "#define ${atomicname} ${atomicname}\n"
    > + if ! meta_has_macro "${meta}"; then
    > + printf "#define ${atomicname} ${atomicname}\n"
    > + fi
    > printf "#endif\n\n"
    > fi
    > }
    > --- a/scripts/atomic/gen-atomic-instrumented.sh
    > +++ b/scripts/atomic/gen-atomic-instrumented.sh
    > @@ -13,9 +13,15 @@ gen_param_check()
    > local type="${arg%%:*}"
    > local name="$(gen_param_name "${arg}")"
    > local rw="write"
    > + local pfx;
    > + local mult;
    >
    > case "${type#c}" in
    > + v) pfx="atomic_";;
    > + P) pfx="atomic_"; mult="2 * ";;
    > i) return;;
    > + L) return;;
    > + V) return;;
    > esac
    >
    > if [ ${type#c} != ${type} ]; then
    > @@ -27,7 +33,16 @@ gen_param_check()
    > rw="read_write"
    > fi
    >
    > - printf "\tinstrument_atomic_${rw}(${name}, sizeof(*${name}));\n"
    > + if meta_has_macro "${meta}"; then
    > + name="__ai_${name}"
    > + fi
    > +
    > + printf "\tinstrument_${pfx}${rw}(${name}, ${mult}sizeof(*${name}));"
    > + if meta_has_macro "${meta}"; then
    > + printf " \\"
    > + fi
    > + printf "\n"
    > +
    > }
    >
    > #gen_params_checks(meta, arg...)
    > @@ -41,6 +56,52 @@ gen_params_checks()
    > done
    > }
    >
    > +#gen_var(arg)
    > +gen_var()
    > +{
    > + local type="${1%%:*}"
    > + local name="$(gen_param_name "$1")"
    > +
    > + case "${type#c}" in
    > + L) return;;
    > + V) return;;
    > + esac
    > +
    > + printf "\ttypeof(${name}) __ai_${name} = (${name}); \\\\\n";
    > +}
    > +
    > +#gen_vars(arg...)
    > +gen_vars()
    > +{
    > + while [ "$#" -gt 0 ]; do
    > + gen_var "$1"
    > + shift
    > + done
    > +}
    > +
    > +#gen_varg(arg)
    > +gen_varg()
    > +{
    > + local type="${1%%:*}"
    > + local name="$(gen_param_name "$1")"
    > +
    > + case "${type#c}" in
    > + L) printf "${name}";;
    > + V) printf "__VA_ARGS__";;
    > + *) printf "__ai_${name}";;
    > + esac
    > +}
    > +
    > +#gen_vargs(arg...)
    > +gen_vargs()
    > +{
    > + while [ "$#" -gt 0 ]; do
    > + printf "$(gen_varg "$1")"
    > + [ "$#" -gt 1 ] && printf ", "
    > + shift
    > + done
    > +}
    > +
    > #gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...)
    > gen_proto_order_variant()
    > {
    > @@ -51,57 +112,49 @@ gen_proto_order_variant()
    > local order="$1"; shift
    > local atomic="$1"; shift
    > local int="$1"; shift
    > + local atomicname;
    >
    > - local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
    > + if [ -n "${atomic}" ]; then
    > + atomicname="${atomic}_${pfx}${name}${sfx}${order}"
    > + else
    > + atomicname="${pfx}${name}${sfx}${order}"
    > + fi
    >
    > - local ret="$(gen_ret_type "${meta}" "${int}")"
    > - local params="$(gen_params "${int}" "${atomic}" "$@")"
    > local checks="$(gen_params_checks "${meta}" "$@")"
    > local args="$(gen_args "$@")"
    > - local retstmt="$(gen_ret_stmt "${meta}")"
    > -
    > -cat <<EOF
    > -static __always_inline ${ret}
    > -${atomicname}(${params})
    > -{
    > -${checks}
    > - ${retstmt}arch_${atomicname}(${args});
    > -}
    > -EOF
    >
    > - printf "\n"
    > -}
    > -
    > -gen_xchg()
    > -{
    > - local xchg="$1"; shift
    > - local mult="$1"; shift
    > + if meta_has_macro "${meta}"; then
    >
    > - if [ "${xchg%${xchg#try_cmpxchg}}" = "try_cmpxchg" ] ; then
    > + local vars="$(gen_vars "$@")"
    > + local vargs="$(gen_vargs "$@")"
    >
    > cat <<EOF
    > -#define ${xchg}(ptr, oldp, ...) \\
    > +#define ${atomicname}(${args}) \\
    > ({ \\
    > - typeof(ptr) __ai_ptr = (ptr); \\
    > - typeof(oldp) __ai_oldp = (oldp); \\
    > - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
    > - instrument_atomic_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\
    > - arch_${xchg}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
    > +${vars}
    > +${checks}
    > + arch_${atomicname}(${vargs}); \\
    > })
    > EOF
    >
    > else
    >
    > + local ret="$(gen_ret_type "${meta}" "${int}")"
    > + local params="$(gen_params "${int}" "${atomic}" "$@")"
    > + local retstmt="$(gen_ret_stmt "${meta}")"
    > +
    > cat <<EOF
    > -#define ${xchg}(ptr, ...) \\
    > -({ \\
    > - typeof(ptr) __ai_ptr = (ptr); \\
    > - instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
    > - arch_${xchg}(__ai_ptr, __VA_ARGS__); \\
    > -})
    > +static __always_inline ${ret}
    > +${atomicname}(${params})
    > +{
    > +${checks}
    > + ${retstmt}arch_${atomicname}(${args});
    > +}
    > EOF
    >
    > fi
    > +
    > + printf "\n"
    > }
    >
    > cat << EOF
    > @@ -142,24 +195,23 @@ grep '^[a-z]' "$1" | while read name met
    > gen_proto "${meta}" "${name}" "atomic_long" "long" ${args}
    > done
    >
    > -
    > -for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do
    > +for xchg in "xchg" "cmpxchg" "cmpxchg64"; do
    > for order in "" "_acquire" "_release" "_relaxed"; do
    > - gen_xchg "${xchg}${order}" ""
    > - printf "\n"
    > + gen_proto_order_variant "m" "" "${xchg}" "" "${order}" "" "" "v:ptr" "V:..."
    > done
    > done
    >
    > -for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do
    > - gen_xchg "${xchg}" ""
    > - printf "\n"
    > +for order in "" "_acquire" "_release" "_relaxed"; do
    > + gen_proto_order_variant "m" "" "try_cmpxchg" "" "${order}" "" "" "v:ptr" "p:oldp" "V:..."
    > done
    >
    > -gen_xchg "cmpxchg_double" "2 * "
    > -
    > -printf "\n\n"
    > +for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do
    > + gen_proto_order_variant "m" "" "${xchg}" "" "" "" "" "v:ptr" "V:..."
    > +done
    >
    > -gen_xchg "cmpxchg_double_local" "2 * "
    > +for xchg in "cmpxchg_double" "cmpxchg_double_local"; do
    > + gen_proto_order_variant "m" "" "${xchg}" "" "" "" "" "P:ptr" "V:..."
    > +done
    >
    > cat <<EOF
    >
    > --- a/scripts/atomic/gen-atomic-long.sh
    > +++ b/scripts/atomic/gen-atomic-long.sh
    > @@ -17,16 +17,21 @@ gen_cast()
    > printf "($(gen_param_type "${arg}" "${int}" "${atomic}"))"
    > }
    >
    > -#gen_args_cast(int, atomic, arg...)
    > +#gen_args_cast(meta, int, atomic, arg...)
    > gen_args_cast()
    > {
    > + local meta=$1; shift
    > local int="$1"; shift
    > local atomic="$1"; shift
    >
    > while [ "$#" -gt 0 ]; do
    > local cast="$(gen_cast "$1" "${int}" "${atomic}")"
    > local arg="$(gen_param_name "$1")"
    > - printf "${cast}${arg}"
    > + if meta_has_macro "${meta}" && [ "${1%%:*}" != "L" ]; then
    > + printf "${cast}(${arg})"
    > + else
    > + printf "${cast}${arg}"
    > + fi
    > [ "$#" -gt 1 ] && printf ", "
    > shift;
    > done
    > @@ -40,10 +45,24 @@ gen_proto_order_variant()
    > local atomic="$1"; shift
    > local int="$1"; shift
    >
    > - local ret="$(gen_ret_type "${meta}" "long")"
    > - local params="$(gen_params "long" "atomic_long" "$@")"
    > - local argscast="$(gen_args_cast "${int}" "${atomic}" "$@")"
    > - local retstmt="$(gen_ret_stmt "${meta}")"
    > + local argscast="$(gen_args_cast "${meta}" "${int}" "${atomic}" "$@")"
    > +
    > + if meta_has_macro "${meta}"; then
    > +
    > + local args="$(gen_args "$@")"
    > +
    > +cat <<EOF
    > +#define arch_atomic_long_${name}(${args}) \\
    > +({ \\
    > + arch_${atomic}_${name}(${argscast}) \\
    > +})
    > +
    > +EOF
    > + else
    > +
    > + local ret="$(gen_ret_type "${meta}" "long")"
    > + local params="$(gen_params "long" "atomic_long" "$@")"
    > + local retstmt="$(gen_ret_stmt "${meta}")"
    >
    > cat <<EOF
    > static __always_inline ${ret}
    > @@ -53,6 +72,7 @@ arch_atomic_long_${name}(${params})
    > }
    >
    > EOF
    > + fi
    > }
    >
    > cat << EOF
    >
    >

    \
     
     \ /
      Last update: 2021-12-10 18:44    [W:3.374 / U:0.044 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site