123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584 |
- /*-
- * Copyright (c) 2015-2024 Ruslan Bukin <br@bsdpad.com>
- * All rights reserved.
- *
- * Portions of this software were developed by SRI International and the
- * University of Cambridge Computer Laboratory under DARPA/AFRL contract
- * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
- *
- * Portions of this software were developed by the University of Cambridge
- * Computer Laboratory as part of the CTSRD Project, with support from the
- * UK Higher Education Innovation Fund (HEIF).
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
- #ifndef _MACHINE_ATOMIC_H_
- #define _MACHINE_ATOMIC_H_
- #include <sys/atomic_common.h>
- #define fence() __asm __volatile("fence" ::: "memory");
- #define mb() fence()
- #define rmb() fence()
- #define wmb() fence()
- static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t);
- static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t);
- static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t);
- static __inline int atomic_fcmpset_16(__volatile uint16_t *, uint16_t *,
- uint16_t);
- #define ATOMIC_ACQ_REL(NAME, WIDTH) \
- static __inline void \
- atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
- { \
- atomic_##NAME##_##WIDTH(p, v); \
- fence(); \
- } \
- \
- static __inline void \
- atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
- { \
- fence(); \
- atomic_##NAME##_##WIDTH(p, v); \
- }
- #define ATOMIC_CMPSET_ACQ_REL(WIDTH) \
- static __inline int \
- atomic_cmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \
- uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \
- { \
- int retval; \
- \
- retval = atomic_cmpset_##WIDTH(p, cmpval, newval); \
- fence(); \
- return (retval); \
- } \
- \
- static __inline int \
- atomic_cmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \
- uint##WIDTH##_t cmpval, uint##WIDTH##_t newval) \
- { \
- fence(); \
- return (atomic_cmpset_##WIDTH(p, cmpval, newval)); \
- }
- #define ATOMIC_FCMPSET_ACQ_REL(WIDTH) \
- static __inline int \
- atomic_fcmpset_acq_##WIDTH(__volatile uint##WIDTH##_t *p, \
- uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \
- { \
- int retval; \
- \
- retval = atomic_fcmpset_##WIDTH(p, cmpval, newval); \
- fence(); \
- return (retval); \
- } \
- \
- static __inline int \
- atomic_fcmpset_rel_##WIDTH(__volatile uint##WIDTH##_t *p, \
- uint##WIDTH##_t *cmpval, uint##WIDTH##_t newval) \
- { \
- fence(); \
- return (atomic_fcmpset_##WIDTH(p, cmpval, newval)); \
- }
- ATOMIC_CMPSET_ACQ_REL(8);
- ATOMIC_FCMPSET_ACQ_REL(8);
- #define atomic_cmpset_char atomic_cmpset_8
- #define atomic_cmpset_acq_char atomic_cmpset_acq_8
- #define atomic_cmpset_rel_char atomic_cmpset_rel_8
- #define atomic_fcmpset_char atomic_fcmpset_8
- #define atomic_fcmpset_acq_char atomic_fcmpset_acq_8
- #define atomic_fcmpset_rel_char atomic_fcmpset_rel_8
- #define atomic_cmpset_short atomic_cmpset_16
- #define atomic_fcmpset_short atomic_fcmpset_16
- ATOMIC_CMPSET_ACQ_REL(16);
- ATOMIC_FCMPSET_ACQ_REL(16);
- #define atomic_load_acq_16 atomic_load_acq_16
- static __inline uint16_t
- atomic_load_acq_16(volatile uint16_t *p)
- {
- uint16_t ret;
- ret = *p;
- fence();
- return (ret);
- }
- static __inline void
- atomic_store_rel_16(volatile uint16_t *p, uint16_t val)
- {
- fence();
- *p = val;
- }
- #define atomic_cmpset_acq_short atomic_cmpset_acq_16
- #define atomic_fcmpset_acq_short atomic_fcmpset_acq_16
- #define atomic_load_acq_short atomic_load_acq_16
- #define atomic_cmpset_rel_short atomic_cmpset_rel_16
- #define atomic_fcmpset_rel_short atomic_fcmpset_rel_16
- #define atomic_store_rel_short atomic_store_rel_16
- static __inline void
- atomic_add_32(volatile uint32_t *p, uint32_t val)
- {
- __asm __volatile("amoadd.w zero, %1, %0"
- : "+A" (*p)
- : "r" (val)
- : "memory");
- }
- static __inline void
- atomic_subtract_32(volatile uint32_t *p, uint32_t val)
- {
- __asm __volatile("amoadd.w zero, %1, %0"
- : "+A" (*p)
- : "r" (-val)
- : "memory");
- }
- static __inline void
- atomic_set_32(volatile uint32_t *p, uint32_t val)
- {
- __asm __volatile("amoor.w zero, %1, %0"
- : "+A" (*p)
- : "r" (val)
- : "memory");
- }
- static __inline void
- atomic_clear_32(volatile uint32_t *p, uint32_t val)
- {
- __asm __volatile("amoand.w zero, %1, %0"
- : "+A" (*p)
- : "r" (~val)
- : "memory");
- }
- static __inline int
- atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
- {
- uint32_t tmp;
- int res;
- res = 0;
- __asm __volatile(
- "0:"
- "li %1, 1\n" /* Preset to fail */
- "lr.w %0, %2\n"
- "bne %0, %z3, 1f\n"
- "sc.w %1, %z4, %2\n"
- "bnez %1, 0b\n"
- "1:"
- : "=&r" (tmp), "=&r" (res), "+A" (*p)
- : "rJ" ((long)(int32_t)cmpval), "rJ" (newval)
- : "memory");
- return (!res);
- }
- static __inline int
- atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval)
- {
- uint32_t tmp;
- int res;
- res = 0;
- __asm __volatile(
- "0:"
- "li %1, 1\n" /* Preset to fail */
- "lr.w %0, %2\n" /* Load old value */
- "bne %0, %z4, 1f\n" /* Compare */
- "sc.w %1, %z5, %2\n" /* Try to store new value */
- "j 2f\n"
- "1:"
- "sw %0, %3\n" /* Save old value */
- "2:"
- : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
- : "rJ" ((long)(int32_t)*cmpval), "rJ" (newval)
- : "memory");
- return (!res);
- }
- static __inline uint32_t
- atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
- {
- uint32_t ret;
- __asm __volatile("amoadd.w %0, %2, %1"
- : "=&r" (ret), "+A" (*p)
- : "r" (val)
- : "memory");
- return (ret);
- }
- static __inline uint32_t
- atomic_readandclear_32(volatile uint32_t *p)
- {
- uint32_t ret;
- uint32_t val;
- val = 0;
- __asm __volatile("amoswap.w %0, %2, %1"
- : "=&r"(ret), "+A" (*p)
- : "r" (val)
- : "memory");
- return (ret);
- }
- #define atomic_add_int atomic_add_32
- #define atomic_clear_int atomic_clear_32
- #define atomic_cmpset_int atomic_cmpset_32
- #define atomic_fcmpset_int atomic_fcmpset_32
- #define atomic_fetchadd_int atomic_fetchadd_32
- #define atomic_readandclear_int atomic_readandclear_32
- #define atomic_set_int atomic_set_32
- #define atomic_subtract_int atomic_subtract_32
- ATOMIC_ACQ_REL(set, 32)
- ATOMIC_ACQ_REL(clear, 32)
- ATOMIC_ACQ_REL(add, 32)
- ATOMIC_ACQ_REL(subtract, 32)
- ATOMIC_CMPSET_ACQ_REL(32);
- ATOMIC_FCMPSET_ACQ_REL(32);
- static __inline uint32_t
- atomic_load_acq_32(volatile uint32_t *p)
- {
- uint32_t ret;
- ret = *p;
- fence();
- return (ret);
- }
- static __inline void
- atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
- {
- fence();
- *p = val;
- }
- #define atomic_add_acq_int atomic_add_acq_32
- #define atomic_clear_acq_int atomic_clear_acq_32
- #define atomic_cmpset_acq_int atomic_cmpset_acq_32
- #define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
- #define atomic_load_acq_int atomic_load_acq_32
- #define atomic_set_acq_int atomic_set_acq_32
- #define atomic_subtract_acq_int atomic_subtract_acq_32
- #define atomic_add_rel_int atomic_add_rel_32
- #define atomic_clear_rel_int atomic_clear_rel_32
- #define atomic_cmpset_rel_int atomic_cmpset_rel_32
- #define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
- #define atomic_set_rel_int atomic_set_rel_32
- #define atomic_subtract_rel_int atomic_subtract_rel_32
- #define atomic_store_rel_int atomic_store_rel_32
- static __inline void
- atomic_add_64(volatile uint64_t *p, uint64_t val)
- {
- __asm __volatile("amoadd.d zero, %1, %0"
- : "+A" (*p)
- : "r" (val)
- : "memory");
- }
- static __inline void
- atomic_subtract_64(volatile uint64_t *p, uint64_t val)
- {
- __asm __volatile("amoadd.d zero, %1, %0"
- : "+A" (*p)
- : "r" (-val)
- : "memory");
- }
- static __inline void
- atomic_set_64(volatile uint64_t *p, uint64_t val)
- {
- __asm __volatile("amoor.d zero, %1, %0"
- : "+A" (*p)
- : "r" (val)
- : "memory");
- }
- static __inline void
- atomic_clear_64(volatile uint64_t *p, uint64_t val)
- {
- __asm __volatile("amoand.d zero, %1, %0"
- : "+A" (*p)
- : "r" (~val)
- : "memory");
- }
- static __inline int
- atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
- {
- uint64_t tmp;
- int res;
- res = 0;
- __asm __volatile(
- "0:"
- "li %1, 1\n" /* Preset to fail */
- "lr.d %0, %2\n"
- "bne %0, %z3, 1f\n"
- "sc.d %1, %z4, %2\n"
- "bnez %1, 0b\n"
- "1:"
- : "=&r" (tmp), "=&r" (res), "+A" (*p)
- : "rJ" (cmpval), "rJ" (newval)
- : "memory");
- return (!res);
- }
- static __inline int
- atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval)
- {
- uint64_t tmp;
- int res;
- res = 0;
- __asm __volatile(
- "0:"
- "li %1, 1\n" /* Preset to fail */
- "lr.d %0, %2\n" /* Load old value */
- "bne %0, %z4, 1f\n" /* Compare */
- "sc.d %1, %z5, %2\n" /* Try to store new value */
- "j 2f\n"
- "1:"
- "sd %0, %3\n" /* Save old value */
- "2:"
- : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval)
- : "rJ" (*cmpval), "rJ" (newval)
- : "memory");
- return (!res);
- }
- static __inline uint64_t
- atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
- {
- uint64_t ret;
- __asm __volatile("amoadd.d %0, %2, %1"
- : "=&r" (ret), "+A" (*p)
- : "r" (val)
- : "memory");
- return (ret);
- }
- static __inline uint64_t
- atomic_readandclear_64(volatile uint64_t *p)
- {
- uint64_t ret;
- uint64_t val;
- val = 0;
- __asm __volatile("amoswap.d %0, %2, %1"
- : "=&r"(ret), "+A" (*p)
- : "r" (val)
- : "memory");
- return (ret);
- }
- static __inline uint32_t
- atomic_swap_32(volatile uint32_t *p, uint32_t val)
- {
- uint32_t old;
- __asm __volatile("amoswap.w %0, %2, %1"
- : "=&r"(old), "+A" (*p)
- : "r" (val)
- : "memory");
- return (old);
- }
- static __inline uint64_t
- atomic_swap_64(volatile uint64_t *p, uint64_t val)
- {
- uint64_t old;
- __asm __volatile("amoswap.d %0, %2, %1"
- : "=&r"(old), "+A" (*p)
- : "r" (val)
- : "memory");
- return (old);
- }
- #define atomic_swap_int atomic_swap_32
- #define atomic_add_long atomic_add_64
- #define atomic_clear_long atomic_clear_64
- #define atomic_cmpset_long atomic_cmpset_64
- #define atomic_fcmpset_long atomic_fcmpset_64
- #define atomic_fetchadd_long atomic_fetchadd_64
- #define atomic_readandclear_long atomic_readandclear_64
- #define atomic_set_long atomic_set_64
- #define atomic_subtract_long atomic_subtract_64
- #define atomic_swap_long atomic_swap_64
- #define atomic_add_ptr atomic_add_64
- #define atomic_clear_ptr atomic_clear_64
- #define atomic_cmpset_ptr atomic_cmpset_64
- #define atomic_fcmpset_ptr atomic_fcmpset_64
- #define atomic_fetchadd_ptr atomic_fetchadd_64
- #define atomic_readandclear_ptr atomic_readandclear_64
- #define atomic_set_ptr atomic_set_64
- #define atomic_subtract_ptr atomic_subtract_64
- #define atomic_swap_ptr atomic_swap_64
- ATOMIC_ACQ_REL(set, 64)
- ATOMIC_ACQ_REL(clear, 64)
- ATOMIC_ACQ_REL(add, 64)
- ATOMIC_ACQ_REL(subtract, 64)
- ATOMIC_CMPSET_ACQ_REL(64);
- ATOMIC_FCMPSET_ACQ_REL(64);
- static __inline uint64_t
- atomic_load_acq_64(volatile uint64_t *p)
- {
- uint64_t ret;
- ret = *p;
- fence();
- return (ret);
- }
- static __inline void
- atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
- {
- fence();
- *p = val;
- }
- #define atomic_add_acq_long atomic_add_acq_64
- #define atomic_clear_acq_long atomic_clear_acq_64
- #define atomic_cmpset_acq_long atomic_cmpset_acq_64
- #define atomic_fcmpset_acq_long atomic_fcmpset_acq_64
- #define atomic_load_acq_long atomic_load_acq_64
- #define atomic_set_acq_long atomic_set_acq_64
- #define atomic_subtract_acq_long atomic_subtract_acq_64
- #define atomic_add_acq_ptr atomic_add_acq_64
- #define atomic_clear_acq_ptr atomic_clear_acq_64
- #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
- #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64
- #define atomic_load_acq_ptr atomic_load_acq_64
- #define atomic_set_acq_ptr atomic_set_acq_64
- #define atomic_subtract_acq_ptr atomic_subtract_acq_64
- #undef ATOMIC_ACQ_REL
- static __inline void
- atomic_thread_fence_acq(void)
- {
- fence();
- }
- static __inline void
- atomic_thread_fence_rel(void)
- {
- fence();
- }
- static __inline void
- atomic_thread_fence_acq_rel(void)
- {
- fence();
- }
- static __inline void
- atomic_thread_fence_seq_cst(void)
- {
- fence();
- }
- #define atomic_add_rel_long atomic_add_rel_64
- #define atomic_clear_rel_long atomic_clear_rel_64
- #define atomic_add_rel_long atomic_add_rel_64
- #define atomic_clear_rel_long atomic_clear_rel_64
- #define atomic_cmpset_rel_long atomic_cmpset_rel_64
- #define atomic_fcmpset_rel_long atomic_fcmpset_rel_64
- #define atomic_set_rel_long atomic_set_rel_64
- #define atomic_subtract_rel_long atomic_subtract_rel_64
- #define atomic_store_rel_long atomic_store_rel_64
- #define atomic_add_rel_ptr atomic_add_rel_64
- #define atomic_clear_rel_ptr atomic_clear_rel_64
- #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
- #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64
- #define atomic_set_rel_ptr atomic_set_rel_64
- #define atomic_subtract_rel_ptr atomic_subtract_rel_64
- #define atomic_store_rel_ptr atomic_store_rel_64
- #include <sys/_atomic_subword.h>
- #endif /* _MACHINE_ATOMIC_H_ */
|