Warning: This file is not a C or C++ file. It does not have highlighting.
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
|---|---|
| 2 | /* |
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited |
| 4 | */ |
| 5 | #ifndef __ASM_BARRIER_H |
| 6 | #define __ASM_BARRIER_H |
| 7 | |
| 8 | /* |
| 9 | * Hint encoding: |
| 10 | * |
| 11 | * Bit4: ordering or completion (0: completion, 1: ordering) |
| 12 | * Bit3: barrier for previous read (0: true, 1: false) |
| 13 | * Bit2: barrier for previous write (0: true, 1: false) |
| 14 | * Bit1: barrier for succeeding read (0: true, 1: false) |
| 15 | * Bit0: barrier for succeeding write (0: true, 1: false) |
| 16 | * |
| 17 | * Hint 0x700: barrier for "read after read" from the same address |
| 18 | */ |
| 19 | |
| 20 | #define DBAR(hint) __asm__ __volatile__("dbar %0 " : : "I"(hint) : "memory") |
| 21 | |
| 22 | #define crwrw 0b00000 |
| 23 | #define cr_r_ 0b00101 |
| 24 | #define c_w_w 0b01010 |
| 25 | |
| 26 | #define orwrw 0b10000 |
| 27 | #define or_r_ 0b10101 |
| 28 | #define o_w_w 0b11010 |
| 29 | |
| 30 | #define orw_w 0b10010 |
| 31 | #define or_rw 0b10100 |
| 32 | |
| 33 | #define c_sync() DBAR(crwrw) |
| 34 | #define c_rsync() DBAR(cr_r_) |
| 35 | #define c_wsync() DBAR(c_w_w) |
| 36 | |
| 37 | #define o_sync() DBAR(orwrw) |
| 38 | #define o_rsync() DBAR(or_r_) |
| 39 | #define o_wsync() DBAR(o_w_w) |
| 40 | |
| 41 | #define ldacq_mb() DBAR(or_rw) |
| 42 | #define strel_mb() DBAR(orw_w) |
| 43 | |
| 44 | #define mb() c_sync() |
| 45 | #define rmb() c_rsync() |
| 46 | #define wmb() c_wsync() |
| 47 | #define iob() c_sync() |
| 48 | #define wbflush() c_sync() |
| 49 | |
| 50 | #define __smp_mb() o_sync() |
| 51 | #define __smp_rmb() o_rsync() |
| 52 | #define __smp_wmb() o_wsync() |
| 53 | |
| 54 | #ifdef CONFIG_SMP |
| 55 | #define __WEAK_LLSC_MB " dbar 0x700 \n" |
| 56 | #else |
| 57 | #define __WEAK_LLSC_MB " \n" |
| 58 | #endif |
| 59 | |
| 60 | #define __smp_mb__before_atomic() barrier() |
| 61 | #define __smp_mb__after_atomic() barrier() |
| 62 | |
| 63 | /** |
| 64 | * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise |
| 65 | * @index: array element index |
| 66 | * @size: number of elements in array |
| 67 | * |
| 68 | * Returns: |
| 69 | * 0 - (@index < @size) |
| 70 | */ |
| 71 | #define array_index_mask_nospec array_index_mask_nospec |
| 72 | static inline unsigned long array_index_mask_nospec(unsigned long index, |
| 73 | unsigned long size) |
| 74 | { |
| 75 | unsigned long mask; |
| 76 | |
| 77 | __asm__ __volatile__( |
| 78 | "sltu %0, %1, %2\n\t" |
| 79 | #if (__SIZEOF_LONG__ == 4) |
| 80 | "sub.w %0, $zero, %0\n\t" |
| 81 | #elif (__SIZEOF_LONG__ == 8) |
| 82 | "sub.d %0, $zero, %0\n\t" |
| 83 | #endif |
| 84 | : "=r" (mask) |
| 85 | : "r" (index), "r" (size) |
| 86 | :); |
| 87 | |
| 88 | return mask; |
| 89 | } |
| 90 | |
| 91 | #define __smp_load_acquire(p) \ |
| 92 | ({ \ |
| 93 | typeof(*p) ___p1 = READ_ONCE(*p); \ |
| 94 | compiletime_assert_atomic_type(*p); \ |
| 95 | ldacq_mb(); \ |
| 96 | ___p1; \ |
| 97 | }) |
| 98 | |
| 99 | #define __smp_store_release(p, v) \ |
| 100 | do { \ |
| 101 | compiletime_assert_atomic_type(*p); \ |
| 102 | strel_mb(); \ |
| 103 | WRITE_ONCE(*p, v); \ |
| 104 | } while (0) |
| 105 | |
| 106 | #define __smp_store_mb(p, v) \ |
| 107 | do { \ |
| 108 | union { typeof(p) __val; char __c[1]; } __u = \ |
| 109 | { .__val = (__force typeof(p)) (v) }; \ |
| 110 | unsigned long __tmp; \ |
| 111 | switch (sizeof(p)) { \ |
| 112 | case 1: \ |
| 113 | *(volatile __u8 *)&p = *(__u8 *)__u.__c; \ |
| 114 | __smp_mb(); \ |
| 115 | break; \ |
| 116 | case 2: \ |
| 117 | *(volatile __u16 *)&p = *(__u16 *)__u.__c; \ |
| 118 | __smp_mb(); \ |
| 119 | break; \ |
| 120 | case 4: \ |
| 121 | __asm__ __volatile__( \ |
| 122 | "amswap_db.w %[tmp], %[val], %[mem] \n" \ |
| 123 | : [mem] "+ZB" (*(u32 *)&p), [tmp] "=&r" (__tmp) \ |
| 124 | : [val] "r" (*(__u32 *)__u.__c) \ |
| 125 | : ); \ |
| 126 | break; \ |
| 127 | case 8: \ |
| 128 | __asm__ __volatile__( \ |
| 129 | "amswap_db.d %[tmp], %[val], %[mem] \n" \ |
| 130 | : [mem] "+ZB" (*(u64 *)&p), [tmp] "=&r" (__tmp) \ |
| 131 | : [val] "r" (*(__u64 *)__u.__c) \ |
| 132 | : ); \ |
| 133 | break; \ |
| 134 | } \ |
| 135 | } while (0) |
| 136 | |
| 137 | #include <asm-generic/barrier.h> |
| 138 | |
| 139 | #endif /* __ASM_BARRIER_H */ |
| 140 |
Warning: This file is not a C or C++ file. It does not have highlighting.