| 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
|---|
| 2 | #include <linux/linkage.h> |
|---|
| 3 | #include <asm/percpu.h> |
|---|
| 4 | #include <asm/processor-flags.h> |
|---|
| 5 | |
|---|
| 6 | .text |
|---|
| 7 | |
|---|
| 8 | /* |
|---|
| 9 | * Emulate 'cmpxchg16b %gs:(%rsi)' |
|---|
| 10 | * |
|---|
| 11 | * Inputs: |
|---|
| 12 | * %rsi : memory location to compare |
|---|
| 13 | * %rax : low 64 bits of old value |
|---|
| 14 | * %rdx : high 64 bits of old value |
|---|
| 15 | * %rbx : low 64 bits of new value |
|---|
| 16 | * %rcx : high 64 bits of new value |
|---|
| 17 | * |
|---|
| 18 | * Notably this is not LOCK prefixed and is not safe against NMIs |
|---|
| 19 | */ |
|---|
| 20 | SYM_FUNC_START(this_cpu_cmpxchg16b_emu) |
|---|
| 21 | |
|---|
| 22 | pushfq |
|---|
| 23 | cli |
|---|
| 24 | |
|---|
| 25 | /* if (*ptr == old) */ |
|---|
| 26 | cmpq PER_CPU_VAR(0(%rsi)), %rax |
|---|
| 27 | jne .Lnot_same |
|---|
| 28 | cmpq PER_CPU_VAR(8(%rsi)), %rdx |
|---|
| 29 | jne .Lnot_same |
|---|
| 30 | |
|---|
| 31 | /* *ptr = new */ |
|---|
| 32 | movq %rbx, PER_CPU_VAR(0(%rsi)) |
|---|
| 33 | movq %rcx, PER_CPU_VAR(8(%rsi)) |
|---|
| 34 | |
|---|
| 35 | /* set ZF in EFLAGS to indicate success */ |
|---|
| 36 | orl $X86_EFLAGS_ZF, (%rsp) |
|---|
| 37 | |
|---|
| 38 | popfq |
|---|
| 39 | RET |
|---|
| 40 | |
|---|
| 41 | .Lnot_same: |
|---|
| 42 | /* *ptr != old */ |
|---|
| 43 | |
|---|
| 44 | /* old = *ptr */ |
|---|
| 45 | movq PER_CPU_VAR(0(%rsi)), %rax |
|---|
| 46 | movq PER_CPU_VAR(8(%rsi)), %rdx |
|---|
| 47 | |
|---|
| 48 | /* clear ZF in EFLAGS to indicate failure */ |
|---|
| 49 | andl $(~X86_EFLAGS_ZF), (%rsp) |
|---|
| 50 | |
|---|
| 51 | popfq |
|---|
| 52 | RET |
|---|
| 53 | |
|---|
| 54 | SYM_FUNC_END(this_cpu_cmpxchg16b_emu) |
|---|
| 55 | |
|---|
