Adding upstream version 0.6.0.
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
parent
c49a9029dc
commit
7f70a05c55
GPG key ID:
FBB4F0E80A80222F
465 changed files with 60158 additions and 0 deletions
167
include/gcc/aarch64/ck_f_pr.h
Normal file
167
include/gcc/aarch64/ck_f_pr.h
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
/* DO NOT EDIT. This is auto-generated from feature.sh */
|
||||
#define CK_F_PR_ADD_16
|
||||
#define CK_F_PR_ADD_32
|
||||
#define CK_F_PR_ADD_64
|
||||
#define CK_F_PR_ADD_8
|
||||
#define CK_F_PR_ADD_CHAR
|
||||
#define CK_F_PR_ADD_INT
|
||||
#define CK_F_PR_ADD_PTR
|
||||
#define CK_F_PR_ADD_SHORT
|
||||
#define CK_F_PR_ADD_UINT
|
||||
#define CK_F_PR_AND_16
|
||||
#define CK_F_PR_AND_32
|
||||
#define CK_F_PR_AND_64
|
||||
#define CK_F_PR_AND_8
|
||||
#define CK_F_PR_AND_CHAR
|
||||
#define CK_F_PR_AND_INT
|
||||
#define CK_F_PR_AND_PTR
|
||||
#define CK_F_PR_AND_SHORT
|
||||
#define CK_F_PR_AND_UINT
|
||||
#define CK_F_PR_BARRIER
|
||||
#define CK_F_PR_CAS_16
|
||||
#define CK_F_PR_CAS_16_VALUE
|
||||
#define CK_F_PR_CAS_32
|
||||
#define CK_F_PR_CAS_32_VALUE
|
||||
#define CK_F_PR_CAS_64
|
||||
#define CK_F_PR_CAS_64_VALUE
|
||||
#define CK_F_PR_CAS_64_2
|
||||
#define CK_F_PR_CAS_64_2_VALUE
|
||||
#define CK_F_PR_CAS_DOUBLE
|
||||
#define CK_F_PR_CAS_DOUBLE_VALUE
|
||||
#define CK_F_PR_CAS_8
|
||||
#define CK_F_PR_CAS_8_VALUE
|
||||
#define CK_F_PR_CAS_CHAR
|
||||
#define CK_F_PR_CAS_CHAR_VALUE
|
||||
#define CK_F_PR_CAS_INT
|
||||
#define CK_F_PR_CAS_INT_VALUE
|
||||
#define CK_F_PR_CAS_PTR
|
||||
#define CK_F_PR_CAS_PTR_2
|
||||
#define CK_F_PR_CAS_PTR_2_VALUE
|
||||
#define CK_F_PR_CAS_PTR_VALUE
|
||||
#define CK_F_PR_CAS_SHORT
|
||||
#define CK_F_PR_CAS_SHORT_VALUE
|
||||
#define CK_F_PR_CAS_UINT
|
||||
#define CK_F_PR_CAS_UINT_VALUE
|
||||
#define CK_F_PR_DEC_16
|
||||
#define CK_F_PR_DEC_32
|
||||
#define CK_F_PR_DEC_64
|
||||
#define CK_F_PR_DEC_8
|
||||
#define CK_F_PR_DEC_CHAR
|
||||
#define CK_F_PR_DEC_INT
|
||||
#define CK_F_PR_DEC_PTR
|
||||
#define CK_F_PR_DEC_SHORT
|
||||
#define CK_F_PR_DEC_UINT
|
||||
#define CK_F_PR_FAA_16
|
||||
#define CK_F_PR_FAA_32
|
||||
#define CK_F_PR_FAA_64
|
||||
#define CK_F_PR_FAA_8
|
||||
#define CK_F_PR_FAA_CHAR
|
||||
#define CK_F_PR_FAA_INT
|
||||
#define CK_F_PR_FAA_PTR
|
||||
#define CK_F_PR_FAA_SHORT
|
||||
#define CK_F_PR_FAA_UINT
|
||||
#define CK_F_PR_FAS_16
|
||||
#define CK_F_PR_FAS_32
|
||||
#define CK_F_PR_FAS_64
|
||||
#define CK_F_PR_FAS_8
|
||||
#define CK_F_PR_FAS_CHAR
|
||||
#define CK_F_PR_FAS_INT
|
||||
#define CK_F_PR_FAS_PTR
|
||||
#define CK_F_PR_FAS_SHORT
|
||||
#define CK_F_PR_FAS_UINT
|
||||
#define CK_F_PR_FENCE_ATOMIC
|
||||
#define CK_F_PR_FENCE_ATOMIC_LOAD
|
||||
#define CK_F_PR_FENCE_ATOMIC_STORE
|
||||
#define CK_F_PR_FENCE_LOAD
|
||||
#define CK_F_PR_FENCE_LOAD_ATOMIC
|
||||
#define CK_F_PR_FENCE_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_LOAD_STORE
|
||||
#define CK_F_PR_FENCE_MEMORY
|
||||
#define CK_F_PR_FENCE_STORE
|
||||
#define CK_F_PR_FENCE_STORE_ATOMIC
|
||||
#define CK_F_PR_FENCE_STORE_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_ATOMIC
|
||||
#define CK_F_PR_FENCE_STRICT_ATOMIC_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_ATOMIC_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD_ATOMIC
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_MEMORY
|
||||
#define CK_F_PR_FENCE_STRICT_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_STORE_ATOMIC
|
||||
#define CK_F_PR_FENCE_STRICT_STORE_LOAD
|
||||
#define CK_F_PR_INC_16
|
||||
#define CK_F_PR_INC_32
|
||||
#define CK_F_PR_INC_64
|
||||
#define CK_F_PR_INC_8
|
||||
#define CK_F_PR_INC_CHAR
|
||||
#define CK_F_PR_INC_INT
|
||||
#define CK_F_PR_INC_PTR
|
||||
#define CK_F_PR_INC_SHORT
|
||||
#define CK_F_PR_INC_UINT
|
||||
#define CK_F_PR_LOAD_16
|
||||
#define CK_F_PR_LOAD_32
|
||||
#define CK_F_PR_LOAD_64
|
||||
#define CK_F_PR_LOAD_DOUBLE
|
||||
#define CK_F_PR_LOAD_8
|
||||
#define CK_F_PR_LOAD_CHAR
|
||||
#define CK_F_PR_LOAD_INT
|
||||
#define CK_F_PR_LOAD_PTR
|
||||
#define CK_F_PR_LOAD_SHORT
|
||||
#define CK_F_PR_LOAD_UINT
|
||||
#define CK_F_PR_NEG_16
|
||||
#define CK_F_PR_NEG_32
|
||||
#define CK_F_PR_NEG_64
|
||||
#define CK_F_PR_NEG_8
|
||||
#define CK_F_PR_NEG_CHAR
|
||||
#define CK_F_PR_NEG_INT
|
||||
#define CK_F_PR_NEG_PTR
|
||||
#define CK_F_PR_NEG_SHORT
|
||||
#define CK_F_PR_NEG_UINT
|
||||
#define CK_F_PR_NOT_16
|
||||
#define CK_F_PR_NOT_32
|
||||
#define CK_F_PR_NOT_64
|
||||
#define CK_F_PR_NOT_8
|
||||
#define CK_F_PR_NOT_CHAR
|
||||
#define CK_F_PR_NOT_INT
|
||||
#define CK_F_PR_NOT_PTR
|
||||
#define CK_F_PR_NOT_SHORT
|
||||
#define CK_F_PR_NOT_UINT
|
||||
#define CK_F_PR_OR_16
|
||||
#define CK_F_PR_OR_32
|
||||
#define CK_F_PR_OR_64
|
||||
#define CK_F_PR_OR_8
|
||||
#define CK_F_PR_OR_CHAR
|
||||
#define CK_F_PR_OR_INT
|
||||
#define CK_F_PR_OR_PTR
|
||||
#define CK_F_PR_OR_SHORT
|
||||
#define CK_F_PR_OR_UINT
|
||||
#define CK_F_PR_STALL
|
||||
#define CK_F_PR_STORE_16
|
||||
#define CK_F_PR_STORE_32
|
||||
#define CK_F_PR_STORE_64
|
||||
#define CK_F_PR_STORE_DOUBLE
|
||||
#define CK_F_PR_STORE_8
|
||||
#define CK_F_PR_STORE_CHAR
|
||||
#define CK_F_PR_STORE_INT
|
||||
#define CK_F_PR_STORE_PTR
|
||||
#define CK_F_PR_STORE_SHORT
|
||||
#define CK_F_PR_STORE_UINT
|
||||
#define CK_F_PR_SUB_16
|
||||
#define CK_F_PR_SUB_32
|
||||
#define CK_F_PR_SUB_64
|
||||
#define CK_F_PR_SUB_8
|
||||
#define CK_F_PR_SUB_CHAR
|
||||
#define CK_F_PR_SUB_INT
|
||||
#define CK_F_PR_SUB_PTR
|
||||
#define CK_F_PR_SUB_SHORT
|
||||
#define CK_F_PR_SUB_UINT
|
||||
#define CK_F_PR_XOR_16
|
||||
#define CK_F_PR_XOR_32
|
||||
#define CK_F_PR_XOR_64
|
||||
#define CK_F_PR_XOR_8
|
||||
#define CK_F_PR_XOR_CHAR
|
||||
#define CK_F_PR_XOR_INT
|
||||
#define CK_F_PR_XOR_PTR
|
||||
#define CK_F_PR_XOR_SHORT
|
||||
#define CK_F_PR_XOR_UINT
|
||||
227
include/gcc/aarch64/ck_pr.h
Normal file
227
include/gcc/aarch64/ck_pr.h
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
/*
|
||||
* Copyright 2009-2016 Samy Al Bahra.
|
||||
* Copyright 2013-2016 Olivier Houchard.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_AARCH64_H
|
||||
#define CK_PR_AARCH64_H
|
||||
|
||||
#ifndef CK_PR_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
#include <ck_cc.h>
|
||||
#include <ck_md.h>
|
||||
|
||||
/*
|
||||
* The following represent supported atomic operations.
|
||||
* These operations may be emulated.
|
||||
*/
|
||||
#include "ck_f_pr.h"
|
||||
|
||||
/*
|
||||
* Minimum interface requirement met.
|
||||
*/
|
||||
#define CK_F_PR
|
||||
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_stall(void)
|
||||
{
|
||||
|
||||
__asm__ __volatile__("" ::: "memory");
|
||||
return;
|
||||
}
|
||||
|
||||
#define CK_DMB_SY __asm __volatile("dmb ish" : : "r" (0) : "memory")
|
||||
#define CK_DMB_LD __asm __volatile("dmb ishld" : : "r" (0) : "memory")
|
||||
#define CK_DMB_ST __asm __volatile("dmb ishst" : : "r" (0) : "memory")
|
||||
|
||||
#define CK_PR_FENCE(T, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_fence_strict_##T(void) \
|
||||
{ \
|
||||
I; \
|
||||
}
|
||||
|
||||
CK_PR_FENCE(atomic, CK_DMB_ST)
|
||||
CK_PR_FENCE(atomic_store, CK_DMB_ST)
|
||||
CK_PR_FENCE(atomic_load, CK_DMB_SY)
|
||||
CK_PR_FENCE(store_atomic, CK_DMB_ST)
|
||||
CK_PR_FENCE(load_atomic, CK_DMB_SY)
|
||||
CK_PR_FENCE(store, CK_DMB_ST)
|
||||
CK_PR_FENCE(store_load, CK_DMB_SY)
|
||||
CK_PR_FENCE(load, CK_DMB_LD)
|
||||
CK_PR_FENCE(load_store, CK_DMB_SY)
|
||||
CK_PR_FENCE(memory, CK_DMB_SY)
|
||||
CK_PR_FENCE(acquire, CK_DMB_SY)
|
||||
CK_PR_FENCE(release, CK_DMB_SY)
|
||||
CK_PR_FENCE(acqrel, CK_DMB_SY)
|
||||
CK_PR_FENCE(lock, CK_DMB_SY)
|
||||
CK_PR_FENCE(unlock, CK_DMB_SY)
|
||||
|
||||
#undef CK_PR_FENCE
|
||||
|
||||
#undef CK_DMB_SI
|
||||
#undef CK_DMB_LD
|
||||
#undef CK_DMB_ST
|
||||
|
||||
#define CK_PR_LOAD(S, M, T, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_md_load_##S(const M *target) \
|
||||
{ \
|
||||
long r = 0; \
|
||||
__asm__ __volatile__(I " %w0, [%1];" \
|
||||
: "=r" (r) \
|
||||
: "r" (target) \
|
||||
: "memory"); \
|
||||
return ((T)r); \
|
||||
}
|
||||
#define CK_PR_LOAD_64(S, M, T, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_md_load_##S(const M *target) \
|
||||
{ \
|
||||
long r = 0; \
|
||||
__asm__ __volatile__(I " %0, [%1];" \
|
||||
: "=r" (r) \
|
||||
: "r" (target) \
|
||||
: "memory"); \
|
||||
return ((T)r); \
|
||||
}
|
||||
|
||||
|
||||
CK_PR_LOAD_64(ptr, void, void *, "ldr")
|
||||
|
||||
#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, I)
|
||||
#define CK_PR_LOAD_S_64(S, T, I) CK_PR_LOAD_64(S, T, T, I)
|
||||
|
||||
CK_PR_LOAD_S_64(64, uint64_t, "ldr")
|
||||
CK_PR_LOAD_S(32, uint32_t, "ldr")
|
||||
CK_PR_LOAD_S(16, uint16_t, "ldrh")
|
||||
CK_PR_LOAD_S(8, uint8_t, "ldrb")
|
||||
CK_PR_LOAD_S(uint, unsigned int, "ldr")
|
||||
CK_PR_LOAD_S(int, int, "ldr")
|
||||
CK_PR_LOAD_S(short, short, "ldrh")
|
||||
CK_PR_LOAD_S(char, char, "ldrb")
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_LOAD_S_64(double, double, "ldr")
|
||||
#endif
|
||||
|
||||
#undef CK_PR_LOAD_S
|
||||
#undef CK_PR_LOAD_S_64
|
||||
#undef CK_PR_LOAD
|
||||
#undef CK_PR_LAOD_64
|
||||
|
||||
#define CK_PR_STORE(S, M, T, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I " %w1, [%0]" \
|
||||
: \
|
||||
: "r" (target), \
|
||||
"r" (v) \
|
||||
: "memory"); \
|
||||
return; \
|
||||
}
|
||||
#define CK_PR_STORE_64(S, M, T, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I " %1, [%0]" \
|
||||
: \
|
||||
: "r" (target), \
|
||||
"r" (v) \
|
||||
: "memory"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_STORE_64(ptr, void, const void *, "str")
|
||||
|
||||
#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, I)
|
||||
#define CK_PR_STORE_S_64(S, T, I) CK_PR_STORE_64(S, T, T, I)
|
||||
|
||||
CK_PR_STORE_S_64(64, uint64_t, "str")
|
||||
CK_PR_STORE_S(32, uint32_t, "str")
|
||||
CK_PR_STORE_S(16, uint16_t, "strh")
|
||||
CK_PR_STORE_S(8, uint8_t, "strb")
|
||||
CK_PR_STORE_S(uint, unsigned int, "str")
|
||||
CK_PR_STORE_S(int, int, "str")
|
||||
CK_PR_STORE_S(short, short, "strh")
|
||||
CK_PR_STORE_S(char, char, "strb")
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_STORE_S_64(double, double, "str")
|
||||
#endif
|
||||
|
||||
#undef CK_PR_STORE_S
|
||||
#undef CK_PR_STORE_S_64
|
||||
#undef CK_PR_STORE
|
||||
#undef CK_PR_STORE_64
|
||||
|
||||
#ifdef CK_MD_LSE_ENABLE
|
||||
#include "ck_pr_lse.h"
|
||||
#else
|
||||
#include "ck_pr_llsc.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* ck_pr_neg_*() functions can only be implemented via LL/SC, as there are no
|
||||
* LSE alternatives.
|
||||
*/
|
||||
#define CK_PR_NEG(N, M, T, W, R) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_neg_##N(M *target) \
|
||||
{ \
|
||||
T previous = 0; \
|
||||
T tmp = 0; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldxr" W " %" R "0, [%2];" \
|
||||
"neg %" R "0, %" R "0;" \
|
||||
"stxr" W " %w1, %" R "0, [%2];" \
|
||||
"cbnz %w1, 1b;" \
|
||||
: "=&r" (previous), \
|
||||
"=&r" (tmp) \
|
||||
: "r" (target) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_NEG(ptr, void, void *, "", "")
|
||||
CK_PR_NEG(64, uint64_t, uint64_t, "", "")
|
||||
|
||||
#define CK_PR_NEG_S(S, T, W) \
|
||||
CK_PR_NEG(S, T, T, W, "w") \
|
||||
|
||||
CK_PR_NEG_S(32, uint32_t, "")
|
||||
CK_PR_NEG_S(uint, unsigned int, "")
|
||||
CK_PR_NEG_S(int, int, "")
|
||||
CK_PR_NEG_S(16, uint16_t, "h")
|
||||
CK_PR_NEG_S(8, uint8_t, "b")
|
||||
CK_PR_NEG_S(short, short, "h")
|
||||
CK_PR_NEG_S(char, char, "b")
|
||||
|
||||
#undef CK_PR_NEG_S
|
||||
#undef CK_PR_NEG
|
||||
|
||||
#endif /* CK_PR_AARCH64_H */
|
||||
|
||||
352
include/gcc/aarch64/ck_pr_llsc.h
Normal file
352
include/gcc/aarch64/ck_pr_llsc.h
Normal file
|
|
@ -0,0 +1,352 @@
|
|||
/*
|
||||
* Copyright 2009-2016 Samy Al Bahra.
|
||||
* Copyright 2013-2016 Olivier Houchard.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_AARCH64_LLSC_H
|
||||
#define CK_PR_AARCH64_LLSC_H
|
||||
|
||||
#ifndef CK_PR_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_64_2_value(uint64_t target[2], uint64_t compare[2], uint64_t set[2], uint64_t value[2])
|
||||
{
|
||||
uint64_t tmp1, tmp2;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"ldxp %0, %1, [%4];"
|
||||
"mov %2, %0;"
|
||||
"mov %3, %1;"
|
||||
"eor %0, %0, %5;"
|
||||
"eor %1, %1, %6;"
|
||||
"orr %1, %0, %1;"
|
||||
"mov %w0, #0;"
|
||||
"cbnz %1, 2f;"
|
||||
"stxp %w0, %7, %8, [%4];"
|
||||
"cbnz %w0, 1b;"
|
||||
"mov %w0, #1;"
|
||||
"2:"
|
||||
: "=&r" (tmp1), "=&r" (tmp2), "=&r" (value[0]), "=&r" (value[1])
|
||||
: "r" (target), "r" (compare[0]), "r" (compare[1]), "r" (set[0]), "r" (set[1])
|
||||
: "cc", "memory");
|
||||
|
||||
return (tmp1);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value)
|
||||
{
|
||||
return (ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *, target),
|
||||
CK_CPP_CAST(uint64_t *, compare),
|
||||
CK_CPP_CAST(uint64_t *, set),
|
||||
CK_CPP_CAST(uint64_t *, value)));
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2])
|
||||
{
|
||||
uint64_t tmp1, tmp2;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"ldxp %0, %1, [%2];"
|
||||
"eor %0, %0, %3;"
|
||||
"eor %1, %1, %4;"
|
||||
"orr %1, %0, %1;"
|
||||
"mov %w0, #0;"
|
||||
"cbnz %1, 2f;"
|
||||
"stxp %w0, %5, %6, [%2];"
|
||||
"cbnz %w0, 1b;"
|
||||
"mov %w0, #1;"
|
||||
"2:"
|
||||
: "=&r" (tmp1), "=&r" (tmp2)
|
||||
: "r" (target), "r" (compare[0]), "r" (compare[1]), "r" (set[0]), "r" (set[1])
|
||||
: "cc", "memory");
|
||||
|
||||
return (tmp1);
|
||||
}
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_2(void *target, void *compare, void *set)
|
||||
{
|
||||
return (ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, target),
|
||||
CK_CPP_CAST(uint64_t *, compare),
|
||||
CK_CPP_CAST(uint64_t *, set)));
|
||||
}
|
||||
|
||||
|
||||
#define CK_PR_CAS(N, M, T, W, R) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \
|
||||
{ \
|
||||
T previous; \
|
||||
T tmp; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldxr" W " %" R "0, [%2];" \
|
||||
"cmp %" R "0, %" R "4;" \
|
||||
"b.ne 2f;" \
|
||||
"stxr" W " %w1, %" R "3, [%2];" \
|
||||
"cbnz %w1, 1b;" \
|
||||
"2:" \
|
||||
: "=&r" (previous), \
|
||||
"=&r" (tmp) \
|
||||
: "r" (target), \
|
||||
"r" (set), \
|
||||
"r" (compare) \
|
||||
: "memory", "cc"); \
|
||||
*(T *)value = previous; \
|
||||
return (previous == compare); \
|
||||
} \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N(M *target, T compare, T set) \
|
||||
{ \
|
||||
T previous; \
|
||||
T tmp; \
|
||||
__asm__ __volatile__( \
|
||||
"1:" \
|
||||
"ldxr" W " %" R "0, [%2];" \
|
||||
"cmp %" R "0, %" R "4;" \
|
||||
"b.ne 2f;" \
|
||||
"stxr" W " %w1, %" R "3, [%2];" \
|
||||
"cbnz %w1, 1b;" \
|
||||
"2:" \
|
||||
: "=&r" (previous), \
|
||||
"=&r" (tmp) \
|
||||
: "r" (target), \
|
||||
"r" (set), \
|
||||
"r" (compare) \
|
||||
: "memory", "cc"); \
|
||||
return (previous == compare); \
|
||||
}
|
||||
|
||||
CK_PR_CAS(ptr, void, void *, "", "")
|
||||
|
||||
#define CK_PR_CAS_S(N, M, W, R) CK_PR_CAS(N, M, M, W, R)
|
||||
CK_PR_CAS_S(64, uint64_t, "", "")
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_CAS_S(double, double, "", "")
|
||||
#endif
|
||||
CK_PR_CAS_S(32, uint32_t, "", "w")
|
||||
CK_PR_CAS_S(uint, unsigned int, "", "w")
|
||||
CK_PR_CAS_S(int, int, "", "w")
|
||||
CK_PR_CAS_S(16, uint16_t, "h", "w")
|
||||
CK_PR_CAS_S(8, uint8_t, "b", "w")
|
||||
CK_PR_CAS_S(short, short, "h", "w")
|
||||
CK_PR_CAS_S(char, char, "b", "w")
|
||||
|
||||
|
||||
#undef CK_PR_CAS_S
|
||||
#undef CK_PR_CAS
|
||||
|
||||
#define CK_PR_FAS(N, M, T, W, R) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_fas_##N(M *target, T v) \
|
||||
{ \
|
||||
T previous; \
|
||||
T tmp; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldxr" W " %" R "0, [%2];" \
|
||||
"stxr" W " %w1, %" R "3, [%2];"\
|
||||
"cbnz %w1, 1b;" \
|
||||
: "=&r" (previous), \
|
||||
"=&r" (tmp) \
|
||||
: "r" (target), \
|
||||
"r" (v) \
|
||||
: "memory", "cc"); \
|
||||
return (previous); \
|
||||
}
|
||||
|
||||
CK_PR_FAS(64, uint64_t, uint64_t, "", "")
|
||||
CK_PR_FAS(32, uint32_t, uint32_t, "", "w")
|
||||
CK_PR_FAS(ptr, void, void *, "", "")
|
||||
CK_PR_FAS(int, int, int, "", "w")
|
||||
CK_PR_FAS(uint, unsigned int, unsigned int, "", "w")
|
||||
CK_PR_FAS(16, uint16_t, uint16_t, "h", "w")
|
||||
CK_PR_FAS(8, uint8_t, uint8_t, "b", "w")
|
||||
CK_PR_FAS(short, short, short, "h", "w")
|
||||
CK_PR_FAS(char, char, char, "b", "w")
|
||||
|
||||
|
||||
#undef CK_PR_FAS
|
||||
|
||||
#define CK_PR_UNARY(O, N, M, T, I, W, R) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##O##_##N(M *target) \
|
||||
{ \
|
||||
T previous = 0; \
|
||||
T tmp = 0; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldxr" W " %" R "0, [%2];" \
|
||||
I ";" \
|
||||
"stxr" W " %w1, %" R "0, [%2];" \
|
||||
"cbnz %w1, 1b;" \
|
||||
: "=&r" (previous), \
|
||||
"=&r" (tmp) \
|
||||
: "r" (target) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_UNARY(inc, ptr, void, void *, "add %0, %0, #1", "", "")
|
||||
CK_PR_UNARY(dec, ptr, void, void *, "sub %0, %0, #1", "", "")
|
||||
CK_PR_UNARY(not, ptr, void, void *, "mvn %0, %0", "", "")
|
||||
CK_PR_UNARY(inc, 64, uint64_t, uint64_t, "add %0, %0, #1", "", "")
|
||||
CK_PR_UNARY(dec, 64, uint64_t, uint64_t, "sub %0, %0, #1", "", "")
|
||||
CK_PR_UNARY(not, 64, uint64_t, uint64_t, "mvn %0, %0", "", "")
|
||||
|
||||
#define CK_PR_UNARY_S(S, T, W) \
|
||||
CK_PR_UNARY(inc, S, T, T, "add %w0, %w0, #1", W, "w") \
|
||||
CK_PR_UNARY(dec, S, T, T, "sub %w0, %w0, #1", W, "w") \
|
||||
CK_PR_UNARY(not, S, T, T, "mvn %w0, %w0", W, "w") \
|
||||
|
||||
CK_PR_UNARY_S(32, uint32_t, "")
|
||||
CK_PR_UNARY_S(uint, unsigned int, "")
|
||||
CK_PR_UNARY_S(int, int, "")
|
||||
CK_PR_UNARY_S(16, uint16_t, "h")
|
||||
CK_PR_UNARY_S(8, uint8_t, "b")
|
||||
CK_PR_UNARY_S(short, short, "h")
|
||||
CK_PR_UNARY_S(char, char, "b")
|
||||
|
||||
#undef CK_PR_UNARY_S
|
||||
#undef CK_PR_UNARY
|
||||
|
||||
#define CK_PR_BINARY(O, N, M, T, I, W, R) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##O##_##N(M *target, T delta) \
|
||||
{ \
|
||||
T previous; \
|
||||
T tmp; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldxr" W " %" R "0, [%2];"\
|
||||
I " %" R "0, %" R "0, %" R "3;" \
|
||||
"stxr" W " %w1, %" R "0, [%2];" \
|
||||
"cbnz %w1, 1b;" \
|
||||
: "=&r" (previous), \
|
||||
"=&r" (tmp) \
|
||||
: "r" (target), \
|
||||
"r" (delta) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "", "")
|
||||
CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "", "")
|
||||
CK_PR_BINARY(or, ptr, void, uintptr_t, "orr", "", "")
|
||||
CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "", "")
|
||||
CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "", "")
|
||||
CK_PR_BINARY(and, 64, uint64_t, uint64_t, "and", "", "")
|
||||
CK_PR_BINARY(add, 64, uint64_t, uint64_t, "add", "", "")
|
||||
CK_PR_BINARY(or, 64, uint64_t, uint64_t, "orr", "", "")
|
||||
CK_PR_BINARY(sub, 64, uint64_t, uint64_t, "sub", "", "")
|
||||
CK_PR_BINARY(xor, 64, uint64_t, uint64_t, "eor", "", "")
|
||||
|
||||
#define CK_PR_BINARY_S(S, T, W) \
|
||||
CK_PR_BINARY(and, S, T, T, "and", W, "w") \
|
||||
CK_PR_BINARY(add, S, T, T, "add", W, "w") \
|
||||
CK_PR_BINARY(or, S, T, T, "orr", W, "w") \
|
||||
CK_PR_BINARY(sub, S, T, T, "sub", W, "w") \
|
||||
CK_PR_BINARY(xor, S, T, T, "eor", W, "w")
|
||||
|
||||
CK_PR_BINARY_S(32, uint32_t, "")
|
||||
CK_PR_BINARY_S(uint, unsigned int, "")
|
||||
CK_PR_BINARY_S(int, int, "")
|
||||
CK_PR_BINARY_S(16, uint16_t, "h")
|
||||
CK_PR_BINARY_S(8, uint8_t, "b")
|
||||
CK_PR_BINARY_S(short, short, "h")
|
||||
CK_PR_BINARY_S(char, char, "b")
|
||||
|
||||
#undef CK_PR_BINARY_S
|
||||
#undef CK_PR_BINARY
|
||||
|
||||
CK_CC_INLINE static void *
|
||||
ck_pr_faa_ptr(void *target, uintptr_t delta)
|
||||
{
|
||||
uintptr_t previous, r, tmp;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"ldxr %0, [%3];"
|
||||
"add %1, %4, %0;"
|
||||
"stxr %w2, %1, [%3];"
|
||||
"cbnz %w2, 1b;"
|
||||
: "=&r" (previous),
|
||||
"=&r" (r),
|
||||
"=&r" (tmp)
|
||||
: "r" (target),
|
||||
"r" (delta)
|
||||
: "memory", "cc");
|
||||
|
||||
return (void *)(previous);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static uint64_t
|
||||
ck_pr_faa_64(uint64_t *target, uint64_t delta)
|
||||
{
|
||||
uint64_t previous, r, tmp;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"ldxr %0, [%3];"
|
||||
"add %1, %4, %0;"
|
||||
"stxr %w2, %1, [%3];"
|
||||
"cbnz %w2, 1b;"
|
||||
: "=&r" (previous),
|
||||
"=&r" (r),
|
||||
"=&r" (tmp)
|
||||
: "r" (target),
|
||||
"r" (delta)
|
||||
: "memory", "cc");
|
||||
|
||||
return (previous);
|
||||
}
|
||||
|
||||
#define CK_PR_FAA(S, T, W) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_faa_##S(T *target, T delta) \
|
||||
{ \
|
||||
T previous, r, tmp; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldxr" W " %w0, [%3];" \
|
||||
"add %w1, %w4, %w0;" \
|
||||
"stxr" W " %w2, %w1, [%3];" \
|
||||
"cbnz %w2, 1b;" \
|
||||
: "=&r" (previous), \
|
||||
"=&r" (r), \
|
||||
"=&r" (tmp) \
|
||||
: "r" (target), \
|
||||
"r" (delta) \
|
||||
: "memory", "cc"); \
|
||||
return (previous); \
|
||||
}
|
||||
|
||||
CK_PR_FAA(32, uint32_t, "")
|
||||
CK_PR_FAA(uint, unsigned int, "")
|
||||
CK_PR_FAA(int, int, "")
|
||||
CK_PR_FAA(16, uint16_t, "h")
|
||||
CK_PR_FAA(8, uint8_t, "b")
|
||||
CK_PR_FAA(short, short, "h")
|
||||
CK_PR_FAA(char, char, "b")
|
||||
|
||||
#undef CK_PR_FAA
|
||||
|
||||
#endif /* CK_PR_AARCH64_LLSC_H */
|
||||
298
include/gcc/aarch64/ck_pr_lse.h
Normal file
298
include/gcc/aarch64/ck_pr_lse.h
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
/*
|
||||
* Copyright 2009-2016 Samy Al Bahra.
|
||||
* Copyright 2013-2016 Olivier Houchard.
|
||||
* Copyright 2016 Alexey Kopytov.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_AARCH64_LSE_H
|
||||
#define CK_PR_AARCH64_LSE_H
|
||||
|
||||
#ifndef CK_PR_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_64_2_value(uint64_t target[2], uint64_t compare[2], uint64_t set[2], uint64_t value[2])
|
||||
{
|
||||
uint64_t tmp1;
|
||||
uint64_t tmp2;
|
||||
register uint64_t x0 __asm__ ("x0") = compare[0];
|
||||
register uint64_t x1 __asm__ ("x1") = compare[1];
|
||||
register uint64_t x2 __asm__ ("x2") = set[0];
|
||||
register uint64_t x3 __asm__ ("x3") = set[1];
|
||||
|
||||
__asm__ __volatile__("casp %0, %1, %4, %5, [%6];"
|
||||
"eor %2, %0, %7;"
|
||||
"eor %3, %1, %8;"
|
||||
"orr %2, %2, %3;"
|
||||
: "+&r" (x0), "+&r" (x1), "=&r" (tmp1), "=&r" (tmp2)
|
||||
: "r" (x2), "r" (x3), "r" (target), "r" (compare[0]), "r" (compare[1])
|
||||
: "memory");
|
||||
|
||||
value[0] = x0;
|
||||
value[1] = x1;
|
||||
|
||||
return (!!tmp1);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value)
|
||||
{
|
||||
return (ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *, target),
|
||||
CK_CPP_CAST(uint64_t *, compare),
|
||||
CK_CPP_CAST(uint64_t *, set),
|
||||
CK_CPP_CAST(uint64_t *, value)));
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2])
|
||||
{
|
||||
register uint64_t x0 __asm__ ("x0") = compare[0];
|
||||
register uint64_t x1 __asm__ ("x1") = compare[1];
|
||||
register uint64_t x2 __asm__ ("x2") = set[0];
|
||||
register uint64_t x3 __asm__ ("x3") = set[1];
|
||||
|
||||
__asm__ __volatile__("casp %0, %1, %2, %3, [%4];"
|
||||
"eor %0, %0, %5;"
|
||||
"eor %1, %1, %6;"
|
||||
"orr %0, %0, %1;"
|
||||
: "+&r" (x0), "+&r" (x1)
|
||||
: "r" (x2), "r" (x3), "r" (target), "r" (compare[0]), "r" (compare[1])
|
||||
: "memory");
|
||||
|
||||
return (!!x0);
|
||||
}
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_2(void *target, void *compare, void *set)
|
||||
{
|
||||
return (ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, target),
|
||||
CK_CPP_CAST(uint64_t *, compare),
|
||||
CK_CPP_CAST(uint64_t *, set)));
|
||||
}
|
||||
|
||||
|
||||
#define CK_PR_CAS(N, M, T, W, R) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \
|
||||
{ \
|
||||
*(T *)value = compare; \
|
||||
__asm__ __volatile__( \
|
||||
"cas" W " %" R "0, %" R "2, [%1];" \
|
||||
: "+&r" (*(T *)value) \
|
||||
: "r" (target), \
|
||||
"r" (set) \
|
||||
: "memory"); \
|
||||
return (*(T *)value == compare); \
|
||||
} \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N(M *target, T compare, T set) \
|
||||
{ \
|
||||
T previous = compare; \
|
||||
__asm__ __volatile__( \
|
||||
"cas" W " %" R "0, %" R "2, [%1];" \
|
||||
: "+&r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (set) \
|
||||
: "memory"); \
|
||||
return (previous == compare); \
|
||||
}
|
||||
|
||||
CK_PR_CAS(ptr, void, void *, "", "")
|
||||
|
||||
#define CK_PR_CAS_S(N, M, W, R) CK_PR_CAS(N, M, M, W, R)
|
||||
CK_PR_CAS_S(64, uint64_t, "", "")
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_CAS_S(double, double, "", "")
|
||||
#endif
|
||||
CK_PR_CAS_S(32, uint32_t, "", "w")
|
||||
CK_PR_CAS_S(uint, unsigned int, "", "w")
|
||||
CK_PR_CAS_S(int, int, "", "w")
|
||||
CK_PR_CAS_S(16, uint16_t, "h", "w")
|
||||
CK_PR_CAS_S(8, uint8_t, "b", "w")
|
||||
CK_PR_CAS_S(short, short, "h", "w")
|
||||
CK_PR_CAS_S(char, char, "b", "w")
|
||||
|
||||
|
||||
#undef CK_PR_CAS_S
|
||||
#undef CK_PR_CAS
|
||||
|
||||
#define CK_PR_FAS(N, M, T, W, R) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_fas_##N(M *target, T v) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__( \
|
||||
"swp" W " %" R "2, %" R "0, [%1];" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (v) \
|
||||
: "memory"); \
|
||||
return (previous); \
|
||||
}
|
||||
|
||||
CK_PR_FAS(64, uint64_t, uint64_t, "", "")
|
||||
CK_PR_FAS(32, uint32_t, uint32_t, "", "w")
|
||||
CK_PR_FAS(ptr, void, void *, "", "")
|
||||
CK_PR_FAS(int, int, int, "", "w")
|
||||
CK_PR_FAS(uint, unsigned int, unsigned int, "", "w")
|
||||
CK_PR_FAS(16, uint16_t, uint16_t, "h", "w")
|
||||
CK_PR_FAS(8, uint8_t, uint8_t, "b", "w")
|
||||
CK_PR_FAS(short, short, short, "h", "w")
|
||||
CK_PR_FAS(char, char, char, "b", "w")
|
||||
|
||||
|
||||
#undef CK_PR_FAS
|
||||
|
||||
#define CK_PR_UNARY(O, N, M, T, I, W, R, S) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##O##_##N(M *target) \
|
||||
{ \
|
||||
__asm__ __volatile__(I ";" \
|
||||
"st" S W " " R "0, [%0];" \
|
||||
: \
|
||||
: "r" (target) \
|
||||
: "x0", "memory"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_UNARY(inc, ptr, void, void *, "mov x0, 1", "", "x", "add")
|
||||
CK_PR_UNARY(dec, ptr, void, void *, "mov x0, -1", "", "x", "add")
|
||||
CK_PR_UNARY(not, ptr, void, void *, "mov x0, -1", "", "x", "eor")
|
||||
CK_PR_UNARY(inc, 64, uint64_t, uint64_t, "mov x0, 1", "", "x", "add")
|
||||
CK_PR_UNARY(dec, 64, uint64_t, uint64_t, "mov x0, -1", "", "x", "add")
|
||||
CK_PR_UNARY(not, 64, uint64_t, uint64_t, "mov x0, -1", "", "x", "eor")
|
||||
|
||||
#define CK_PR_UNARY_S(S, T, W) \
|
||||
CK_PR_UNARY(inc, S, T, T, "mov w0, 1", W, "w", "add") \
|
||||
CK_PR_UNARY(dec, S, T, T, "mov w0, -1", W, "w", "add") \
|
||||
CK_PR_UNARY(not, S, T, T, "mov w0, -1", W, "w", "eor") \
|
||||
|
||||
CK_PR_UNARY_S(32, uint32_t, "")
|
||||
CK_PR_UNARY_S(uint, unsigned int, "")
|
||||
CK_PR_UNARY_S(int, int, "")
|
||||
CK_PR_UNARY_S(16, uint16_t, "h")
|
||||
CK_PR_UNARY_S(8, uint8_t, "b")
|
||||
CK_PR_UNARY_S(short, short, "h")
|
||||
CK_PR_UNARY_S(char, char, "b")
|
||||
|
||||
#undef CK_PR_UNARY_S
|
||||
#undef CK_PR_UNARY
|
||||
|
||||
#define CK_PR_BINARY(O, N, M, T, S, W, R, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##O##_##N(M *target, T delta) \
|
||||
{ \
|
||||
__asm__ __volatile__(I ";" \
|
||||
"st" S W " %" R "0, [%1];" \
|
||||
: "+&r" (delta) \
|
||||
: "r" (target) \
|
||||
: "memory"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_BINARY(and, ptr, void, uintptr_t, "clr", "", "", "mvn %0, %0")
|
||||
CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "", "", "")
|
||||
CK_PR_BINARY(or, ptr, void, uintptr_t, "set", "", "", "")
|
||||
CK_PR_BINARY(sub, ptr, void, uintptr_t, "add", "", "", "neg %0, %0")
|
||||
CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "", "", "")
|
||||
CK_PR_BINARY(and, 64, uint64_t, uint64_t, "clr", "", "", "mvn %0, %0")
|
||||
CK_PR_BINARY(add, 64, uint64_t, uint64_t, "add", "", "", "")
|
||||
CK_PR_BINARY(or, 64, uint64_t, uint64_t, "set", "", "", "")
|
||||
CK_PR_BINARY(sub, 64, uint64_t, uint64_t, "add", "", "", "neg %0, %0")
|
||||
CK_PR_BINARY(xor, 64, uint64_t, uint64_t, "eor", "", "", "")
|
||||
|
||||
#define CK_PR_BINARY_S(S, T, W) \
|
||||
CK_PR_BINARY(and, S, T, T, "clr", W, "w", "mvn %w0, %w0") \
|
||||
CK_PR_BINARY(add, S, T, T, "add", W, "w", "") \
|
||||
CK_PR_BINARY(or, S, T, T, "set", W, "w", "") \
|
||||
CK_PR_BINARY(sub, S, T, T, "add", W, "w", "neg %w0, %w0") \
|
||||
CK_PR_BINARY(xor, S, T, T, "eor", W, "w", "")
|
||||
|
||||
CK_PR_BINARY_S(32, uint32_t, "")
|
||||
CK_PR_BINARY_S(uint, unsigned int, "")
|
||||
CK_PR_BINARY_S(int, int, "")
|
||||
CK_PR_BINARY_S(16, uint16_t, "h")
|
||||
CK_PR_BINARY_S(8, uint8_t, "b")
|
||||
CK_PR_BINARY_S(short, short, "h")
|
||||
CK_PR_BINARY_S(char, char, "b")
|
||||
|
||||
#undef CK_PR_BINARY_S
|
||||
#undef CK_PR_BINARY
|
||||
|
||||
CK_CC_INLINE static void *
|
||||
ck_pr_faa_ptr(void *target, uintptr_t delta)
|
||||
{
|
||||
uintptr_t previous;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"ldadd %2, %0, [%1];"
|
||||
: "=r" (previous)
|
||||
: "r" (target),
|
||||
"r" (delta)
|
||||
: "memory");
|
||||
|
||||
return (void *)(previous);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static uint64_t
|
||||
ck_pr_faa_64(uint64_t *target, uint64_t delta)
|
||||
{
|
||||
uint64_t previous;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"ldadd %2, %0, [%1];"
|
||||
: "=r" (previous)
|
||||
: "r" (target),
|
||||
"r" (delta)
|
||||
: "memory");
|
||||
|
||||
return (previous);
|
||||
}
|
||||
|
||||
#define CK_PR_FAA(S, T, W) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_faa_##S(T *target, T delta) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__( \
|
||||
"ldadd" W " %w2, %w0, [%1];" \
|
||||
: "=r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (delta) \
|
||||
: "memory"); \
|
||||
return (previous); \
|
||||
}
|
||||
|
||||
CK_PR_FAA(32, uint32_t, "")
|
||||
CK_PR_FAA(uint, unsigned int, "")
|
||||
CK_PR_FAA(int, int, "")
|
||||
CK_PR_FAA(16, uint16_t, "h")
|
||||
CK_PR_FAA(8, uint8_t, "b")
|
||||
CK_PR_FAA(short, short, "h")
|
||||
CK_PR_FAA(char, char, "b")
|
||||
|
||||
#undef CK_PR_FAA
|
||||
|
||||
#endif /* CK_PR_AARCH64_LSE_H */
|
||||
162
include/gcc/arm/ck_f_pr.h
Normal file
162
include/gcc/arm/ck_f_pr.h
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
/* DO NOT EDIT. This is auto-generated from feature.sh */
|
||||
#define CK_F_PR_ADD_16
|
||||
#define CK_F_PR_ADD_32
|
||||
#define CK_F_PR_ADD_8
|
||||
#define CK_F_PR_ADD_CHAR
|
||||
#define CK_F_PR_ADD_INT
|
||||
#define CK_F_PR_ADD_PTR
|
||||
#define CK_F_PR_ADD_SHORT
|
||||
#define CK_F_PR_ADD_UINT
|
||||
#define CK_F_PR_AND_16
|
||||
#define CK_F_PR_AND_32
|
||||
#define CK_F_PR_AND_8
|
||||
#define CK_F_PR_AND_CHAR
|
||||
#define CK_F_PR_AND_INT
|
||||
#define CK_F_PR_AND_PTR
|
||||
#define CK_F_PR_AND_SHORT
|
||||
#define CK_F_PR_AND_UINT
|
||||
#define CK_F_PR_BARRIER
|
||||
#define CK_F_PR_CAS_16
|
||||
#define CK_F_PR_CAS_16_VALUE
|
||||
#define CK_F_PR_CAS_32
|
||||
#define CK_F_PR_CAS_32_VALUE
|
||||
#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
|
||||
#define CK_F_PR_CAS_64
|
||||
#define CK_F_PR_CAS_64_VALUE
|
||||
#define CK_F_PR_CAS_DOUBLE
|
||||
#define CK_F_PR_CAS_DOUBLE_VALUE
|
||||
#endif
|
||||
#define CK_F_PR_CAS_8
|
||||
#define CK_F_PR_CAS_8_VALUE
|
||||
#define CK_F_PR_CAS_CHAR
|
||||
#define CK_F_PR_CAS_CHAR_VALUE
|
||||
#define CK_F_PR_CAS_INT
|
||||
#define CK_F_PR_CAS_INT_VALUE
|
||||
#define CK_F_PR_CAS_PTR
|
||||
#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
|
||||
#define CK_F_PR_CAS_PTR_2
|
||||
#define CK_F_PR_CAS_PTR_2_VALUE
|
||||
#endif
|
||||
#define CK_F_PR_CAS_PTR_VALUE
|
||||
#define CK_F_PR_CAS_SHORT
|
||||
#define CK_F_PR_CAS_SHORT_VALUE
|
||||
#define CK_F_PR_CAS_UINT
|
||||
#define CK_F_PR_CAS_UINT_VALUE
|
||||
#define CK_F_PR_DEC_16
|
||||
#define CK_F_PR_DEC_32
|
||||
#define CK_F_PR_DEC_8
|
||||
#define CK_F_PR_DEC_CHAR
|
||||
#define CK_F_PR_DEC_INT
|
||||
#define CK_F_PR_DEC_PTR
|
||||
#define CK_F_PR_DEC_SHORT
|
||||
#define CK_F_PR_DEC_UINT
|
||||
#define CK_F_PR_FAA_16
|
||||
#define CK_F_PR_FAA_32
|
||||
#define CK_F_PR_FAA_8
|
||||
#define CK_F_PR_FAA_CHAR
|
||||
#define CK_F_PR_FAA_INT
|
||||
#define CK_F_PR_FAA_PTR
|
||||
#define CK_F_PR_FAA_SHORT
|
||||
#define CK_F_PR_FAA_UINT
|
||||
#define CK_F_PR_FAS_16
|
||||
#define CK_F_PR_FAS_32
|
||||
#define CK_F_PR_FAS_8
|
||||
#define CK_F_PR_FAS_CHAR
|
||||
#define CK_F_PR_FAS_INT
|
||||
#define CK_F_PR_FAS_PTR
|
||||
#define CK_F_PR_FAS_SHORT
|
||||
#define CK_F_PR_FAS_UINT
|
||||
#define CK_F_PR_FENCE_ATOMIC
|
||||
#define CK_F_PR_FENCE_ATOMIC_LOAD
|
||||
#define CK_F_PR_FENCE_ATOMIC_STORE
|
||||
#define CK_F_PR_FENCE_LOAD
|
||||
#define CK_F_PR_FENCE_LOAD_ATOMIC
|
||||
#define CK_F_PR_FENCE_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_LOAD_STORE
|
||||
#define CK_F_PR_FENCE_MEMORY
|
||||
#define CK_F_PR_FENCE_STORE
|
||||
#define CK_F_PR_FENCE_STORE_ATOMIC
|
||||
#define CK_F_PR_FENCE_STORE_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_ATOMIC
|
||||
#define CK_F_PR_FENCE_STRICT_ATOMIC_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_ATOMIC_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD_ATOMIC
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_MEMORY
|
||||
#define CK_F_PR_FENCE_STRICT_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_STORE_ATOMIC
|
||||
#define CK_F_PR_FENCE_STRICT_STORE_LOAD
|
||||
#define CK_F_PR_INC_16
|
||||
#define CK_F_PR_INC_32
|
||||
#define CK_F_PR_INC_8
|
||||
#define CK_F_PR_INC_CHAR
|
||||
#define CK_F_PR_INC_INT
|
||||
#define CK_F_PR_INC_PTR
|
||||
#define CK_F_PR_INC_SHORT
|
||||
#define CK_F_PR_INC_UINT
|
||||
#define CK_F_PR_LOAD_16
|
||||
#define CK_F_PR_LOAD_32
|
||||
#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
|
||||
#define CK_F_PR_LOAD_64
|
||||
#define CK_F_PR_LOAD_DOUBLE
|
||||
#endif
|
||||
#define CK_F_PR_LOAD_8
|
||||
#define CK_F_PR_LOAD_CHAR
|
||||
#define CK_F_PR_LOAD_INT
|
||||
#define CK_F_PR_LOAD_PTR
|
||||
#define CK_F_PR_LOAD_SHORT
|
||||
#define CK_F_PR_LOAD_UINT
|
||||
#define CK_F_PR_NEG_16
|
||||
#define CK_F_PR_NEG_32
|
||||
#define CK_F_PR_NEG_8
|
||||
#define CK_F_PR_NEG_CHAR
|
||||
#define CK_F_PR_NEG_INT
|
||||
#define CK_F_PR_NEG_PTR
|
||||
#define CK_F_PR_NEG_SHORT
|
||||
#define CK_F_PR_NEG_UINT
|
||||
#define CK_F_PR_NOT_16
|
||||
#define CK_F_PR_NOT_32
|
||||
#define CK_F_PR_NOT_8
|
||||
#define CK_F_PR_NOT_CHAR
|
||||
#define CK_F_PR_NOT_INT
|
||||
#define CK_F_PR_NOT_PTR
|
||||
#define CK_F_PR_NOT_SHORT
|
||||
#define CK_F_PR_NOT_UINT
|
||||
#define CK_F_PR_OR_16
|
||||
#define CK_F_PR_OR_32
|
||||
#define CK_F_PR_OR_8
|
||||
#define CK_F_PR_OR_CHAR
|
||||
#define CK_F_PR_OR_INT
|
||||
#define CK_F_PR_OR_PTR
|
||||
#define CK_F_PR_OR_SHORT
|
||||
#define CK_F_PR_OR_UINT
|
||||
#define CK_F_PR_STALL
|
||||
#define CK_F_PR_STORE_16
|
||||
#define CK_F_PR_STORE_32
|
||||
#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
|
||||
#define CK_F_PR_STORE_64
|
||||
#define CK_F_PR_STORE_DOUBLE
|
||||
#endif
|
||||
#define CK_F_PR_STORE_8
|
||||
#define CK_F_PR_STORE_CHAR
|
||||
#define CK_F_PR_STORE_INT
|
||||
#define CK_F_PR_STORE_PTR
|
||||
#define CK_F_PR_STORE_SHORT
|
||||
#define CK_F_PR_STORE_UINT
|
||||
#define CK_F_PR_SUB_16
|
||||
#define CK_F_PR_SUB_32
|
||||
#define CK_F_PR_SUB_8
|
||||
#define CK_F_PR_SUB_CHAR
|
||||
#define CK_F_PR_SUB_INT
|
||||
#define CK_F_PR_SUB_PTR
|
||||
#define CK_F_PR_SUB_SHORT
|
||||
#define CK_F_PR_SUB_UINT
|
||||
#define CK_F_PR_XOR_16
|
||||
#define CK_F_PR_XOR_32
|
||||
#define CK_F_PR_XOR_8
|
||||
#define CK_F_PR_XOR_CHAR
|
||||
#define CK_F_PR_XOR_INT
|
||||
#define CK_F_PR_XOR_PTR
|
||||
#define CK_F_PR_XOR_SHORT
|
||||
#define CK_F_PR_XOR_UINT
|
||||
563
include/gcc/arm/ck_pr.h
Normal file
563
include/gcc/arm/ck_pr.h
Normal file
|
|
@ -0,0 +1,563 @@
|
|||
/*
|
||||
* Copyright 2009-2015 Samy Al Bahra.
|
||||
* Copyright 2013-2015 Olivier Houchard.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_ARM_H
|
||||
#define CK_PR_ARM_H
|
||||
|
||||
#ifndef CK_PR_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
#include <ck_cc.h>
|
||||
#include <ck_md.h>
|
||||
|
||||
/*
|
||||
* The following represent supported atomic operations.
|
||||
* These operations may be emulated.
|
||||
*/
|
||||
#include "ck_f_pr.h"
|
||||
|
||||
/*
|
||||
* Minimum interface requirement met.
|
||||
*/
|
||||
#define CK_F_PR
|
||||
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_stall(void)
|
||||
{
|
||||
|
||||
__asm__ __volatile__("" ::: "memory");
|
||||
return;
|
||||
}
|
||||
|
||||
#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
|
||||
#define CK_ISB __asm __volatile("isb" : : "r" (0) : "memory")
|
||||
#define CK_DMB __asm __volatile("dmb" : : "r" (0) : "memory")
|
||||
#define CK_DSB __asm __volatile("dsb" : : "r" (0) : "memory")
|
||||
/* FreeBSD's toolchain doesn't accept dmb st, so use the opcode instead */
|
||||
#ifdef __FreeBSD__
|
||||
#define CK_DMB_ST __asm __volatile(".word 0xf57ff05e" : : "r" (0) : "memory")
|
||||
#else
|
||||
#define CK_DMB_ST __asm __volatile("dmb st" : : "r" (0) : "memory")
|
||||
#endif /* __FreeBSD__ */
|
||||
#else
|
||||
/* armv6 doesn't have dsb/dmb/isb, and no way to wait only for stores */
|
||||
#define CK_ISB \
|
||||
__asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory")
|
||||
#define CK_DSB \
|
||||
__asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory")
|
||||
#define CK_DMB \
|
||||
__asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory")
|
||||
#define CK_DMB_ST CK_DMB
|
||||
#endif
|
||||
|
||||
#define CK_PR_FENCE(T, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_fence_strict_##T(void) \
|
||||
{ \
|
||||
I; \
|
||||
}
|
||||
|
||||
CK_PR_FENCE(atomic, CK_DMB_ST)
|
||||
CK_PR_FENCE(atomic_store, CK_DMB_ST)
|
||||
CK_PR_FENCE(atomic_load, CK_DMB_ST)
|
||||
CK_PR_FENCE(store_atomic, CK_DMB_ST)
|
||||
CK_PR_FENCE(load_atomic, CK_DMB)
|
||||
CK_PR_FENCE(store, CK_DMB_ST)
|
||||
CK_PR_FENCE(store_load, CK_DMB)
|
||||
CK_PR_FENCE(load, CK_DMB)
|
||||
CK_PR_FENCE(load_store, CK_DMB)
|
||||
CK_PR_FENCE(memory, CK_DMB)
|
||||
CK_PR_FENCE(acquire, CK_DMB)
|
||||
CK_PR_FENCE(release, CK_DMB)
|
||||
CK_PR_FENCE(acqrel, CK_DMB)
|
||||
CK_PR_FENCE(lock, CK_DMB)
|
||||
CK_PR_FENCE(unlock, CK_DMB)
|
||||
|
||||
#undef CK_PR_FENCE
|
||||
|
||||
#undef CK_ISB
|
||||
#undef CK_DSB
|
||||
#undef CK_DMB
|
||||
#undef CK_DMB_ST
|
||||
|
||||
#define CK_PR_LOAD(S, M, T, C, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_md_load_##S(const M *target) \
|
||||
{ \
|
||||
long r = 0; \
|
||||
__asm__ __volatile__(I " %0, [%1];" \
|
||||
: "=r" (r) \
|
||||
: "r" (target) \
|
||||
: "memory"); \
|
||||
return ((T)r); \
|
||||
}
|
||||
|
||||
CK_PR_LOAD(ptr, void, void *, uint32_t, "ldr")
|
||||
|
||||
#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I)
|
||||
|
||||
CK_PR_LOAD_S(32, uint32_t, "ldr")
|
||||
CK_PR_LOAD_S(16, uint16_t, "ldrh")
|
||||
CK_PR_LOAD_S(8, uint8_t, "ldrb")
|
||||
CK_PR_LOAD_S(uint, unsigned int, "ldr")
|
||||
CK_PR_LOAD_S(int, int, "ldr")
|
||||
CK_PR_LOAD_S(short, short, "ldrh")
|
||||
CK_PR_LOAD_S(char, char, "ldrb")
|
||||
|
||||
#undef CK_PR_LOAD_S
|
||||
#undef CK_PR_LOAD
|
||||
|
||||
#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
|
||||
|
||||
#define CK_PR_DOUBLE_LOAD(T, N) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_md_load_##N(const T *target) \
|
||||
{ \
|
||||
register T ret; \
|
||||
\
|
||||
__asm __volatile("ldrexd %0, [%1]" \
|
||||
: "=&r" (ret) \
|
||||
: "r" (target) \
|
||||
: "memory", "cc"); \
|
||||
return (ret); \
|
||||
}
|
||||
|
||||
CK_PR_DOUBLE_LOAD(uint64_t, 64)
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_DOUBLE_LOAD(double, double)
|
||||
#endif
|
||||
#undef CK_PR_DOUBLE_LOAD
|
||||
#endif
|
||||
|
||||
#define CK_PR_STORE(S, M, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I " %1, [%0]" \
|
||||
: \
|
||||
: "r" (target), \
|
||||
"r" (v) \
|
||||
: "memory"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_STORE(ptr, void, const void *, uint32_t, "str")
|
||||
|
||||
#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I)
|
||||
|
||||
CK_PR_STORE_S(32, uint32_t, "str")
|
||||
CK_PR_STORE_S(16, uint16_t, "strh")
|
||||
CK_PR_STORE_S(8, uint8_t, "strb")
|
||||
CK_PR_STORE_S(uint, unsigned int, "str")
|
||||
CK_PR_STORE_S(int, int, "str")
|
||||
CK_PR_STORE_S(short, short, "strh")
|
||||
CK_PR_STORE_S(char, char, "strb")
|
||||
|
||||
#undef CK_PR_STORE_S
|
||||
#undef CK_PR_STORE
|
||||
|
||||
#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
|
||||
|
||||
#define CK_PR_DOUBLE_STORE(T, N) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##N(const T *target, T value) \
|
||||
{ \
|
||||
T tmp; \
|
||||
uint32_t flag; \
|
||||
__asm __volatile("1: \n" \
|
||||
"ldrexd %0, [%2]\n" \
|
||||
"strexd %1, %3, [%2]\n" \
|
||||
"teq %1, #0\n" \
|
||||
"it ne \n" \
|
||||
"bne 1b\n" \
|
||||
: "=&r" (tmp), "=&r" (flag) \
|
||||
: "r" (target), "r" (value) \
|
||||
: "memory", "cc"); \
|
||||
}
|
||||
|
||||
CK_PR_DOUBLE_STORE(uint64_t, 64)
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_DOUBLE_STORE(double, double)
|
||||
#endif
|
||||
|
||||
#undef CK_PR_DOUBLE_STORE
|
||||
|
||||
#define CK_PR_DOUBLE_CAS_VALUE(T, N) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \
|
||||
{ \
|
||||
T previous; \
|
||||
int tmp; \
|
||||
\
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldrexd %0, [%4];" \
|
||||
"cmp %Q0, %Q2;" \
|
||||
"ittt eq;" \
|
||||
"cmpeq %R0, %R2;" \
|
||||
"strexdeq %1, %3, [%4];" \
|
||||
"cmpeq %1, #1;" \
|
||||
"beq 1b;" \
|
||||
:"=&r" (previous), "=&r" (tmp) \
|
||||
: "r" (compare), "r" (set) , \
|
||||
"r"(target) \
|
||||
: "memory", "cc"); \
|
||||
*value = previous; \
|
||||
return (*value == compare); \
|
||||
}
|
||||
|
||||
CK_PR_DOUBLE_CAS_VALUE(uint64_t, 64)
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_DOUBLE_CAS_VALUE(double, double)
|
||||
#endif
|
||||
|
||||
#undef CK_PR_DOUBLE_CAS_VALUE
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value)
|
||||
{
|
||||
uint32_t *_compare = CK_CPP_CAST(uint32_t *, compare);
|
||||
uint32_t *_set = CK_CPP_CAST(uint32_t *, set);
|
||||
uint64_t __compare = ((uint64_t)_compare[0]) | ((uint64_t)_compare[1] << 32);
|
||||
uint64_t __set = ((uint64_t)_set[0]) | ((uint64_t)_set[1] << 32);
|
||||
|
||||
return (ck_pr_cas_64_value(CK_CPP_CAST(uint64_t *, target),
|
||||
__compare,
|
||||
__set,
|
||||
CK_CPP_CAST(uint64_t *, value)));
|
||||
}
|
||||
|
||||
#define CK_PR_DOUBLE_CAS(T, N) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N(T *target, T compare, T set) \
|
||||
{ \
|
||||
int ret; \
|
||||
T tmp; \
|
||||
\
|
||||
__asm__ __volatile__("1:" \
|
||||
"mov %0, #0;" \
|
||||
"ldrexd %1, [%4];" \
|
||||
"cmp %Q1, %Q2;" \
|
||||
"itttt eq;" \
|
||||
"cmpeq %R1, %R2;" \
|
||||
"strexdeq %1, %3, [%4];" \
|
||||
"moveq %0, #1;" \
|
||||
"cmpeq %1, #1;" \
|
||||
"beq 1b;" \
|
||||
: "=&r" (ret), "=&r" (tmp) \
|
||||
: "r" (compare), "r" (set) , \
|
||||
"r"(target) \
|
||||
: "memory", "cc"); \
|
||||
\
|
||||
return (ret); \
|
||||
}
|
||||
|
||||
CK_PR_DOUBLE_CAS(uint64_t, 64)
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_DOUBLE_CAS(double, double)
|
||||
#endif
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_2(void *target, void *compare, void *set)
|
||||
{
|
||||
uint32_t *_compare = CK_CPP_CAST(uint32_t *, compare);
|
||||
uint32_t *_set = CK_CPP_CAST(uint32_t *, set);
|
||||
uint64_t __compare = ((uint64_t)_compare[0]) | ((uint64_t)_compare[1] << 32);
|
||||
uint64_t __set = ((uint64_t)_set[0]) | ((uint64_t)_set[1] << 32);
|
||||
return (ck_pr_cas_64(CK_CPP_CAST(uint64_t *, target),
|
||||
__compare,
|
||||
__set));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value)
|
||||
{
|
||||
void *previous, *tmp;
|
||||
__asm__ __volatile__("1:"
|
||||
"ldrex %0, [%2];"
|
||||
"cmp %0, %4;"
|
||||
"itt eq;"
|
||||
"strexeq %1, %3, [%2];"
|
||||
"cmpeq %1, #1;"
|
||||
"beq 1b;"
|
||||
: "=&r" (previous),
|
||||
"=&r" (tmp)
|
||||
: "r" (target),
|
||||
"r" (set),
|
||||
"r" (compare)
|
||||
: "memory", "cc");
|
||||
*(void **)value = previous;
|
||||
return (previous == compare);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr(void *target, void *compare, void *set)
|
||||
{
|
||||
void *previous, *tmp;
|
||||
__asm__ __volatile__("1:"
|
||||
"ldrex %0, [%2];"
|
||||
"cmp %0, %4;"
|
||||
"itt eq;"
|
||||
"strexeq %1, %3, [%2];"
|
||||
"cmpeq %1, #1;"
|
||||
"beq 1b;"
|
||||
: "=&r" (previous),
|
||||
"=&r" (tmp)
|
||||
: "r" (target),
|
||||
"r" (set),
|
||||
"r" (compare)
|
||||
: "memory", "cc");
|
||||
return (previous == compare);
|
||||
}
|
||||
|
||||
#define CK_PR_CAS(N, T, W) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \
|
||||
{ \
|
||||
T previous = 0, tmp = 0; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldrex" W " %0, [%2];" \
|
||||
"cmp %0, %4;" \
|
||||
"itt eq;" \
|
||||
"strex" W "eq %1, %3, [%2];" \
|
||||
"cmpeq %1, #1;" \
|
||||
"beq 1b;" \
|
||||
/* \
|
||||
* Using "+&" instead of "=&" to avoid bogus \
|
||||
* clang warnings. \
|
||||
*/ \
|
||||
: "+&r" (previous), \
|
||||
"+&r" (tmp) \
|
||||
: "r" (target), \
|
||||
"r" (set), \
|
||||
"r" (compare) \
|
||||
: "memory", "cc"); \
|
||||
*value = previous; \
|
||||
return (previous == compare); \
|
||||
} \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N(T *target, T compare, T set) \
|
||||
{ \
|
||||
T previous = 0, tmp = 0; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldrex" W " %0, [%2];" \
|
||||
"cmp %0, %4;" \
|
||||
"itt eq;" \
|
||||
"strex" W "eq %1, %3, [%2];" \
|
||||
"cmpeq %1, #1;" \
|
||||
"beq 1b;" \
|
||||
: "+&r" (previous), \
|
||||
"+&r" (tmp) \
|
||||
: "r" (target), \
|
||||
"r" (set), \
|
||||
"r" (compare) \
|
||||
: "memory", "cc"); \
|
||||
return (previous == compare); \
|
||||
}
|
||||
|
||||
CK_PR_CAS(32, uint32_t, "")
|
||||
CK_PR_CAS(uint, unsigned int, "")
|
||||
CK_PR_CAS(int, int, "")
|
||||
CK_PR_CAS(16, uint16_t, "h")
|
||||
CK_PR_CAS(8, uint8_t, "b")
|
||||
CK_PR_CAS(short, short, "h")
|
||||
CK_PR_CAS(char, char, "b")
|
||||
|
||||
|
||||
#undef CK_PR_CAS
|
||||
|
||||
#define CK_PR_FAS(N, M, T, W) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_fas_##N(M *target, T v) \
|
||||
{ \
|
||||
T previous = 0; \
|
||||
T tmp = 0; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldrex" W " %0, [%2];" \
|
||||
"strex" W " %1, %3, [%2];" \
|
||||
"cmp %1, #0;" \
|
||||
"bne 1b;" \
|
||||
: "+&r" (previous), \
|
||||
"+&r" (tmp) \
|
||||
: "r" (target), \
|
||||
"r" (v) \
|
||||
: "memory", "cc"); \
|
||||
return (previous); \
|
||||
}
|
||||
|
||||
CK_PR_FAS(32, uint32_t, uint32_t, "")
|
||||
CK_PR_FAS(ptr, void, void *, "")
|
||||
CK_PR_FAS(int, int, int, "")
|
||||
CK_PR_FAS(uint, unsigned int, unsigned int, "")
|
||||
CK_PR_FAS(16, uint16_t, uint16_t, "h")
|
||||
CK_PR_FAS(8, uint8_t, uint8_t, "b")
|
||||
CK_PR_FAS(short, short, short, "h")
|
||||
CK_PR_FAS(char, char, char, "b")
|
||||
|
||||
|
||||
#undef CK_PR_FAS
|
||||
|
||||
#define CK_PR_UNARY(O, N, M, T, I, W) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##O##_##N(M *target) \
|
||||
{ \
|
||||
T previous = 0; \
|
||||
T tmp = 0; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldrex" W " %0, [%2];" \
|
||||
I ";" \
|
||||
"strex" W " %1, %0, [%2];" \
|
||||
"cmp %1, #0;" \
|
||||
"bne 1b;" \
|
||||
: "+&r" (previous), \
|
||||
"+&r" (tmp) \
|
||||
: "r" (target) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_UNARY(inc, ptr, void, void *, "add %0, %0, #1", "")
|
||||
CK_PR_UNARY(dec, ptr, void, void *, "sub %0, %0, #1", "")
|
||||
CK_PR_UNARY(not, ptr, void, void *, "mvn %0, %0", "")
|
||||
CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "")
|
||||
|
||||
#define CK_PR_UNARY_S(S, T, W) \
|
||||
CK_PR_UNARY(inc, S, T, T, "add %0, %0, #1", W) \
|
||||
CK_PR_UNARY(dec, S, T, T, "sub %0, %0, #1", W) \
|
||||
CK_PR_UNARY(not, S, T, T, "mvn %0, %0", W) \
|
||||
CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) \
|
||||
|
||||
CK_PR_UNARY_S(32, uint32_t, "")
|
||||
CK_PR_UNARY_S(uint, unsigned int, "")
|
||||
CK_PR_UNARY_S(int, int, "")
|
||||
CK_PR_UNARY_S(16, uint16_t, "h")
|
||||
CK_PR_UNARY_S(8, uint8_t, "b")
|
||||
CK_PR_UNARY_S(short, short, "h")
|
||||
CK_PR_UNARY_S(char, char, "b")
|
||||
|
||||
#undef CK_PR_UNARY_S
|
||||
#undef CK_PR_UNARY
|
||||
|
||||
#define CK_PR_BINARY(O, N, M, T, I, W) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##O##_##N(M *target, T delta) \
|
||||
{ \
|
||||
T previous = 0; \
|
||||
T tmp = 0; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldrex" W " %0, [%2];" \
|
||||
I " %0, %0, %3;" \
|
||||
"strex" W " %1, %0, [%2];" \
|
||||
"cmp %1, #0;" \
|
||||
"bne 1b;" \
|
||||
: "+&r" (previous), \
|
||||
"+&r" (tmp) \
|
||||
: "r" (target), \
|
||||
"r" (delta) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "")
|
||||
CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "")
|
||||
CK_PR_BINARY(or, ptr, void, uintptr_t, "orr", "")
|
||||
CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "")
|
||||
CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "")
|
||||
|
||||
#define CK_PR_BINARY_S(S, T, W) \
|
||||
CK_PR_BINARY(and, S, T, T, "and", W) \
|
||||
CK_PR_BINARY(add, S, T, T, "add", W) \
|
||||
CK_PR_BINARY(or, S, T, T, "orr", W) \
|
||||
CK_PR_BINARY(sub, S, T, T, "sub", W) \
|
||||
CK_PR_BINARY(xor, S, T, T, "eor", W)
|
||||
|
||||
CK_PR_BINARY_S(32, uint32_t, "")
|
||||
CK_PR_BINARY_S(uint, unsigned int, "")
|
||||
CK_PR_BINARY_S(int, int, "")
|
||||
CK_PR_BINARY_S(16, uint16_t, "h")
|
||||
CK_PR_BINARY_S(8, uint8_t, "b")
|
||||
CK_PR_BINARY_S(short, short, "h")
|
||||
CK_PR_BINARY_S(char, char, "b")
|
||||
|
||||
#undef CK_PR_BINARY_S
|
||||
#undef CK_PR_BINARY
|
||||
|
||||
CK_CC_INLINE static void *
|
||||
ck_pr_faa_ptr(void *target, uintptr_t delta)
|
||||
{
|
||||
uintptr_t previous, r, tmp;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"ldrex %0, [%3];"
|
||||
"add %1, %4, %0;"
|
||||
"strex %2, %1, [%3];"
|
||||
"cmp %2, #0;"
|
||||
"bne 1b;"
|
||||
: "=&r" (previous),
|
||||
"=&r" (r),
|
||||
"=&r" (tmp)
|
||||
: "r" (target),
|
||||
"r" (delta)
|
||||
: "memory", "cc");
|
||||
|
||||
return (void *)(previous);
|
||||
}
|
||||
|
||||
#define CK_PR_FAA(S, T, W) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_faa_##S(T *target, T delta) \
|
||||
{ \
|
||||
T previous = 0, r = 0, tmp = 0; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"ldrex" W " %0, [%3];" \
|
||||
"add %1, %4, %0;" \
|
||||
"strex" W " %2, %1, [%3];" \
|
||||
"cmp %2, #0;" \
|
||||
"bne 1b;" \
|
||||
: "+&r" (previous), \
|
||||
"+&r" (r), \
|
||||
"+&r" (tmp) \
|
||||
: "r" (target), \
|
||||
"r" (delta) \
|
||||
: "memory", "cc"); \
|
||||
return (previous); \
|
||||
}
|
||||
|
||||
CK_PR_FAA(32, uint32_t, "")
|
||||
CK_PR_FAA(uint, unsigned int, "")
|
||||
CK_PR_FAA(int, int, "")
|
||||
CK_PR_FAA(16, uint16_t, "h")
|
||||
CK_PR_FAA(8, uint8_t, "b")
|
||||
CK_PR_FAA(short, short, "h")
|
||||
CK_PR_FAA(char, char, "b")
|
||||
|
||||
#undef CK_PR_FAA
|
||||
|
||||
#endif /* CK_PR_ARM_H */
|
||||
|
||||
142
include/gcc/ck_cc.h
Normal file
142
include/gcc/ck_cc.h
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
/*
|
||||
* Copyright 2009-2015 Samy Al Bahra.
|
||||
* Copyright 2014 Paul Khuong.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_GCC_CC_H
|
||||
#define CK_GCC_CC_H
|
||||
|
||||
#include <ck_md.h>
|
||||
|
||||
#ifdef __SUNPRO_C
|
||||
#define CK_CC_UNUSED
|
||||
#define CK_CC_USED
|
||||
#define CK_CC_IMM
|
||||
#define CK_CC_IMM_U32
|
||||
#else
|
||||
#define CK_CC_UNUSED __attribute__((unused))
|
||||
#define CK_CC_USED __attribute__((used))
|
||||
#define CK_CC_IMM "i"
|
||||
#if defined(__x86_64__) || defined(__x86__)
|
||||
#define CK_CC_IMM_U32 "Z"
|
||||
#define CK_CC_IMM_S32 "e"
|
||||
#else
|
||||
#define CK_CC_IMM_U32 CK_CC_IMM
|
||||
#define CK_CC_IMM_S32 CK_CC_IMM
|
||||
#endif /* __x86_64__ || __x86__ */
|
||||
#endif
|
||||
|
||||
#ifdef __OPTIMIZE__
|
||||
#define CK_CC_INLINE CK_CC_UNUSED inline
|
||||
#else
|
||||
#define CK_CC_INLINE CK_CC_UNUSED
|
||||
#endif
|
||||
|
||||
#define CK_CC_FORCE_INLINE CK_CC_UNUSED __attribute__((always_inline)) inline
|
||||
#define CK_CC_RESTRICT __restrict__
|
||||
|
||||
/*
|
||||
* Packed attribute.
|
||||
*/
|
||||
#define CK_CC_PACKED __attribute__((packed))
|
||||
|
||||
/*
|
||||
* Weak reference.
|
||||
*/
|
||||
#define CK_CC_WEAKREF __attribute__((weakref))
|
||||
|
||||
/*
|
||||
* Alignment attribute.
|
||||
*/
|
||||
#define CK_CC_ALIGN(B) __attribute__((aligned(B)))
|
||||
|
||||
/*
|
||||
* Cache align.
|
||||
*/
|
||||
#define CK_CC_CACHELINE CK_CC_ALIGN(CK_MD_CACHELINE)
|
||||
|
||||
/*
|
||||
* These are functions which should be avoided.
|
||||
*/
|
||||
#ifdef __freestanding__
|
||||
#pragma GCC poison malloc free
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Branch execution hints.
|
||||
*/
|
||||
#define CK_CC_LIKELY(x) (__builtin_expect(!!(x), 1))
|
||||
#define CK_CC_UNLIKELY(x) (__builtin_expect(!!(x), 0))
|
||||
|
||||
/*
|
||||
* Some compilers are overly strict regarding aliasing semantics.
|
||||
* Unfortunately, in many cases it makes more sense to pay aliasing
|
||||
* cost rather than overly expensive register spillage.
|
||||
*/
|
||||
#define CK_CC_ALIASED __attribute__((__may_alias__))
|
||||
|
||||
/*
|
||||
* Compile-time typeof
|
||||
*/
|
||||
#define CK_CC_TYPEOF(X, DEFAULT) __typeof__(X)
|
||||
|
||||
/*
|
||||
* Portability wrappers for bitwise ops.
|
||||
*/
|
||||
|
||||
#define CK_F_CC_FFS
|
||||
#define CK_F_CC_CLZ
|
||||
#define CK_F_CC_CTZ
|
||||
#define CK_F_CC_POPCOUNT
|
||||
|
||||
CK_CC_INLINE static int
|
||||
ck_cc_ffs(unsigned int x)
|
||||
{
|
||||
|
||||
return __builtin_ffs(x);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static int
|
||||
ck_cc_clz(unsigned int x)
|
||||
{
|
||||
|
||||
return __builtin_clz(x);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static int
|
||||
ck_cc_ctz(unsigned int x)
|
||||
{
|
||||
|
||||
return __builtin_ctz(x);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static int
|
||||
ck_cc_popcount(unsigned int x)
|
||||
{
|
||||
|
||||
return __builtin_popcount(x);
|
||||
}
|
||||
|
||||
#endif /* CK_GCC_CC_H */
|
||||
105
include/gcc/ck_f_pr.h
Normal file
105
include/gcc/ck_f_pr.h
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
/* DO NOT EDIT. This is auto-generated from feature.sh */
|
||||
#define CK_F_PR_ADD_16
|
||||
#define CK_F_PR_ADD_32
|
||||
#define CK_F_PR_ADD_64
|
||||
#define CK_F_PR_ADD_8
|
||||
#define CK_F_PR_ADD_CHAR
|
||||
#define CK_F_PR_ADD_INT
|
||||
#define CK_F_PR_ADD_PTR
|
||||
#define CK_F_PR_ADD_UINT
|
||||
#define CK_F_PR_AND_16
|
||||
#define CK_F_PR_AND_32
|
||||
#define CK_F_PR_AND_64
|
||||
#define CK_F_PR_AND_8
|
||||
#define CK_F_PR_AND_CHAR
|
||||
#define CK_F_PR_AND_INT
|
||||
#define CK_F_PR_AND_PTR
|
||||
#define CK_F_PR_AND_UINT
|
||||
#define CK_F_PR_CAS_16
|
||||
#define CK_F_PR_CAS_16_VALUE
|
||||
#define CK_F_PR_CAS_32
|
||||
#define CK_F_PR_CAS_32_VALUE
|
||||
#define CK_F_PR_CAS_64
|
||||
#define CK_F_PR_CAS_64_VALUE
|
||||
#define CK_F_PR_CAS_8
|
||||
#define CK_F_PR_CAS_8_VALUE
|
||||
#define CK_F_PR_CAS_CHAR
|
||||
#define CK_F_PR_CAS_CHAR_VALUE
|
||||
#define CK_F_PR_CAS_INT
|
||||
#define CK_F_PR_CAS_INT_VALUE
|
||||
#define CK_F_PR_CAS_PTR
|
||||
#define CK_F_PR_CAS_PTR_VALUE
|
||||
#define CK_F_PR_CAS_UINT
|
||||
#define CK_F_PR_CAS_UINT_VALUE
|
||||
#define CK_F_PR_DEC_16
|
||||
#define CK_F_PR_DEC_32
|
||||
#define CK_F_PR_DEC_64
|
||||
#define CK_F_PR_DEC_8
|
||||
#define CK_F_PR_DEC_CHAR
|
||||
#define CK_F_PR_DEC_INT
|
||||
#define CK_F_PR_DEC_PTR
|
||||
#define CK_F_PR_DEC_UINT
|
||||
#define CK_F_PR_FAA_16
|
||||
#define CK_F_PR_FAA_32
|
||||
#define CK_F_PR_FAA_64
|
||||
#define CK_F_PR_FAA_8
|
||||
#define CK_F_PR_FAA_CHAR
|
||||
#define CK_F_PR_FAA_INT
|
||||
#define CK_F_PR_FAA_PTR
|
||||
#define CK_F_PR_FAA_UINT
|
||||
#define CK_F_PR_FENCE_LOAD
|
||||
#define CK_F_PR_FENCE_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_MEMORY
|
||||
#define CK_F_PR_FENCE_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_MEMORY
|
||||
#define CK_F_PR_FENCE_STRICT_STORE
|
||||
#define CK_F_PR_INC_16
|
||||
#define CK_F_PR_INC_32
|
||||
#define CK_F_PR_INC_64
|
||||
#define CK_F_PR_INC_8
|
||||
#define CK_F_PR_INC_CHAR
|
||||
#define CK_F_PR_INC_INT
|
||||
#define CK_F_PR_INC_PTR
|
||||
#define CK_F_PR_INC_UINT
|
||||
#define CK_F_PR_LOAD_16
|
||||
#define CK_F_PR_LOAD_32
|
||||
#define CK_F_PR_LOAD_64
|
||||
#define CK_F_PR_LOAD_8
|
||||
#define CK_F_PR_LOAD_CHAR
|
||||
#define CK_F_PR_LOAD_INT
|
||||
#define CK_F_PR_LOAD_PTR
|
||||
#define CK_F_PR_LOAD_UINT
|
||||
#define CK_F_PR_OR_16
|
||||
#define CK_F_PR_OR_32
|
||||
#define CK_F_PR_OR_64
|
||||
#define CK_F_PR_OR_8
|
||||
#define CK_F_PR_OR_CHAR
|
||||
#define CK_F_PR_OR_INT
|
||||
#define CK_F_PR_OR_PTR
|
||||
#define CK_F_PR_OR_UINT
|
||||
#define CK_F_PR_STALL
|
||||
#define CK_F_PR_STORE_16
|
||||
#define CK_F_PR_STORE_32
|
||||
#define CK_F_PR_STORE_64
|
||||
#define CK_F_PR_STORE_8
|
||||
#define CK_F_PR_STORE_CHAR
|
||||
#define CK_F_PR_STORE_INT
|
||||
#define CK_F_PR_STORE_PTR
|
||||
#define CK_F_PR_STORE_UINT
|
||||
#define CK_F_PR_SUB_16
|
||||
#define CK_F_PR_SUB_32
|
||||
#define CK_F_PR_SUB_64
|
||||
#define CK_F_PR_SUB_8
|
||||
#define CK_F_PR_SUB_CHAR
|
||||
#define CK_F_PR_SUB_INT
|
||||
#define CK_F_PR_SUB_PTR
|
||||
#define CK_F_PR_SUB_UINT
|
||||
#define CK_F_PR_XOR_16
|
||||
#define CK_F_PR_XOR_32
|
||||
#define CK_F_PR_XOR_64
|
||||
#define CK_F_PR_XOR_8
|
||||
#define CK_F_PR_XOR_CHAR
|
||||
#define CK_F_PR_XOR_INT
|
||||
#define CK_F_PR_XOR_PTR
|
||||
#define CK_F_PR_XOR_UINT
|
||||
297
include/gcc/ck_pr.h
Normal file
297
include/gcc/ck_pr.h
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
/*
|
||||
* Copyright 2010 Samy Al Bahra.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_GCC_H
|
||||
#define CK_PR_GCC_H
|
||||
|
||||
#ifndef CK_PR_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
#include <ck_cc.h>
|
||||
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_barrier(void)
|
||||
{
|
||||
|
||||
__asm__ __volatile__("" ::: "memory");
|
||||
return;
|
||||
}
|
||||
|
||||
#ifndef CK_F_PR
|
||||
#define CK_F_PR
|
||||
|
||||
#include <ck_stdbool.h>
|
||||
#include <ck_stdint.h>
|
||||
|
||||
/*
|
||||
* The following represent supported atomic operations.
|
||||
* These operations may be emulated.
|
||||
*/
|
||||
#include "ck_f_pr.h"
|
||||
|
||||
#define CK_PR_ACCESS(x) (*(volatile __typeof__(x) *)&(x))
|
||||
|
||||
#define CK_PR_LOAD(S, M, T) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_md_load_##S(const M *target) \
|
||||
{ \
|
||||
T r; \
|
||||
ck_pr_barrier(); \
|
||||
r = CK_PR_ACCESS(*(const T *)target); \
|
||||
ck_pr_barrier(); \
|
||||
return (r); \
|
||||
} \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##S(M *target, T v) \
|
||||
{ \
|
||||
ck_pr_barrier(); \
|
||||
CK_PR_ACCESS(*(T *)target) = v; \
|
||||
ck_pr_barrier(); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_CC_INLINE static void *
|
||||
ck_pr_md_load_ptr(const void *target)
|
||||
{
|
||||
void *r;
|
||||
|
||||
ck_pr_barrier();
|
||||
r = CK_CC_DECONST_PTR(CK_PR_ACCESS(target));
|
||||
ck_pr_barrier();
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_md_store_ptr(void *target, const void *v)
|
||||
{
|
||||
|
||||
ck_pr_barrier();
|
||||
CK_PR_ACCESS(target) = CK_CC_DECONST_PTR(v);
|
||||
ck_pr_barrier();
|
||||
return;
|
||||
}
|
||||
|
||||
#define CK_PR_LOAD_S(S, T) CK_PR_LOAD(S, T, T)
|
||||
|
||||
CK_PR_LOAD_S(char, char)
|
||||
CK_PR_LOAD_S(uint, unsigned int)
|
||||
CK_PR_LOAD_S(int, int)
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_LOAD_S(double, double)
|
||||
#endif
|
||||
CK_PR_LOAD_S(64, uint64_t)
|
||||
CK_PR_LOAD_S(32, uint32_t)
|
||||
CK_PR_LOAD_S(16, uint16_t)
|
||||
CK_PR_LOAD_S(8, uint8_t)
|
||||
|
||||
#undef CK_PR_LOAD_S
|
||||
#undef CK_PR_LOAD
|
||||
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_stall(void)
|
||||
{
|
||||
|
||||
ck_pr_barrier();
|
||||
}
|
||||
|
||||
/*
|
||||
* Load and store fences are equivalent to full fences in the GCC port.
|
||||
*/
|
||||
#define CK_PR_FENCE(T) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_fence_strict_##T(void) \
|
||||
{ \
|
||||
__sync_synchronize(); \
|
||||
}
|
||||
|
||||
CK_PR_FENCE(atomic)
|
||||
CK_PR_FENCE(atomic_atomic)
|
||||
CK_PR_FENCE(atomic_load)
|
||||
CK_PR_FENCE(atomic_store)
|
||||
CK_PR_FENCE(store_atomic)
|
||||
CK_PR_FENCE(load_atomic)
|
||||
CK_PR_FENCE(load)
|
||||
CK_PR_FENCE(load_load)
|
||||
CK_PR_FENCE(load_store)
|
||||
CK_PR_FENCE(store)
|
||||
CK_PR_FENCE(store_store)
|
||||
CK_PR_FENCE(store_load)
|
||||
CK_PR_FENCE(memory)
|
||||
CK_PR_FENCE(acquire)
|
||||
CK_PR_FENCE(release)
|
||||
CK_PR_FENCE(acqrel)
|
||||
CK_PR_FENCE(lock)
|
||||
CK_PR_FENCE(unlock)
|
||||
|
||||
#undef CK_PR_FENCE
|
||||
|
||||
/*
|
||||
* Atomic compare and swap.
|
||||
*/
|
||||
#define CK_PR_CAS(S, M, T) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##S(M *target, T compare, T set) \
|
||||
{ \
|
||||
bool z; \
|
||||
z = __sync_bool_compare_and_swap((T *)target, compare, set); \
|
||||
return z; \
|
||||
}
|
||||
|
||||
CK_PR_CAS(ptr, void, void *)
|
||||
|
||||
#define CK_PR_CAS_S(S, T) CK_PR_CAS(S, T, T)
|
||||
|
||||
CK_PR_CAS_S(char, char)
|
||||
CK_PR_CAS_S(int, int)
|
||||
CK_PR_CAS_S(uint, unsigned int)
|
||||
CK_PR_CAS_S(64, uint64_t)
|
||||
CK_PR_CAS_S(32, uint32_t)
|
||||
CK_PR_CAS_S(16, uint16_t)
|
||||
CK_PR_CAS_S(8, uint8_t)
|
||||
|
||||
#undef CK_PR_CAS_S
|
||||
#undef CK_PR_CAS
|
||||
|
||||
/*
|
||||
* Compare and swap, set *v to old value of target.
|
||||
*/
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *v)
|
||||
{
|
||||
set = __sync_val_compare_and_swap((void **)target, compare, set);
|
||||
*(void **)v = set;
|
||||
return (set == compare);
|
||||
}
|
||||
|
||||
#define CK_PR_CAS_O(S, T) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##S##_value(T *target, T compare, T set, T *v) \
|
||||
{ \
|
||||
set = __sync_val_compare_and_swap(target, compare, set);\
|
||||
*v = set; \
|
||||
return (set == compare); \
|
||||
}
|
||||
|
||||
CK_PR_CAS_O(char, char)
|
||||
CK_PR_CAS_O(int, int)
|
||||
CK_PR_CAS_O(uint, unsigned int)
|
||||
CK_PR_CAS_O(64, uint64_t)
|
||||
CK_PR_CAS_O(32, uint32_t)
|
||||
CK_PR_CAS_O(16, uint16_t)
|
||||
CK_PR_CAS_O(8, uint8_t)
|
||||
|
||||
#undef CK_PR_CAS_O
|
||||
|
||||
/*
|
||||
* Atomic fetch-and-add operations.
|
||||
*/
|
||||
#define CK_PR_FAA(S, M, T) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_faa_##S(M *target, T d) \
|
||||
{ \
|
||||
d = __sync_fetch_and_add((T *)target, d); \
|
||||
return (d); \
|
||||
}
|
||||
|
||||
CK_PR_FAA(ptr, void, void *)
|
||||
|
||||
#define CK_PR_FAA_S(S, T) CK_PR_FAA(S, T, T)
|
||||
|
||||
CK_PR_FAA_S(char, char)
|
||||
CK_PR_FAA_S(uint, unsigned int)
|
||||
CK_PR_FAA_S(int, int)
|
||||
CK_PR_FAA_S(64, uint64_t)
|
||||
CK_PR_FAA_S(32, uint32_t)
|
||||
CK_PR_FAA_S(16, uint16_t)
|
||||
CK_PR_FAA_S(8, uint8_t)
|
||||
|
||||
#undef CK_PR_FAA_S
|
||||
#undef CK_PR_FAA
|
||||
|
||||
/*
|
||||
* Atomic store-only binary operations.
|
||||
*/
|
||||
#define CK_PR_BINARY(K, S, M, T) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##K##_##S(M *target, T d) \
|
||||
{ \
|
||||
d = __sync_fetch_and_##K((T *)target, d); \
|
||||
return; \
|
||||
}
|
||||
|
||||
#define CK_PR_BINARY_S(K, S, T) CK_PR_BINARY(K, S, T, T)
|
||||
|
||||
#define CK_PR_GENERATE(K) \
|
||||
CK_PR_BINARY(K, ptr, void, void *) \
|
||||
CK_PR_BINARY_S(K, char, char) \
|
||||
CK_PR_BINARY_S(K, int, int) \
|
||||
CK_PR_BINARY_S(K, uint, unsigned int) \
|
||||
CK_PR_BINARY_S(K, 64, uint64_t) \
|
||||
CK_PR_BINARY_S(K, 32, uint32_t) \
|
||||
CK_PR_BINARY_S(K, 16, uint16_t) \
|
||||
CK_PR_BINARY_S(K, 8, uint8_t)
|
||||
|
||||
CK_PR_GENERATE(add)
|
||||
CK_PR_GENERATE(sub)
|
||||
CK_PR_GENERATE(and)
|
||||
CK_PR_GENERATE(or)
|
||||
CK_PR_GENERATE(xor)
|
||||
|
||||
#undef CK_PR_GENERATE
|
||||
#undef CK_PR_BINARY_S
|
||||
#undef CK_PR_BINARY
|
||||
|
||||
#define CK_PR_UNARY(S, M, T) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_inc_##S(M *target) \
|
||||
{ \
|
||||
ck_pr_add_##S(target, (T)1); \
|
||||
return; \
|
||||
} \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_dec_##S(M *target) \
|
||||
{ \
|
||||
ck_pr_sub_##S(target, (T)1); \
|
||||
return; \
|
||||
}
|
||||
|
||||
#define CK_PR_UNARY_S(S, M) CK_PR_UNARY(S, M, M)
|
||||
|
||||
CK_PR_UNARY(ptr, void, void *)
|
||||
CK_PR_UNARY_S(char, char)
|
||||
CK_PR_UNARY_S(int, int)
|
||||
CK_PR_UNARY_S(uint, unsigned int)
|
||||
CK_PR_UNARY_S(64, uint64_t)
|
||||
CK_PR_UNARY_S(32, uint32_t)
|
||||
CK_PR_UNARY_S(16, uint16_t)
|
||||
CK_PR_UNARY_S(8, uint8_t)
|
||||
|
||||
#undef CK_PR_UNARY_S
|
||||
#undef CK_PR_UNARY
|
||||
#endif /* !CK_F_PR */
|
||||
#endif /* CK_PR_GCC_H */
|
||||
79
include/gcc/ppc/ck_f_pr.h
Normal file
79
include/gcc/ppc/ck_f_pr.h
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
/* DO NOT EDIT. This is auto-generated from feature.sh */
|
||||
#define CK_F_PR_ADD_32
|
||||
#define CK_F_PR_ADD_INT
|
||||
#define CK_F_PR_ADD_PTR
|
||||
#define CK_F_PR_ADD_UINT
|
||||
#define CK_F_PR_AND_32
|
||||
#define CK_F_PR_AND_INT
|
||||
#define CK_F_PR_AND_PTR
|
||||
#define CK_F_PR_AND_UINT
|
||||
#define CK_F_PR_CAS_32
|
||||
#define CK_F_PR_CAS_32_VALUE
|
||||
#define CK_F_PR_CAS_INT
|
||||
#define CK_F_PR_CAS_INT_VALUE
|
||||
#define CK_F_PR_CAS_PTR
|
||||
#define CK_F_PR_CAS_PTR_VALUE
|
||||
#define CK_F_PR_CAS_UINT
|
||||
#define CK_F_PR_CAS_UINT_VALUE
|
||||
#define CK_F_PR_DEC_32
|
||||
#define CK_F_PR_DEC_INT
|
||||
#define CK_F_PR_DEC_PTR
|
||||
#define CK_F_PR_DEC_UINT
|
||||
#define CK_F_PR_FAA_32
|
||||
#define CK_F_PR_FAA_INT
|
||||
#define CK_F_PR_FAA_PTR
|
||||
#define CK_F_PR_FAA_UINT
|
||||
#define CK_F_PR_FAS_32
|
||||
#define CK_F_PR_FAS_INT
|
||||
#define CK_F_PR_FAS_PTR
|
||||
#define CK_F_PR_FAS_UINT
|
||||
#define CK_F_PR_FENCE_LOAD
|
||||
#define CK_F_PR_FENCE_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_MEMORY
|
||||
#define CK_F_PR_FENCE_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_STRICT_MEMORY
|
||||
#define CK_F_PR_FENCE_STRICT_STORE
|
||||
#define CK_F_PR_INC_32
|
||||
#define CK_F_PR_INC_INT
|
||||
#define CK_F_PR_INC_PTR
|
||||
#define CK_F_PR_INC_UINT
|
||||
#define CK_F_PR_LOAD_16
|
||||
#define CK_F_PR_LOAD_32
|
||||
#define CK_F_PR_LOAD_8
|
||||
#define CK_F_PR_LOAD_CHAR
|
||||
#define CK_F_PR_LOAD_INT
|
||||
#define CK_F_PR_LOAD_PTR
|
||||
#define CK_F_PR_LOAD_SHORT
|
||||
#define CK_F_PR_LOAD_UINT
|
||||
#define CK_F_PR_NEG_32
|
||||
#define CK_F_PR_NEG_INT
|
||||
#define CK_F_PR_NEG_PTR
|
||||
#define CK_F_PR_NEG_UINT
|
||||
#define CK_F_PR_NOT_32
|
||||
#define CK_F_PR_NOT_INT
|
||||
#define CK_F_PR_NOT_PTR
|
||||
#define CK_F_PR_NOT_UINT
|
||||
#define CK_F_PR_OR_32
|
||||
#define CK_F_PR_OR_INT
|
||||
#define CK_F_PR_OR_PTR
|
||||
#define CK_F_PR_OR_UINT
|
||||
#define CK_F_PR_STALL
|
||||
#define CK_F_PR_STORE_16
|
||||
#define CK_F_PR_STORE_32
|
||||
#define CK_F_PR_STORE_8
|
||||
#define CK_F_PR_STORE_CHAR
|
||||
#define CK_F_PR_STORE_INT
|
||||
#define CK_F_PR_STORE_PTR
|
||||
#define CK_F_PR_STORE_SHORT
|
||||
#define CK_F_PR_STORE_UINT
|
||||
#define CK_F_PR_SUB_32
|
||||
#define CK_F_PR_SUB_INT
|
||||
#define CK_F_PR_SUB_PTR
|
||||
#define CK_F_PR_SUB_UINT
|
||||
#define CK_F_PR_XOR_32
|
||||
#define CK_F_PR_XOR_INT
|
||||
#define CK_F_PR_XOR_PTR
|
||||
#define CK_F_PR_XOR_UINT
|
||||
|
||||
327
include/gcc/ppc/ck_pr.h
Normal file
327
include/gcc/ppc/ck_pr.h
Normal file
|
|
@ -0,0 +1,327 @@
|
|||
/*
|
||||
* Copyright 2009-2015 Samy Al Bahra.
|
||||
* Copyright 2012 João Fernandes.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_PPC_H
|
||||
#define CK_PR_PPC_H
|
||||
|
||||
#ifndef CK_PR_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
#include <ck_cc.h>
|
||||
#include <ck_md.h>
|
||||
|
||||
/*
|
||||
* The following represent supported atomic operations.
|
||||
* These operations may be emulated.
|
||||
*/
|
||||
#include "ck_f_pr.h"
|
||||
|
||||
/*
|
||||
* Minimum interface requirement met.
|
||||
*/
|
||||
#define CK_F_PR
|
||||
|
||||
/*
|
||||
* This bounces the hardware thread from low to medium
|
||||
* priority. I am unsure of the benefits of this approach
|
||||
* but it is used by the Linux kernel.
|
||||
*/
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_stall(void)
|
||||
{
|
||||
|
||||
__asm__ __volatile__("or 1, 1, 1;"
|
||||
"or 2, 2, 2;" ::: "memory");
|
||||
return;
|
||||
}
|
||||
|
||||
#define CK_PR_FENCE(T, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_fence_strict_##T(void) \
|
||||
{ \
|
||||
__asm__ __volatile__(I ::: "memory"); \
|
||||
}
|
||||
|
||||
CK_PR_FENCE(atomic, "lwsync")
|
||||
CK_PR_FENCE(atomic_store, "lwsync")
|
||||
CK_PR_FENCE(atomic_load, "sync")
|
||||
CK_PR_FENCE(store_atomic, "lwsync")
|
||||
CK_PR_FENCE(load_atomic, "lwsync")
|
||||
CK_PR_FENCE(store, "lwsync")
|
||||
CK_PR_FENCE(store_load, "sync")
|
||||
CK_PR_FENCE(load, "lwsync")
|
||||
CK_PR_FENCE(load_store, "lwsync")
|
||||
CK_PR_FENCE(memory, "sync")
|
||||
CK_PR_FENCE(acquire, "lwsync")
|
||||
CK_PR_FENCE(release, "lwsync")
|
||||
CK_PR_FENCE(acqrel, "lwsync")
|
||||
CK_PR_FENCE(lock, "lwsync")
|
||||
CK_PR_FENCE(unlock, "lwsync")
|
||||
|
||||
#undef CK_PR_FENCE
|
||||
|
||||
#define CK_PR_LOAD(S, M, T, C, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_md_load_##S(const M *target) \
|
||||
{ \
|
||||
T r; \
|
||||
__asm__ __volatile__(I "%U1%X1 %0, %1" \
|
||||
: "=r" (r) \
|
||||
: "m" (*(const C *)target) \
|
||||
: "memory"); \
|
||||
return (r); \
|
||||
}
|
||||
|
||||
CK_PR_LOAD(ptr, void, void *, uint32_t, "lwz")
|
||||
|
||||
#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I)
|
||||
|
||||
CK_PR_LOAD_S(32, uint32_t, "lwz")
|
||||
CK_PR_LOAD_S(16, uint16_t, "lhz")
|
||||
CK_PR_LOAD_S(8, uint8_t, "lbz")
|
||||
CK_PR_LOAD_S(uint, unsigned int, "lwz")
|
||||
CK_PR_LOAD_S(int, int, "lwz")
|
||||
CK_PR_LOAD_S(short, short, "lhz")
|
||||
CK_PR_LOAD_S(char, char, "lbz")
|
||||
|
||||
#undef CK_PR_LOAD_S
|
||||
#undef CK_PR_LOAD
|
||||
|
||||
#define CK_PR_STORE(S, M, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I "%U0%X0 %1, %0" \
|
||||
: "=m" (*(C *)target) \
|
||||
: "r" (v) \
|
||||
: "memory"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_STORE(ptr, void, const void *, uint32_t, "stw")
|
||||
|
||||
#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I)
|
||||
|
||||
CK_PR_STORE_S(32, uint32_t, "stw")
|
||||
CK_PR_STORE_S(16, uint16_t, "sth")
|
||||
CK_PR_STORE_S(8, uint8_t, "stb")
|
||||
CK_PR_STORE_S(uint, unsigned int, "stw")
|
||||
CK_PR_STORE_S(int, int, "stw")
|
||||
CK_PR_STORE_S(short, short, "sth")
|
||||
CK_PR_STORE_S(char, char, "stb")
|
||||
|
||||
#undef CK_PR_STORE_S
|
||||
#undef CK_PR_STORE
|
||||
|
||||
#define CK_PR_CAS(N, T, M) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"lwarx %0, 0, %1;" \
|
||||
"cmpw 0, %0, %3;" \
|
||||
"bne- 2f;" \
|
||||
"stwcx. %2, 0, %1;" \
|
||||
"bne- 1b;" \
|
||||
"2:" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (set), \
|
||||
"r" (compare) \
|
||||
: "memory", "cc"); \
|
||||
*(T *)value = previous; \
|
||||
return (previous == compare); \
|
||||
} \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N(M *target, T compare, T set) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"lwarx %0, 0, %1;" \
|
||||
"cmpw 0, %0, %3;" \
|
||||
"bne- 2f;" \
|
||||
"stwcx. %2, 0, %1;" \
|
||||
"bne- 1b;" \
|
||||
"2:" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (set), \
|
||||
"r" (compare) \
|
||||
: "memory", "cc"); \
|
||||
return (previous == compare); \
|
||||
}
|
||||
|
||||
CK_PR_CAS(ptr, void *, void)
|
||||
#define CK_PR_CAS_S(a, b) CK_PR_CAS(a, b, b)
|
||||
CK_PR_CAS_S(32, uint32_t)
|
||||
CK_PR_CAS_S(uint, unsigned int)
|
||||
CK_PR_CAS_S(int, int)
|
||||
|
||||
#undef CK_PR_CAS_S
|
||||
#undef CK_PR_CAS
|
||||
|
||||
#define CK_PR_FAS(N, M, T, W) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_fas_##N(M *target, T v) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"l" W "arx %0, 0, %1;" \
|
||||
"st" W "cx. %2, 0, %1;" \
|
||||
"bne- 1b;" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (v) \
|
||||
: "memory", "cc"); \
|
||||
return (previous); \
|
||||
}
|
||||
|
||||
CK_PR_FAS(32, uint32_t, uint32_t, "w")
|
||||
CK_PR_FAS(ptr, void, void *, "w")
|
||||
CK_PR_FAS(int, int, int, "w")
|
||||
CK_PR_FAS(uint, unsigned int, unsigned int, "w")
|
||||
|
||||
#undef CK_PR_FAS
|
||||
|
||||
#define CK_PR_UNARY(O, N, M, T, I, W) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##O##_##N(M *target) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"l" W "arx %0, 0, %1;" \
|
||||
I ";" \
|
||||
"st" W "cx. %0, 0, %1;" \
|
||||
"bne- 1b;" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "w")
|
||||
CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "w")
|
||||
CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "w")
|
||||
CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "w")
|
||||
|
||||
#define CK_PR_UNARY_S(S, T, W) \
|
||||
CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \
|
||||
CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \
|
||||
CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \
|
||||
CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W)
|
||||
|
||||
CK_PR_UNARY_S(32, uint32_t, "w")
|
||||
CK_PR_UNARY_S(uint, unsigned int, "w")
|
||||
CK_PR_UNARY_S(int, int, "w")
|
||||
|
||||
#undef CK_PR_UNARY_S
|
||||
#undef CK_PR_UNARY
|
||||
|
||||
#define CK_PR_BINARY(O, N, M, T, I, W) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##O##_##N(M *target, T delta) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"l" W "arx %0, 0, %1;" \
|
||||
I " %0, %2, %0;" \
|
||||
"st" W "cx. %0, 0, %1;" \
|
||||
"bne- 1b;" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (delta) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "w")
|
||||
CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "w")
|
||||
CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "w")
|
||||
CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "w")
|
||||
CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "w")
|
||||
|
||||
#define CK_PR_BINARY_S(S, T, W) \
|
||||
CK_PR_BINARY(and, S, T, T, "and", W) \
|
||||
CK_PR_BINARY(add, S, T, T, "add", W) \
|
||||
CK_PR_BINARY(or, S, T, T, "or", W) \
|
||||
CK_PR_BINARY(sub, S, T, T, "subf", W) \
|
||||
CK_PR_BINARY(xor, S, T, T, "xor", W)
|
||||
|
||||
CK_PR_BINARY_S(32, uint32_t, "w")
|
||||
CK_PR_BINARY_S(uint, unsigned int, "w")
|
||||
CK_PR_BINARY_S(int, int, "w")
|
||||
|
||||
#undef CK_PR_BINARY_S
|
||||
#undef CK_PR_BINARY
|
||||
|
||||
CK_CC_INLINE static void *
|
||||
ck_pr_faa_ptr(void *target, uintptr_t delta)
|
||||
{
|
||||
uintptr_t previous, r;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"lwarx %0, 0, %2;"
|
||||
"add %1, %3, %0;"
|
||||
"stwcx. %1, 0, %2;"
|
||||
"bne- 1b;"
|
||||
: "=&r" (previous),
|
||||
"=&r" (r)
|
||||
: "r" (target),
|
||||
"r" (delta)
|
||||
: "memory", "cc");
|
||||
|
||||
return (void *)(previous);
|
||||
}
|
||||
|
||||
#define CK_PR_FAA(S, T, W) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_faa_##S(T *target, T delta) \
|
||||
{ \
|
||||
T previous, r; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"l" W "arx %0, 0, %2;" \
|
||||
"add %1, %3, %0;" \
|
||||
"st" W "cx. %1, 0, %2;" \
|
||||
"bne- 1b;" \
|
||||
: "=&r" (previous), \
|
||||
"=&r" (r) \
|
||||
: "r" (target), \
|
||||
"r" (delta) \
|
||||
: "memory", "cc"); \
|
||||
return (previous); \
|
||||
}
|
||||
|
||||
CK_PR_FAA(32, uint32_t, "w")
|
||||
CK_PR_FAA(uint, unsigned int, "w")
|
||||
CK_PR_FAA(int, int, "w")
|
||||
|
||||
#undef CK_PR_FAA
|
||||
|
||||
#endif /* CK_PR_PPC_H */
|
||||
|
||||
97
include/gcc/ppc64/ck_f_pr.h
Normal file
97
include/gcc/ppc64/ck_f_pr.h
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
/* DO NOT EDIT. This is auto-generated from feature.sh */
|
||||
#define CK_F_PR_ADD_32
|
||||
#define CK_F_PR_ADD_64
|
||||
#define CK_F_PR_ADD_INT
|
||||
#define CK_F_PR_ADD_PTR
|
||||
#define CK_F_PR_ADD_UINT
|
||||
#define CK_F_PR_AND_32
|
||||
#define CK_F_PR_AND_64
|
||||
#define CK_F_PR_AND_INT
|
||||
#define CK_F_PR_AND_PTR
|
||||
#define CK_F_PR_AND_UINT
|
||||
#define CK_F_PR_CAS_32
|
||||
#define CK_F_PR_CAS_32_VALUE
|
||||
#define CK_F_PR_CAS_64
|
||||
#define CK_F_PR_CAS_64_VALUE
|
||||
#define CK_F_PR_CAS_INT
|
||||
#define CK_F_PR_CAS_INT_VALUE
|
||||
#define CK_F_PR_CAS_PTR
|
||||
#define CK_F_PR_CAS_PTR_VALUE
|
||||
#define CK_F_PR_CAS_UINT
|
||||
#define CK_F_PR_CAS_UINT_VALUE
|
||||
#define CK_F_PR_DEC_32
|
||||
#define CK_F_PR_DEC_64
|
||||
#define CK_F_PR_DEC_INT
|
||||
#define CK_F_PR_DEC_PTR
|
||||
#define CK_F_PR_DEC_UINT
|
||||
#define CK_F_PR_FAA_32
|
||||
#define CK_F_PR_FAA_64
|
||||
#define CK_F_PR_FAA_INT
|
||||
#define CK_F_PR_FAA_PTR
|
||||
#define CK_F_PR_FAA_UINT
|
||||
#define CK_F_PR_FAS_32
|
||||
#define CK_F_PR_FAS_64
|
||||
#define CK_F_PR_FAS_INT
|
||||
#define CK_F_PR_FAS_PTR
|
||||
#define CK_F_PR_FAS_UINT
|
||||
#define CK_F_PR_FAS_DOUBLE
|
||||
#define CK_F_PR_FENCE_LOAD
|
||||
#define CK_F_PR_FENCE_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_MEMORY
|
||||
#define CK_F_PR_FENCE_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_STRICT_MEMORY
|
||||
#define CK_F_PR_FENCE_STRICT_STORE
|
||||
#define CK_F_PR_INC_32
|
||||
#define CK_F_PR_INC_64
|
||||
#define CK_F_PR_INC_INT
|
||||
#define CK_F_PR_INC_PTR
|
||||
#define CK_F_PR_INC_UINT
|
||||
#define CK_F_PR_LOAD_16
|
||||
#define CK_F_PR_LOAD_32
|
||||
#define CK_F_PR_LOAD_64
|
||||
#define CK_F_PR_LOAD_8
|
||||
#define CK_F_PR_LOAD_CHAR
|
||||
#define CK_F_PR_LOAD_DOUBLE
|
||||
#define CK_F_PR_LOAD_INT
|
||||
#define CK_F_PR_LOAD_PTR
|
||||
#define CK_F_PR_LOAD_SHORT
|
||||
#define CK_F_PR_LOAD_UINT
|
||||
#define CK_F_PR_NEG_32
|
||||
#define CK_F_PR_NEG_64
|
||||
#define CK_F_PR_NEG_INT
|
||||
#define CK_F_PR_NEG_PTR
|
||||
#define CK_F_PR_NEG_UINT
|
||||
#define CK_F_PR_NOT_32
|
||||
#define CK_F_PR_NOT_64
|
||||
#define CK_F_PR_NOT_INT
|
||||
#define CK_F_PR_NOT_PTR
|
||||
#define CK_F_PR_NOT_UINT
|
||||
#define CK_F_PR_OR_32
|
||||
#define CK_F_PR_OR_64
|
||||
#define CK_F_PR_OR_INT
|
||||
#define CK_F_PR_OR_PTR
|
||||
#define CK_F_PR_OR_UINT
|
||||
#define CK_F_PR_STALL
|
||||
#define CK_F_PR_STORE_16
|
||||
#define CK_F_PR_STORE_32
|
||||
#define CK_F_PR_STORE_64
|
||||
#define CK_F_PR_STORE_8
|
||||
#define CK_F_PR_STORE_CHAR
|
||||
#define CK_F_PR_STORE_DOUBLE
|
||||
#define CK_F_PR_STORE_INT
|
||||
#define CK_F_PR_STORE_PTR
|
||||
#define CK_F_PR_STORE_SHORT
|
||||
#define CK_F_PR_STORE_UINT
|
||||
#define CK_F_PR_SUB_32
|
||||
#define CK_F_PR_SUB_64
|
||||
#define CK_F_PR_SUB_INT
|
||||
#define CK_F_PR_SUB_PTR
|
||||
#define CK_F_PR_SUB_UINT
|
||||
#define CK_F_PR_XOR_32
|
||||
#define CK_F_PR_XOR_64
|
||||
#define CK_F_PR_XOR_INT
|
||||
#define CK_F_PR_XOR_PTR
|
||||
#define CK_F_PR_XOR_UINT
|
||||
|
||||
427
include/gcc/ppc64/ck_pr.h
Normal file
427
include/gcc/ppc64/ck_pr.h
Normal file
|
|
@ -0,0 +1,427 @@
|
|||
/*
|
||||
* Copyright 2009-2015 Samy Al Bahra.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_PPC64_H
|
||||
#define CK_PR_PPC64_H
|
||||
|
||||
#ifndef CK_PR_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
#include <ck_cc.h>
|
||||
#include <ck_md.h>
|
||||
|
||||
/*
|
||||
* The following represent supported atomic operations.
|
||||
* These operations may be emulated.
|
||||
*/
|
||||
#include "ck_f_pr.h"
|
||||
|
||||
/*
|
||||
* Minimum interface requirement met.
|
||||
*/
|
||||
#define CK_F_PR
|
||||
|
||||
/*
|
||||
* This bounces the hardware thread from low to medium
|
||||
* priority. I am unsure of the benefits of this approach
|
||||
* but it is used by the Linux kernel.
|
||||
*/
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_stall(void)
|
||||
{
|
||||
|
||||
__asm__ __volatile__("or 1, 1, 1;"
|
||||
"or 2, 2, 2;" ::: "memory");
|
||||
return;
|
||||
}
|
||||
|
||||
#define CK_PR_FENCE(T, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_fence_strict_##T(void) \
|
||||
{ \
|
||||
__asm__ __volatile__(I ::: "memory"); \
|
||||
}
|
||||
|
||||
/*
|
||||
* These are derived from:
|
||||
* http://www.ibm.com/developerworks/systems/articles/powerpc.html
|
||||
*/
|
||||
CK_PR_FENCE(atomic, "lwsync")
|
||||
CK_PR_FENCE(atomic_store, "lwsync")
|
||||
CK_PR_FENCE(atomic_load, "sync")
|
||||
CK_PR_FENCE(store_atomic, "lwsync")
|
||||
CK_PR_FENCE(load_atomic, "lwsync")
|
||||
CK_PR_FENCE(store, "lwsync")
|
||||
CK_PR_FENCE(store_load, "sync")
|
||||
CK_PR_FENCE(load, "lwsync")
|
||||
CK_PR_FENCE(load_store, "lwsync")
|
||||
CK_PR_FENCE(memory, "sync")
|
||||
CK_PR_FENCE(acquire, "lwsync")
|
||||
CK_PR_FENCE(release, "lwsync")
|
||||
CK_PR_FENCE(acqrel, "lwsync")
|
||||
CK_PR_FENCE(lock, "lwsync")
|
||||
CK_PR_FENCE(unlock, "lwsync")
|
||||
|
||||
#undef CK_PR_FENCE
|
||||
|
||||
#define CK_PR_LOAD(S, M, T, C, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_md_load_##S(const M *target) \
|
||||
{ \
|
||||
T r; \
|
||||
__asm__ __volatile__(I "%U1%X1 %0, %1" \
|
||||
: "=r" (r) \
|
||||
: "m" (*(const C *)target) \
|
||||
: "memory"); \
|
||||
return (r); \
|
||||
}
|
||||
|
||||
CK_PR_LOAD(ptr, void, void *, uint64_t, "ld")
|
||||
|
||||
#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I)
|
||||
|
||||
CK_PR_LOAD_S(64, uint64_t, "ld")
|
||||
CK_PR_LOAD_S(32, uint32_t, "lwz")
|
||||
CK_PR_LOAD_S(16, uint16_t, "lhz")
|
||||
CK_PR_LOAD_S(8, uint8_t, "lbz")
|
||||
CK_PR_LOAD_S(uint, unsigned int, "lwz")
|
||||
CK_PR_LOAD_S(int, int, "lwz")
|
||||
CK_PR_LOAD_S(short, short, "lhz")
|
||||
CK_PR_LOAD_S(char, char, "lbz")
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_LOAD_S(double, double, "ld")
|
||||
#endif
|
||||
|
||||
#undef CK_PR_LOAD_S
|
||||
#undef CK_PR_LOAD
|
||||
|
||||
#define CK_PR_STORE(S, M, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I "%U0%X0 %1, %0" \
|
||||
: "=m" (*(C *)target) \
|
||||
: "r" (v) \
|
||||
: "memory"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_STORE(ptr, void, const void *, uint64_t, "std")
|
||||
|
||||
#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I)
|
||||
|
||||
CK_PR_STORE_S(64, uint64_t, "std")
|
||||
CK_PR_STORE_S(32, uint32_t, "stw")
|
||||
CK_PR_STORE_S(16, uint16_t, "sth")
|
||||
CK_PR_STORE_S(8, uint8_t, "stb")
|
||||
CK_PR_STORE_S(uint, unsigned int, "stw")
|
||||
CK_PR_STORE_S(int, int, "stw")
|
||||
CK_PR_STORE_S(short, short, "sth")
|
||||
CK_PR_STORE_S(char, char, "stb")
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_STORE_S(double, double, "std")
|
||||
#endif
|
||||
|
||||
#undef CK_PR_STORE_S
|
||||
#undef CK_PR_STORE
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value)
|
||||
{
|
||||
uint64_t previous;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"ldarx %0, 0, %1;"
|
||||
"cmpd 0, %0, %3;"
|
||||
"bne- 2f;"
|
||||
"stdcx. %2, 0, %1;"
|
||||
"bne- 1b;"
|
||||
"2:"
|
||||
: "=&r" (previous)
|
||||
: "r" (target),
|
||||
"r" (set),
|
||||
"r" (compare)
|
||||
: "memory", "cc");
|
||||
|
||||
*value = previous;
|
||||
return (previous == compare);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value)
|
||||
{
|
||||
void *previous;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"ldarx %0, 0, %1;"
|
||||
"cmpd 0, %0, %3;"
|
||||
"bne- 2f;"
|
||||
"stdcx. %2, 0, %1;"
|
||||
"bne- 1b;"
|
||||
"2:"
|
||||
: "=&r" (previous)
|
||||
: "r" (target),
|
||||
"r" (set),
|
||||
"r" (compare)
|
||||
: "memory", "cc");
|
||||
|
||||
ck_pr_md_store_ptr(value, previous);
|
||||
return (previous == compare);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set)
|
||||
{
|
||||
uint64_t previous;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"ldarx %0, 0, %1;"
|
||||
"cmpd 0, %0, %3;"
|
||||
"bne- 2f;"
|
||||
"stdcx. %2, 0, %1;"
|
||||
"bne- 1b;"
|
||||
"2:"
|
||||
: "=&r" (previous)
|
||||
: "r" (target),
|
||||
"r" (set),
|
||||
"r" (compare)
|
||||
: "memory", "cc");
|
||||
|
||||
return (previous == compare);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr(void *target, void *compare, void *set)
|
||||
{
|
||||
void *previous;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"ldarx %0, 0, %1;"
|
||||
"cmpd 0, %0, %3;"
|
||||
"bne- 2f;"
|
||||
"stdcx. %2, 0, %1;"
|
||||
"bne- 1b;"
|
||||
"2:"
|
||||
: "=&r" (previous)
|
||||
: "r" (target),
|
||||
"r" (set),
|
||||
"r" (compare)
|
||||
: "memory", "cc");
|
||||
|
||||
return (previous == compare);
|
||||
}
|
||||
|
||||
#define CK_PR_CAS(N, T) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"lwarx %0, 0, %1;" \
|
||||
"cmpw 0, %0, %3;" \
|
||||
"bne- 2f;" \
|
||||
"stwcx. %2, 0, %1;" \
|
||||
"bne- 1b;" \
|
||||
"2:" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (set), \
|
||||
"r" (compare) \
|
||||
: "memory", "cc"); \
|
||||
*value = previous; \
|
||||
return (previous == compare); \
|
||||
} \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N(T *target, T compare, T set) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"lwarx %0, 0, %1;" \
|
||||
"cmpw 0, %0, %3;" \
|
||||
"bne- 2f;" \
|
||||
"stwcx. %2, 0, %1;" \
|
||||
"bne- 1b;" \
|
||||
"2:" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (set), \
|
||||
"r" (compare) \
|
||||
: "memory", "cc"); \
|
||||
return (previous == compare); \
|
||||
}
|
||||
|
||||
CK_PR_CAS(32, uint32_t)
|
||||
CK_PR_CAS(uint, unsigned int)
|
||||
CK_PR_CAS(int, int)
|
||||
|
||||
#undef CK_PR_CAS
|
||||
|
||||
#define CK_PR_FAS(N, M, T, W) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_fas_##N(M *target, T v) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"l" W "arx %0, 0, %1;" \
|
||||
"st" W "cx. %2, 0, %1;" \
|
||||
"bne- 1b;" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (v) \
|
||||
: "memory", "cc"); \
|
||||
return (previous); \
|
||||
}
|
||||
|
||||
CK_PR_FAS(64, uint64_t, uint64_t, "d")
|
||||
CK_PR_FAS(32, uint32_t, uint32_t, "w")
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_FAS(double, double, double, "d")
|
||||
#endif
|
||||
CK_PR_FAS(ptr, void, void *, "d")
|
||||
CK_PR_FAS(int, int, int, "w")
|
||||
CK_PR_FAS(uint, unsigned int, unsigned int, "w")
|
||||
|
||||
#undef CK_PR_FAS
|
||||
|
||||
#define CK_PR_UNARY(O, N, M, T, I, W) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##O##_##N(M *target) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"l" W "arx %0, 0, %1;" \
|
||||
I ";" \
|
||||
"st" W "cx. %0, 0, %1;" \
|
||||
"bne- 1b;" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "d")
|
||||
CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "d")
|
||||
CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "d")
|
||||
CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "d")
|
||||
|
||||
#define CK_PR_UNARY_S(S, T, W) \
|
||||
CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \
|
||||
CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \
|
||||
CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \
|
||||
CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W)
|
||||
|
||||
CK_PR_UNARY_S(64, uint64_t, "d")
|
||||
CK_PR_UNARY_S(32, uint32_t, "w")
|
||||
CK_PR_UNARY_S(uint, unsigned int, "w")
|
||||
CK_PR_UNARY_S(int, int, "w")
|
||||
|
||||
#undef CK_PR_UNARY_S
|
||||
#undef CK_PR_UNARY
|
||||
|
||||
#define CK_PR_BINARY(O, N, M, T, I, W) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##O##_##N(M *target, T delta) \
|
||||
{ \
|
||||
T previous; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"l" W "arx %0, 0, %1;" \
|
||||
I " %0, %2, %0;" \
|
||||
"st" W "cx. %0, 0, %1;" \
|
||||
"bne- 1b;" \
|
||||
: "=&r" (previous) \
|
||||
: "r" (target), \
|
||||
"r" (delta) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "d")
|
||||
CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "d")
|
||||
CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "d")
|
||||
CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "d")
|
||||
CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "d")
|
||||
|
||||
#define CK_PR_BINARY_S(S, T, W) \
|
||||
CK_PR_BINARY(and, S, T, T, "and", W) \
|
||||
CK_PR_BINARY(add, S, T, T, "add", W) \
|
||||
CK_PR_BINARY(or, S, T, T, "or", W) \
|
||||
CK_PR_BINARY(sub, S, T, T, "subf", W) \
|
||||
CK_PR_BINARY(xor, S, T, T, "xor", W)
|
||||
|
||||
CK_PR_BINARY_S(64, uint64_t, "d")
|
||||
CK_PR_BINARY_S(32, uint32_t, "w")
|
||||
CK_PR_BINARY_S(uint, unsigned int, "w")
|
||||
CK_PR_BINARY_S(int, int, "w")
|
||||
|
||||
#undef CK_PR_BINARY_S
|
||||
#undef CK_PR_BINARY
|
||||
|
||||
CK_CC_INLINE static void *
|
||||
ck_pr_faa_ptr(void *target, uintptr_t delta)
|
||||
{
|
||||
uintptr_t previous, r;
|
||||
|
||||
__asm__ __volatile__("1:"
|
||||
"ldarx %0, 0, %2;"
|
||||
"add %1, %3, %0;"
|
||||
"stdcx. %1, 0, %2;"
|
||||
"bne- 1b;"
|
||||
: "=&r" (previous),
|
||||
"=&r" (r)
|
||||
: "r" (target),
|
||||
"r" (delta)
|
||||
: "memory", "cc");
|
||||
|
||||
return (void *)(previous);
|
||||
}
|
||||
|
||||
#define CK_PR_FAA(S, T, W) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_faa_##S(T *target, T delta) \
|
||||
{ \
|
||||
T previous, r; \
|
||||
__asm__ __volatile__("1:" \
|
||||
"l" W "arx %0, 0, %2;" \
|
||||
"add %1, %3, %0;" \
|
||||
"st" W "cx. %1, 0, %2;" \
|
||||
"bne- 1b;" \
|
||||
: "=&r" (previous), \
|
||||
"=&r" (r) \
|
||||
: "r" (target), \
|
||||
"r" (delta) \
|
||||
: "memory", "cc"); \
|
||||
return (previous); \
|
||||
}
|
||||
|
||||
CK_PR_FAA(64, uint64_t, "d")
|
||||
CK_PR_FAA(32, uint32_t, "w")
|
||||
CK_PR_FAA(uint, unsigned int, "w")
|
||||
CK_PR_FAA(int, int, "w")
|
||||
|
||||
#undef CK_PR_FAA
|
||||
|
||||
#endif /* CK_PR_PPC64_H */
|
||||
26
include/gcc/sparcv9/ck_f_pr.h
Normal file
26
include/gcc/sparcv9/ck_f_pr.h
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
#define CK_F_PR_CAS_64
|
||||
#define CK_F_PR_CAS_64_VALUE
|
||||
#define CK_F_PR_CAS_PTR
|
||||
#define CK_F_PR_CAS_PTR_VALUE
|
||||
#define CK_F_PR_FAS_32
|
||||
#define CK_F_PR_FAS_UINT
|
||||
#define CK_F_PR_FAS_INT
|
||||
#define CK_F_PR_CAS_32
|
||||
#define CK_F_PR_CAS_32_VALUE
|
||||
#define CK_F_PR_CAS_UINT
|
||||
#define CK_F_PR_CAS_INT
|
||||
#define CK_F_PR_CAS_UINT_VALUE
|
||||
#define CK_F_PR_CAS_INT_VALUE
|
||||
#define CK_F_PR_STORE_64
|
||||
#define CK_F_PR_STORE_32
|
||||
#define CK_F_PR_STORE_DOUBLE
|
||||
#define CK_F_PR_STORE_UINT
|
||||
#define CK_F_PR_STORE_INT
|
||||
#define CK_F_PR_STORE_PTR
|
||||
#define CK_F_PR_LOAD_64
|
||||
#define CK_F_PR_LOAD_32
|
||||
#define CK_F_PR_LOAD_DOUBLE
|
||||
#define CK_F_PR_LOAD_UINT
|
||||
#define CK_F_PR_LOAD_INT
|
||||
#define CK_F_PR_LOAD_PTR
|
||||
|
||||
228
include/gcc/sparcv9/ck_pr.h
Normal file
228
include/gcc/sparcv9/ck_pr.h
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
/*
|
||||
* Copyright 2009, 2010 Samy Al Bahra.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_SPARCV9_H
|
||||
#define CK_PR_SPARCV9_H
|
||||
|
||||
#ifndef CK_PR_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
#include <ck_cc.h>
|
||||
#include <ck_md.h>
|
||||
|
||||
/*
|
||||
* The following represent supported atomic operations.
|
||||
* These operations may be emulated.
|
||||
*/
|
||||
#include "ck_f_pr.h"
|
||||
|
||||
/*
|
||||
* Minimum interface requirement met.
|
||||
*/
|
||||
#define CK_F_PR
|
||||
|
||||
/*
|
||||
* Order loads at the least.
|
||||
*/
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_stall(void)
|
||||
{
|
||||
|
||||
__asm__ __volatile__("membar #LoadLoad" ::: "memory");
|
||||
return;
|
||||
}
|
||||
|
||||
#define CK_PR_FENCE(T, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_fence_strict_##T(void) \
|
||||
{ \
|
||||
__asm__ __volatile__(I ::: "memory"); \
|
||||
}
|
||||
|
||||
/*
|
||||
* Atomic operations are treated as both load and store
|
||||
* operations on SPARCv9.
|
||||
*/
|
||||
CK_PR_FENCE(atomic, "membar #StoreStore")
|
||||
CK_PR_FENCE(atomic_store, "membar #StoreStore")
|
||||
CK_PR_FENCE(atomic_load, "membar #StoreLoad")
|
||||
CK_PR_FENCE(store_atomic, "membar #StoreStore")
|
||||
CK_PR_FENCE(load_atomic, "membar #LoadStore")
|
||||
CK_PR_FENCE(store, "membar #StoreStore")
|
||||
CK_PR_FENCE(store_load, "membar #StoreLoad")
|
||||
CK_PR_FENCE(load, "membar #LoadLoad")
|
||||
CK_PR_FENCE(load_store, "membar #LoadStore")
|
||||
CK_PR_FENCE(memory, "membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad")
|
||||
CK_PR_FENCE(acquire, "membar #LoadLoad | #LoadStore")
|
||||
CK_PR_FENCE(release, "membar #LoadStore | #StoreStore")
|
||||
CK_PR_FENCE(acqrel, "membar #LoadLoad | #LoadStore | #StoreStore")
|
||||
CK_PR_FENCE(lock, "membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad")
|
||||
CK_PR_FENCE(unlock, "membar #LoadStore | #StoreStore")
|
||||
|
||||
#undef CK_PR_FENCE
|
||||
|
||||
#define CK_PR_LOAD(S, M, T, C, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_md_load_##S(const M *target) \
|
||||
{ \
|
||||
T r; \
|
||||
__asm__ __volatile__(I " [%1], %0" \
|
||||
: "=&r" (r) \
|
||||
: "r" (target) \
|
||||
: "memory"); \
|
||||
return (r); \
|
||||
}
|
||||
|
||||
CK_PR_LOAD(ptr, void, void *, uint64_t, "ldx")
|
||||
|
||||
#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I)
|
||||
|
||||
CK_PR_LOAD_S(64, uint64_t, "ldx")
|
||||
CK_PR_LOAD_S(32, uint32_t, "lduw")
|
||||
CK_PR_LOAD_S(uint, unsigned int, "lduw")
|
||||
CK_PR_LOAD_S(double, double, "ldx")
|
||||
CK_PR_LOAD_S(int, int, "ldsw")
|
||||
|
||||
#undef CK_PR_LOAD_S
|
||||
#undef CK_PR_LOAD
|
||||
|
||||
#define CK_PR_STORE(S, M, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I " %0, [%1]" \
|
||||
: \
|
||||
: "r" (v), \
|
||||
"r" (target) \
|
||||
: "memory"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_STORE(ptr, void, const void *, uint64_t, "stx")
|
||||
|
||||
#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I)
|
||||
|
||||
CK_PR_STORE_S(8, uint8_t, "stub")
|
||||
CK_PR_STORE_S(64, uint64_t, "stx")
|
||||
CK_PR_STORE_S(32, uint32_t, "stuw")
|
||||
CK_PR_STORE_S(uint, unsigned int, "stuw")
|
||||
CK_PR_STORE_S(double, double, "stx")
|
||||
CK_PR_STORE_S(int, int, "stsw")
|
||||
|
||||
#undef CK_PR_STORE_S
|
||||
#undef CK_PR_STORE
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value)
|
||||
{
|
||||
|
||||
__asm__ __volatile__("casx [%1], %2, %0"
|
||||
: "+&r" (set)
|
||||
: "r" (target),
|
||||
"r" (compare)
|
||||
: "memory");
|
||||
|
||||
*value = set;
|
||||
return (compare == set);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set)
|
||||
{
|
||||
|
||||
__asm__ __volatile__("casx [%1], %2, %0"
|
||||
: "+&r" (set)
|
||||
: "r" (target),
|
||||
"r" (compare)
|
||||
: "memory");
|
||||
|
||||
return (compare == set);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr(void *target, void *compare, void *set)
|
||||
{
|
||||
|
||||
return ck_pr_cas_64(target, (uint64_t)compare, (uint64_t)set);
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *previous)
|
||||
{
|
||||
|
||||
return ck_pr_cas_64_value(target, (uint64_t)compare, (uint64_t)set, previous);
|
||||
}
|
||||
|
||||
#define CK_PR_CAS(N, T) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \
|
||||
{ \
|
||||
__asm__ __volatile__("cas [%1], %2, %0" \
|
||||
: "+&r" (set) \
|
||||
: "r" (target), \
|
||||
"r" (compare) \
|
||||
: "memory"); \
|
||||
*value = set; \
|
||||
return (compare == set); \
|
||||
} \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##N(T *target, T compare, T set) \
|
||||
{ \
|
||||
__asm__ __volatile__("cas [%1], %2, %0" \
|
||||
: "+&r" (set) \
|
||||
: "r" (target), \
|
||||
"r" (compare) \
|
||||
: "memory"); \
|
||||
return (compare == set); \
|
||||
}
|
||||
|
||||
CK_PR_CAS(32, uint32_t)
|
||||
CK_PR_CAS(uint, unsigned int)
|
||||
CK_PR_CAS(int, int)
|
||||
|
||||
#undef CK_PR_CAS
|
||||
|
||||
#define CK_PR_FAS(N, T) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_fas_##N(T *target, T update) \
|
||||
{ \
|
||||
\
|
||||
__asm__ __volatile__("swap [%1], %0" \
|
||||
: "+&r" (update) \
|
||||
: "r" (target) \
|
||||
: "memory"); \
|
||||
return (update); \
|
||||
}
|
||||
|
||||
CK_PR_FAS(int, int)
|
||||
CK_PR_FAS(uint, unsigned int)
|
||||
CK_PR_FAS(32, uint32_t)
|
||||
|
||||
#undef CK_PR_FAS
|
||||
|
||||
#endif /* CK_PR_SPARCV9_H */
|
||||
|
||||
152
include/gcc/x86/ck_f_pr.h
Normal file
152
include/gcc/x86/ck_f_pr.h
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
/* DO NOT EDIT. This is auto-generated from feature.sh */
|
||||
#define CK_F_PR_ADD_16
|
||||
#define CK_F_PR_ADD_32
|
||||
#define CK_F_PR_ADD_8
|
||||
#define CK_F_PR_ADD_CHAR
|
||||
#define CK_F_PR_ADD_INT
|
||||
#define CK_F_PR_ADD_PTR
|
||||
#define CK_F_PR_ADD_UINT
|
||||
#define CK_F_PR_AND_16
|
||||
#define CK_F_PR_AND_32
|
||||
#define CK_F_PR_AND_8
|
||||
#define CK_F_PR_AND_CHAR
|
||||
#define CK_F_PR_AND_INT
|
||||
#define CK_F_PR_AND_PTR
|
||||
#define CK_F_PR_AND_UINT
|
||||
#define CK_F_PR_BTC_16
|
||||
#define CK_F_PR_BTC_32
|
||||
#define CK_F_PR_BTC_INT
|
||||
#define CK_F_PR_BTC_PTR
|
||||
#define CK_F_PR_BTC_UINT
|
||||
#define CK_F_PR_BTR_16
|
||||
#define CK_F_PR_BTR_32
|
||||
#define CK_F_PR_BTR_INT
|
||||
#define CK_F_PR_BTR_PTR
|
||||
#define CK_F_PR_BTR_UINT
|
||||
#define CK_F_PR_BTS_16
|
||||
#define CK_F_PR_BTS_32
|
||||
#define CK_F_PR_BTS_INT
|
||||
#define CK_F_PR_BTS_PTR
|
||||
#define CK_F_PR_BTS_UINT
|
||||
#define CK_F_PR_CAS_16
|
||||
#define CK_F_PR_CAS_16_VALUE
|
||||
#define CK_F_PR_CAS_32
|
||||
#define CK_F_PR_CAS_32_VALUE
|
||||
#define CK_F_PR_CAS_8
|
||||
#define CK_F_PR_CAS_8_VALUE
|
||||
#define CK_F_PR_CAS_CHAR
|
||||
#define CK_F_PR_CAS_CHAR_VALUE
|
||||
#define CK_F_PR_CAS_INT
|
||||
#define CK_F_PR_CAS_INT_VALUE
|
||||
#define CK_F_PR_CAS_PTR
|
||||
#define CK_F_PR_CAS_PTR_VALUE
|
||||
#define CK_F_PR_CAS_UINT
|
||||
#define CK_F_PR_CAS_UINT_VALUE
|
||||
#define CK_F_PR_DEC_16
|
||||
#define CK_F_PR_DEC_16_ZERO
|
||||
#define CK_F_PR_DEC_32
|
||||
#define CK_F_PR_DEC_32_ZERO
|
||||
#define CK_F_PR_DEC_8
|
||||
#define CK_F_PR_DEC_8_ZERO
|
||||
#define CK_F_PR_DEC_CHAR
|
||||
#define CK_F_PR_DEC_CHAR_ZERO
|
||||
#define CK_F_PR_DEC_INT
|
||||
#define CK_F_PR_DEC_INT_ZERO
|
||||
#define CK_F_PR_DEC_PTR
|
||||
#define CK_F_PR_DEC_PTR_ZERO
|
||||
#define CK_F_PR_DEC_UINT
|
||||
#define CK_F_PR_DEC_UINT_ZERO
|
||||
#define CK_F_PR_FAA_16
|
||||
#define CK_F_PR_FAA_32
|
||||
#define CK_F_PR_FAA_8
|
||||
#define CK_F_PR_FAA_CHAR
|
||||
#define CK_F_PR_FAA_INT
|
||||
#define CK_F_PR_FAA_PTR
|
||||
#define CK_F_PR_FAA_UINT
|
||||
#define CK_F_PR_FAS_16
|
||||
#define CK_F_PR_FAS_32
|
||||
#define CK_F_PR_FAS_8
|
||||
#define CK_F_PR_FAS_CHAR
|
||||
#define CK_F_PR_FAS_INT
|
||||
#define CK_F_PR_FAS_PTR
|
||||
#define CK_F_PR_FAS_UINT
|
||||
#define CK_F_PR_FENCE_LOAD
|
||||
#define CK_F_PR_FENCE_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_MEMORY
|
||||
#define CK_F_PR_FENCE_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_STRICT_MEMORY
|
||||
#define CK_F_PR_FENCE_STRICT_STORE
|
||||
#define CK_F_PR_INC_16
|
||||
#define CK_F_PR_INC_16_ZERO
|
||||
#define CK_F_PR_INC_32
|
||||
#define CK_F_PR_INC_32_ZERO
|
||||
#define CK_F_PR_INC_8
|
||||
#define CK_F_PR_INC_8_ZERO
|
||||
#define CK_F_PR_INC_CHAR
|
||||
#define CK_F_PR_INC_CHAR_ZERO
|
||||
#define CK_F_PR_INC_INT
|
||||
#define CK_F_PR_INC_INT_ZERO
|
||||
#define CK_F_PR_INC_PTR
|
||||
#define CK_F_PR_INC_PTR_ZERO
|
||||
#define CK_F_PR_INC_UINT
|
||||
#define CK_F_PR_INC_UINT_ZERO
|
||||
#define CK_F_PR_LOAD_16
|
||||
#define CK_F_PR_LOAD_32
|
||||
#define CK_F_PR_LOAD_8
|
||||
#define CK_F_PR_LOAD_CHAR
|
||||
#define CK_F_PR_LOAD_INT
|
||||
#define CK_F_PR_LOAD_PTR
|
||||
#define CK_F_PR_LOAD_UINT
|
||||
#define CK_F_PR_NEG_16
|
||||
#define CK_F_PR_NEG_16_ZERO
|
||||
#define CK_F_PR_NEG_32
|
||||
#define CK_F_PR_NEG_32_ZERO
|
||||
#define CK_F_PR_NEG_8
|
||||
#define CK_F_PR_NEG_8_ZERO
|
||||
#define CK_F_PR_NEG_CHAR
|
||||
#define CK_F_PR_NEG_CHAR_ZERO
|
||||
#define CK_F_PR_NEG_INT
|
||||
#define CK_F_PR_NEG_INT_ZERO
|
||||
#define CK_F_PR_NEG_PTR
|
||||
#define CK_F_PR_NEG_PTR_ZERO
|
||||
#define CK_F_PR_NEG_UINT
|
||||
#define CK_F_PR_NEG_UINT_ZERO
|
||||
#define CK_F_PR_NOT_16
|
||||
#define CK_F_PR_NOT_32
|
||||
#define CK_F_PR_NOT_8
|
||||
#define CK_F_PR_NOT_CHAR
|
||||
#define CK_F_PR_NOT_INT
|
||||
#define CK_F_PR_NOT_PTR
|
||||
#define CK_F_PR_NOT_UINT
|
||||
#define CK_F_PR_OR_16
|
||||
#define CK_F_PR_OR_32
|
||||
#define CK_F_PR_OR_8
|
||||
#define CK_F_PR_OR_CHAR
|
||||
#define CK_F_PR_OR_INT
|
||||
#define CK_F_PR_OR_PTR
|
||||
#define CK_F_PR_OR_UINT
|
||||
#define CK_F_PR_STALL
|
||||
#define CK_F_PR_STORE_16
|
||||
#define CK_F_PR_STORE_32
|
||||
#define CK_F_PR_STORE_8
|
||||
#define CK_F_PR_STORE_CHAR
|
||||
#define CK_F_PR_STORE_INT
|
||||
#define CK_F_PR_STORE_PTR
|
||||
#define CK_F_PR_STORE_UINT
|
||||
#define CK_F_PR_SUB_16
|
||||
#define CK_F_PR_SUB_32
|
||||
#define CK_F_PR_SUB_8
|
||||
#define CK_F_PR_SUB_CHAR
|
||||
#define CK_F_PR_SUB_INT
|
||||
#define CK_F_PR_SUB_PTR
|
||||
#define CK_F_PR_SUB_UINT
|
||||
#define CK_F_PR_XOR_16
|
||||
#define CK_F_PR_XOR_32
|
||||
#define CK_F_PR_XOR_8
|
||||
#define CK_F_PR_XOR_CHAR
|
||||
#define CK_F_PR_XOR_INT
|
||||
#define CK_F_PR_XOR_PTR
|
||||
#define CK_F_PR_XOR_UINT
|
||||
|
||||
390
include/gcc/x86/ck_pr.h
Normal file
390
include/gcc/x86/ck_pr.h
Normal file
|
|
@ -0,0 +1,390 @@
|
|||
/*
|
||||
* Copyright 2009-2015 Samy Al Bahra.
|
||||
* Copyright 2011 Devon H. O'Dell <devon.odell@gmail.com>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_X86_H
|
||||
#define CK_PR_X86_H
|
||||
|
||||
#ifndef CK_PR_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
#include <ck_cc.h>
|
||||
#include <ck_md.h>
|
||||
#include <ck_stdint.h>
|
||||
|
||||
/*
|
||||
* The following represent supported atomic operations.
|
||||
* These operations may be emulated.
|
||||
*/
|
||||
#include "ck_f_pr.h"
|
||||
|
||||
/* Minimum requirements for the CK_PR interface are met. */
|
||||
#define CK_F_PR
|
||||
|
||||
#ifdef CK_MD_UMP
|
||||
#define CK_PR_LOCK_PREFIX
|
||||
#else
|
||||
#define CK_PR_LOCK_PREFIX "lock "
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Prevent speculative execution in busy-wait loops (P4 <=)
|
||||
* or "predefined delay".
|
||||
*/
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_stall(void)
|
||||
{
|
||||
__asm__ __volatile__("pause" ::: "memory");
|
||||
return;
|
||||
}
|
||||
|
||||
#define CK_PR_FENCE(T, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_fence_strict_##T(void) \
|
||||
{ \
|
||||
__asm__ __volatile__(I ::: "memory"); \
|
||||
}
|
||||
|
||||
CK_PR_FENCE(atomic, "sfence")
|
||||
CK_PR_FENCE(atomic_store, "sfence")
|
||||
CK_PR_FENCE(atomic_load, "mfence")
|
||||
CK_PR_FENCE(store_atomic, "sfence")
|
||||
CK_PR_FENCE(load_atomic, "mfence")
|
||||
CK_PR_FENCE(load, "lfence")
|
||||
CK_PR_FENCE(load_store, "mfence")
|
||||
CK_PR_FENCE(store, "sfence")
|
||||
CK_PR_FENCE(store_load, "mfence")
|
||||
CK_PR_FENCE(memory, "mfence")
|
||||
CK_PR_FENCE(release, "mfence")
|
||||
CK_PR_FENCE(acquire, "mfence")
|
||||
CK_PR_FENCE(acqrel, "mfence")
|
||||
CK_PR_FENCE(lock, "mfence")
|
||||
CK_PR_FENCE(unlock, "mfence")
|
||||
|
||||
#undef CK_PR_FENCE
|
||||
|
||||
/*
|
||||
* Atomic fetch-and-store operations.
|
||||
*/
|
||||
#define CK_PR_FAS(S, M, T, C, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_fas_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I " %0, %1" \
|
||||
: "+m" (*(C *)target), \
|
||||
"+q" (v) \
|
||||
: \
|
||||
: "memory"); \
|
||||
return v; \
|
||||
}
|
||||
|
||||
CK_PR_FAS(ptr, void, void *, char, "xchgl")
|
||||
|
||||
#define CK_PR_FAS_S(S, T, I) CK_PR_FAS(S, T, T, T, I)
|
||||
|
||||
CK_PR_FAS_S(char, char, "xchgb")
|
||||
CK_PR_FAS_S(uint, unsigned int, "xchgl")
|
||||
CK_PR_FAS_S(int, int, "xchgl")
|
||||
CK_PR_FAS_S(32, uint32_t, "xchgl")
|
||||
CK_PR_FAS_S(16, uint16_t, "xchgw")
|
||||
CK_PR_FAS_S(8, uint8_t, "xchgb")
|
||||
|
||||
#undef CK_PR_FAS_S
|
||||
#undef CK_PR_FAS
|
||||
|
||||
#define CK_PR_LOAD(S, M, T, C, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_md_load_##S(const M *target) \
|
||||
{ \
|
||||
T r; \
|
||||
__asm__ __volatile__(I " %1, %0" \
|
||||
: "=q" (r) \
|
||||
: "m" (*(const C *)target) \
|
||||
: "memory"); \
|
||||
return (r); \
|
||||
}
|
||||
|
||||
CK_PR_LOAD(ptr, void, void *, char, "movl")
|
||||
|
||||
#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I)
|
||||
|
||||
CK_PR_LOAD_S(char, char, "movb")
|
||||
CK_PR_LOAD_S(uint, unsigned int, "movl")
|
||||
CK_PR_LOAD_S(int, int, "movl")
|
||||
CK_PR_LOAD_S(32, uint32_t, "movl")
|
||||
CK_PR_LOAD_S(16, uint16_t, "movw")
|
||||
CK_PR_LOAD_S(8, uint8_t, "movb")
|
||||
|
||||
#undef CK_PR_LOAD_S
|
||||
#undef CK_PR_LOAD
|
||||
|
||||
#define CK_PR_STORE(S, M, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I " %1, %0" \
|
||||
: "=m" (*(C *)target) \
|
||||
: CK_CC_IMM "q" (v) \
|
||||
: "memory"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_STORE(ptr, void, const void *, char, "movl")
|
||||
|
||||
#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I)
|
||||
|
||||
CK_PR_STORE_S(char, char, "movb")
|
||||
CK_PR_STORE_S(uint, unsigned int, "movl")
|
||||
CK_PR_STORE_S(int, int, "movl")
|
||||
CK_PR_STORE_S(32, uint32_t, "movl")
|
||||
CK_PR_STORE_S(16, uint16_t, "movw")
|
||||
CK_PR_STORE_S(8, uint8_t, "movb")
|
||||
|
||||
#undef CK_PR_STORE_S
|
||||
#undef CK_PR_STORE
|
||||
|
||||
/*
|
||||
* Atomic fetch-and-add operations.
|
||||
*/
|
||||
#define CK_PR_FAA(S, M, T, C, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_faa_##S(M *target, T d) \
|
||||
{ \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \
|
||||
: "+m" (*(C *)target), \
|
||||
"+q" (d) \
|
||||
: \
|
||||
: "memory", "cc"); \
|
||||
return (d); \
|
||||
}
|
||||
|
||||
CK_PR_FAA(ptr, void, uintptr_t, char, "xaddl")
|
||||
|
||||
#define CK_PR_FAA_S(S, T, I) CK_PR_FAA(S, T, T, T, I)
|
||||
|
||||
CK_PR_FAA_S(char, char, "xaddb")
|
||||
CK_PR_FAA_S(uint, unsigned int, "xaddl")
|
||||
CK_PR_FAA_S(int, int, "xaddl")
|
||||
CK_PR_FAA_S(32, uint32_t, "xaddl")
|
||||
CK_PR_FAA_S(16, uint16_t, "xaddw")
|
||||
CK_PR_FAA_S(8, uint8_t, "xaddb")
|
||||
|
||||
#undef CK_PR_FAA_S
|
||||
#undef CK_PR_FAA
|
||||
|
||||
/*
|
||||
* Atomic store-only unary operations.
|
||||
*/
|
||||
#define CK_PR_UNARY(K, S, T, C, I) \
|
||||
CK_PR_UNARY_R(K, S, T, C, I) \
|
||||
CK_PR_UNARY_V(K, S, T, C, I)
|
||||
|
||||
#define CK_PR_UNARY_R(K, S, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##K##_##S(T *target) \
|
||||
{ \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0" \
|
||||
: "+m" (*(C *)target) \
|
||||
: \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
#define CK_PR_UNARY_V(K, S, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##K##_##S##_zero(T *target, bool *r) \
|
||||
{ \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \
|
||||
: "+m" (*(C *)target), \
|
||||
"=m" (*r) \
|
||||
: \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
|
||||
#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I)
|
||||
|
||||
#define CK_PR_GENERATE(K) \
|
||||
CK_PR_UNARY(K, ptr, void, char, #K "l") \
|
||||
CK_PR_UNARY_S(K, char, char, #K "b") \
|
||||
CK_PR_UNARY_S(K, int, int, #K "l") \
|
||||
CK_PR_UNARY_S(K, uint, unsigned int, #K "l") \
|
||||
CK_PR_UNARY_S(K, 32, uint32_t, #K "l") \
|
||||
CK_PR_UNARY_S(K, 16, uint16_t, #K "w") \
|
||||
CK_PR_UNARY_S(K, 8, uint8_t, #K "b")
|
||||
|
||||
CK_PR_GENERATE(inc)
|
||||
CK_PR_GENERATE(dec)
|
||||
CK_PR_GENERATE(neg)
|
||||
|
||||
/* not does not affect condition flags. */
|
||||
#undef CK_PR_UNARY_V
|
||||
#define CK_PR_UNARY_V(a, b, c, d, e)
|
||||
CK_PR_GENERATE(not)
|
||||
|
||||
#undef CK_PR_GENERATE
|
||||
#undef CK_PR_UNARY_S
|
||||
#undef CK_PR_UNARY_V
|
||||
#undef CK_PR_UNARY_R
|
||||
#undef CK_PR_UNARY
|
||||
|
||||
/*
|
||||
* Atomic store-only binary operations.
|
||||
*/
|
||||
#define CK_PR_BINARY(K, S, M, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##K##_##S(M *target, T d) \
|
||||
{ \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \
|
||||
: "+m" (*(C *)target) \
|
||||
: CK_CC_IMM "q" (d) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
#define CK_PR_BINARY_S(K, S, T, I) CK_PR_BINARY(K, S, T, T, T, I)
|
||||
|
||||
#define CK_PR_GENERATE(K) \
|
||||
CK_PR_BINARY(K, ptr, void, uintptr_t, char, #K "l") \
|
||||
CK_PR_BINARY_S(K, char, char, #K "b") \
|
||||
CK_PR_BINARY_S(K, int, int, #K "l") \
|
||||
CK_PR_BINARY_S(K, uint, unsigned int, #K "l") \
|
||||
CK_PR_BINARY_S(K, 32, uint32_t, #K "l") \
|
||||
CK_PR_BINARY_S(K, 16, uint16_t, #K "w") \
|
||||
CK_PR_BINARY_S(K, 8, uint8_t, #K "b")
|
||||
|
||||
CK_PR_GENERATE(add)
|
||||
CK_PR_GENERATE(sub)
|
||||
CK_PR_GENERATE(and)
|
||||
CK_PR_GENERATE(or)
|
||||
CK_PR_GENERATE(xor)
|
||||
|
||||
#undef CK_PR_GENERATE
|
||||
#undef CK_PR_BINARY_S
|
||||
#undef CK_PR_BINARY
|
||||
|
||||
/*
|
||||
* Atomic compare and swap.
|
||||
*/
|
||||
#define CK_PR_CAS(S, M, T, C, I) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##S(M *target, T compare, T set) \
|
||||
{ \
|
||||
bool z; \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %2, %0; setz %1" \
|
||||
: "+m" (*(C *)target), \
|
||||
"=a" (z) \
|
||||
: "q" (set), \
|
||||
"a" (compare) \
|
||||
: "memory", "cc"); \
|
||||
return z; \
|
||||
}
|
||||
|
||||
CK_PR_CAS(ptr, void, void *, char, "cmpxchgl")
|
||||
|
||||
#define CK_PR_CAS_S(S, T, I) CK_PR_CAS(S, T, T, T, I)
|
||||
|
||||
CK_PR_CAS_S(char, char, "cmpxchgb")
|
||||
CK_PR_CAS_S(int, int, "cmpxchgl")
|
||||
CK_PR_CAS_S(uint, unsigned int, "cmpxchgl")
|
||||
CK_PR_CAS_S(32, uint32_t, "cmpxchgl")
|
||||
CK_PR_CAS_S(16, uint16_t, "cmpxchgw")
|
||||
CK_PR_CAS_S(8, uint8_t, "cmpxchgb")
|
||||
|
||||
#undef CK_PR_CAS_S
|
||||
#undef CK_PR_CAS
|
||||
|
||||
/*
|
||||
* Compare and swap, set *v to old value of target.
|
||||
*/
|
||||
#define CK_PR_CAS_O(S, M, T, C, I, R) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \
|
||||
{ \
|
||||
bool z; \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \
|
||||
"mov %% " R ", %2;" \
|
||||
"setz %1;" \
|
||||
: "+m" (*(C *)target), \
|
||||
"=a" (z), \
|
||||
"=m" (*(C *)v) \
|
||||
: "q" (set), \
|
||||
"a" (compare) \
|
||||
: "memory", "cc"); \
|
||||
return (bool)z; \
|
||||
}
|
||||
|
||||
CK_PR_CAS_O(ptr, void, void *, char, "l", "eax")
|
||||
|
||||
#define CK_PR_CAS_O_S(S, T, I, R) \
|
||||
CK_PR_CAS_O(S, T, T, T, I, R)
|
||||
|
||||
CK_PR_CAS_O_S(char, char, "b", "al")
|
||||
CK_PR_CAS_O_S(int, int, "l", "eax")
|
||||
CK_PR_CAS_O_S(uint, unsigned int, "l", "eax")
|
||||
CK_PR_CAS_O_S(32, uint32_t, "l", "eax")
|
||||
CK_PR_CAS_O_S(16, uint16_t, "w", "ax")
|
||||
CK_PR_CAS_O_S(8, uint8_t, "b", "al")
|
||||
|
||||
#undef CK_PR_CAS_O_S
|
||||
#undef CK_PR_CAS_O
|
||||
|
||||
/*
|
||||
* Atomic bit test operations.
|
||||
*/
|
||||
#define CK_PR_BT(K, S, T, P, C, I) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_##K##_##S(T *target, unsigned int b) \
|
||||
{ \
|
||||
bool c; \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I "; setc %1" \
|
||||
: "+m" (*(C *)target), \
|
||||
"=q" (c) \
|
||||
: "q" ((P)b) \
|
||||
: "memory", "cc"); \
|
||||
return (bool)c; \
|
||||
}
|
||||
|
||||
#define CK_PR_BT_S(K, S, T, I) CK_PR_BT(K, S, T, T, T, I)
|
||||
|
||||
#define CK_PR_GENERATE(K) \
|
||||
CK_PR_BT(K, ptr, void, uint32_t, char, #K "l %2, %0") \
|
||||
CK_PR_BT_S(K, uint, unsigned int, #K "l %2, %0") \
|
||||
CK_PR_BT_S(K, int, int, #K "l %2, %0") \
|
||||
CK_PR_BT_S(K, 32, uint32_t, #K "l %2, %0") \
|
||||
CK_PR_BT_S(K, 16, uint16_t, #K "w %w2, %0")
|
||||
|
||||
CK_PR_GENERATE(btc)
|
||||
CK_PR_GENERATE(bts)
|
||||
CK_PR_GENERATE(btr)
|
||||
|
||||
#undef CK_PR_GENERATE
|
||||
#undef CK_PR_BT
|
||||
|
||||
#endif /* CK_PR_X86_H */
|
||||
|
||||
202
include/gcc/x86_64/ck_f_pr.h
Normal file
202
include/gcc/x86_64/ck_f_pr.h
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
/* DO NOT EDIT. This is auto-generated from feature.sh */
|
||||
#define CK_F_PR_ADD_16
|
||||
#define CK_F_PR_ADD_32
|
||||
#define CK_F_PR_ADD_64
|
||||
#define CK_F_PR_ADD_8
|
||||
#define CK_F_PR_ADD_CHAR
|
||||
#define CK_F_PR_ADD_INT
|
||||
#define CK_F_PR_ADD_PTR
|
||||
#define CK_F_PR_ADD_UINT
|
||||
#define CK_F_PR_AND_16
|
||||
#define CK_F_PR_AND_32
|
||||
#define CK_F_PR_AND_64
|
||||
#define CK_F_PR_AND_8
|
||||
#define CK_F_PR_AND_CHAR
|
||||
#define CK_F_PR_AND_INT
|
||||
#define CK_F_PR_AND_PTR
|
||||
#define CK_F_PR_AND_UINT
|
||||
#define CK_F_PR_BTC_16
|
||||
#define CK_F_PR_BTC_32
|
||||
#define CK_F_PR_BTC_64
|
||||
#define CK_F_PR_BTC_INT
|
||||
#define CK_F_PR_BTC_PTR
|
||||
#define CK_F_PR_BTC_UINT
|
||||
#define CK_F_PR_BTR_16
|
||||
#define CK_F_PR_BTR_32
|
||||
#define CK_F_PR_BTR_64
|
||||
#define CK_F_PR_BTR_INT
|
||||
#define CK_F_PR_BTR_PTR
|
||||
#define CK_F_PR_BTR_UINT
|
||||
#define CK_F_PR_BTS_16
|
||||
#define CK_F_PR_BTS_32
|
||||
#define CK_F_PR_BTS_64
|
||||
#define CK_F_PR_BTS_INT
|
||||
#define CK_F_PR_BTS_PTR
|
||||
#define CK_F_PR_BTS_UINT
|
||||
#define CK_F_PR_CAS_16
|
||||
#define CK_F_PR_CAS_16_8
|
||||
#define CK_F_PR_CAS_16_8_VALUE
|
||||
#define CK_F_PR_CAS_16_VALUE
|
||||
#define CK_F_PR_CAS_32
|
||||
#define CK_F_PR_CAS_32_4
|
||||
#define CK_F_PR_CAS_32_4_VALUE
|
||||
#define CK_F_PR_CAS_32_VALUE
|
||||
#define CK_F_PR_CAS_64
|
||||
#define CK_F_PR_CAS_64_2
|
||||
#define CK_F_PR_CAS_64_2_VALUE
|
||||
#define CK_F_PR_CAS_64_VALUE
|
||||
#define CK_F_PR_CAS_8
|
||||
#define CK_F_PR_CAS_8_16
|
||||
#define CK_F_PR_CAS_8_16_VALUE
|
||||
#define CK_F_PR_CAS_8_VALUE
|
||||
#define CK_F_PR_CAS_CHAR
|
||||
#define CK_F_PR_CAS_CHAR_16
|
||||
#define CK_F_PR_CAS_CHAR_16_VALUE
|
||||
#define CK_F_PR_CAS_CHAR_VALUE
|
||||
#define CK_F_PR_CAS_INT
|
||||
#define CK_F_PR_CAS_INT_4
|
||||
#define CK_F_PR_CAS_INT_4_VALUE
|
||||
#define CK_F_PR_CAS_INT_VALUE
|
||||
#define CK_F_PR_CAS_PTR
|
||||
#define CK_F_PR_CAS_PTR_2
|
||||
#define CK_F_PR_CAS_PTR_2_VALUE
|
||||
#define CK_F_PR_CAS_PTR_VALUE
|
||||
#define CK_F_PR_CAS_DOUBLE
|
||||
#define CK_F_PR_CAS_DOUBLE_2
|
||||
#define CK_F_PR_CAS_DOUBLE_VALUE
|
||||
#define CK_F_PR_CAS_UINT
|
||||
#define CK_F_PR_CAS_UINT_4
|
||||
#define CK_F_PR_CAS_UINT_4_VALUE
|
||||
#define CK_F_PR_CAS_UINT_VALUE
|
||||
#define CK_F_PR_DEC_16
|
||||
#define CK_F_PR_DEC_16_ZERO
|
||||
#define CK_F_PR_DEC_32
|
||||
#define CK_F_PR_DEC_32_ZERO
|
||||
#define CK_F_PR_DEC_64
|
||||
#define CK_F_PR_DEC_64_ZERO
|
||||
#define CK_F_PR_DEC_8
|
||||
#define CK_F_PR_DEC_8_ZERO
|
||||
#define CK_F_PR_DEC_CHAR
|
||||
#define CK_F_PR_DEC_CHAR_ZERO
|
||||
#define CK_F_PR_DEC_INT
|
||||
#define CK_F_PR_DEC_INT_ZERO
|
||||
#define CK_F_PR_DEC_PTR
|
||||
#define CK_F_PR_DEC_PTR_ZERO
|
||||
#define CK_F_PR_DEC_UINT
|
||||
#define CK_F_PR_DEC_UINT_ZERO
|
||||
#define CK_F_PR_FAA_16
|
||||
#define CK_F_PR_FAA_32
|
||||
#define CK_F_PR_FAA_64
|
||||
#define CK_F_PR_FAA_8
|
||||
#define CK_F_PR_FAA_CHAR
|
||||
#define CK_F_PR_FAA_INT
|
||||
#define CK_F_PR_FAA_PTR
|
||||
#define CK_F_PR_FAA_UINT
|
||||
#define CK_F_PR_FAS_16
|
||||
#define CK_F_PR_FAS_32
|
||||
#define CK_F_PR_FAS_64
|
||||
#define CK_F_PR_FAS_8
|
||||
#define CK_F_PR_FAS_CHAR
|
||||
#define CK_F_PR_FAS_INT
|
||||
#define CK_F_PR_FAS_PTR
|
||||
#define CK_F_PR_FAS_UINT
|
||||
#define CK_F_PR_FAS_DOUBLE
|
||||
#define CK_F_PR_FENCE_LOAD
|
||||
#define CK_F_PR_FENCE_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_MEMORY
|
||||
#define CK_F_PR_FENCE_STORE
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD
|
||||
#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS
|
||||
#define CK_F_PR_FENCE_STRICT_MEMORY
|
||||
#define CK_F_PR_FENCE_STRICT_STORE
|
||||
#define CK_F_PR_INC_16
|
||||
#define CK_F_PR_INC_16_ZERO
|
||||
#define CK_F_PR_INC_32
|
||||
#define CK_F_PR_INC_32_ZERO
|
||||
#define CK_F_PR_INC_64
|
||||
#define CK_F_PR_INC_64_ZERO
|
||||
#define CK_F_PR_INC_8
|
||||
#define CK_F_PR_INC_8_ZERO
|
||||
#define CK_F_PR_INC_CHAR
|
||||
#define CK_F_PR_INC_CHAR_ZERO
|
||||
#define CK_F_PR_INC_INT
|
||||
#define CK_F_PR_INC_INT_ZERO
|
||||
#define CK_F_PR_INC_PTR
|
||||
#define CK_F_PR_INC_PTR_ZERO
|
||||
#define CK_F_PR_INC_UINT
|
||||
#define CK_F_PR_INC_UINT_ZERO
|
||||
#define CK_F_PR_LOAD_16
|
||||
#define CK_F_PR_LOAD_16_8
|
||||
#define CK_F_PR_LOAD_32
|
||||
#define CK_F_PR_LOAD_32_4
|
||||
#define CK_F_PR_LOAD_64
|
||||
#define CK_F_PR_LOAD_64_2
|
||||
#define CK_F_PR_LOAD_8
|
||||
#define CK_F_PR_LOAD_8_16
|
||||
#define CK_F_PR_LOAD_CHAR
|
||||
#define CK_F_PR_LOAD_CHAR_16
|
||||
#define CK_F_PR_LOAD_INT
|
||||
#define CK_F_PR_LOAD_INT_4
|
||||
#define CK_F_PR_LOAD_PTR
|
||||
#define CK_F_PR_LOAD_PTR_2
|
||||
#define CK_F_PR_LOAD_DOUBLE
|
||||
#define CK_F_PR_LOAD_UINT
|
||||
#define CK_F_PR_LOAD_UINT_4
|
||||
#define CK_F_PR_NEG_16
|
||||
#define CK_F_PR_NEG_16_ZERO
|
||||
#define CK_F_PR_NEG_32
|
||||
#define CK_F_PR_NEG_32_ZERO
|
||||
#define CK_F_PR_NEG_64
|
||||
#define CK_F_PR_NEG_64_ZERO
|
||||
#define CK_F_PR_NEG_8
|
||||
#define CK_F_PR_NEG_8_ZERO
|
||||
#define CK_F_PR_NEG_CHAR
|
||||
#define CK_F_PR_NEG_CHAR_ZERO
|
||||
#define CK_F_PR_NEG_INT
|
||||
#define CK_F_PR_NEG_INT_ZERO
|
||||
#define CK_F_PR_NEG_PTR
|
||||
#define CK_F_PR_NEG_PTR_ZERO
|
||||
#define CK_F_PR_NEG_UINT
|
||||
#define CK_F_PR_NEG_UINT_ZERO
|
||||
#define CK_F_PR_NOT_16
|
||||
#define CK_F_PR_NOT_32
|
||||
#define CK_F_PR_NOT_64
|
||||
#define CK_F_PR_NOT_8
|
||||
#define CK_F_PR_NOT_CHAR
|
||||
#define CK_F_PR_NOT_INT
|
||||
#define CK_F_PR_NOT_PTR
|
||||
#define CK_F_PR_NOT_UINT
|
||||
#define CK_F_PR_OR_16
|
||||
#define CK_F_PR_OR_32
|
||||
#define CK_F_PR_OR_64
|
||||
#define CK_F_PR_OR_8
|
||||
#define CK_F_PR_OR_CHAR
|
||||
#define CK_F_PR_OR_INT
|
||||
#define CK_F_PR_OR_PTR
|
||||
#define CK_F_PR_OR_UINT
|
||||
#define CK_F_PR_STORE_16
|
||||
#define CK_F_PR_STORE_32
|
||||
#define CK_F_PR_STORE_64
|
||||
#define CK_F_PR_STORE_8
|
||||
#define CK_F_PR_STORE_CHAR
|
||||
#define CK_F_PR_STORE_INT
|
||||
#define CK_F_PR_STORE_DOUBLE
|
||||
#define CK_F_PR_STORE_PTR
|
||||
#define CK_F_PR_STORE_UINT
|
||||
#define CK_F_PR_SUB_16
|
||||
#define CK_F_PR_SUB_32
|
||||
#define CK_F_PR_SUB_64
|
||||
#define CK_F_PR_SUB_8
|
||||
#define CK_F_PR_SUB_CHAR
|
||||
#define CK_F_PR_SUB_INT
|
||||
#define CK_F_PR_SUB_PTR
|
||||
#define CK_F_PR_SUB_UINT
|
||||
#define CK_F_PR_XOR_16
|
||||
#define CK_F_PR_XOR_32
|
||||
#define CK_F_PR_XOR_64
|
||||
#define CK_F_PR_XOR_8
|
||||
#define CK_F_PR_XOR_CHAR
|
||||
#define CK_F_PR_XOR_INT
|
||||
#define CK_F_PR_XOR_PTR
|
||||
#define CK_F_PR_XOR_UINT
|
||||
|
||||
585
include/gcc/x86_64/ck_pr.h
Normal file
585
include/gcc/x86_64/ck_pr.h
Normal file
|
|
@ -0,0 +1,585 @@
|
|||
/*
|
||||
* Copyright 2009-2015 Samy Al Bahra.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_X86_64_H
|
||||
#define CK_PR_X86_64_H
|
||||
|
||||
#ifndef CK_PR_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
#include <ck_cc.h>
|
||||
#include <ck_md.h>
|
||||
#include <ck_stdint.h>
|
||||
|
||||
/*
|
||||
* The following represent supported atomic operations.
|
||||
* These operations may be emulated.
|
||||
*/
|
||||
#include "ck_f_pr.h"
|
||||
|
||||
/*
|
||||
* Support for TSX extensions.
|
||||
*/
|
||||
#ifdef CK_MD_RTM_ENABLE
|
||||
#include "ck_pr_rtm.h"
|
||||
#endif
|
||||
|
||||
/* Minimum requirements for the CK_PR interface are met. */
|
||||
#define CK_F_PR
|
||||
|
||||
#ifdef CK_MD_UMP
|
||||
#define CK_PR_LOCK_PREFIX
|
||||
#else
|
||||
#define CK_PR_LOCK_PREFIX "lock "
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Prevent speculative execution in busy-wait loops (P4 <=)
|
||||
* or "predefined delay".
|
||||
*/
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_stall(void)
|
||||
{
|
||||
__asm__ __volatile__("pause" ::: "memory");
|
||||
return;
|
||||
}
|
||||
|
||||
#define CK_PR_FENCE(T, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_fence_strict_##T(void) \
|
||||
{ \
|
||||
__asm__ __volatile__(I ::: "memory"); \
|
||||
}
|
||||
|
||||
CK_PR_FENCE(atomic, "sfence")
|
||||
CK_PR_FENCE(atomic_store, "sfence")
|
||||
CK_PR_FENCE(atomic_load, "mfence")
|
||||
CK_PR_FENCE(store_atomic, "sfence")
|
||||
CK_PR_FENCE(load_atomic, "mfence")
|
||||
CK_PR_FENCE(load, "lfence")
|
||||
CK_PR_FENCE(load_store, "mfence")
|
||||
CK_PR_FENCE(store, "sfence")
|
||||
CK_PR_FENCE(store_load, "mfence")
|
||||
CK_PR_FENCE(memory, "mfence")
|
||||
CK_PR_FENCE(release, "mfence")
|
||||
CK_PR_FENCE(acquire, "mfence")
|
||||
CK_PR_FENCE(acqrel, "mfence")
|
||||
CK_PR_FENCE(lock, "mfence")
|
||||
CK_PR_FENCE(unlock, "mfence")
|
||||
|
||||
#undef CK_PR_FENCE
|
||||
|
||||
/*
|
||||
* Read for ownership. Older compilers will generate the 32-bit
|
||||
* 3DNow! variant which is binary compatible with x86-64 variant
|
||||
* of prefetchw.
|
||||
*/
|
||||
#ifndef CK_F_PR_RFO
|
||||
#define CK_F_PR_RFO
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_rfo(const void *m)
|
||||
{
|
||||
|
||||
__asm__ __volatile__("prefetchw (%0)"
|
||||
:
|
||||
: "r" (m)
|
||||
: "memory");
|
||||
|
||||
return;
|
||||
}
|
||||
#endif /* CK_F_PR_RFO */
|
||||
|
||||
/*
|
||||
* Atomic fetch-and-store operations.
|
||||
*/
|
||||
#define CK_PR_FAS(S, M, T, C, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_fas_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I " %0, %1" \
|
||||
: "+m" (*(C *)target), \
|
||||
"+q" (v) \
|
||||
: \
|
||||
: "memory"); \
|
||||
return v; \
|
||||
}
|
||||
|
||||
CK_PR_FAS(ptr, void, void *, char, "xchgq")
|
||||
|
||||
#define CK_PR_FAS_S(S, T, I) CK_PR_FAS(S, T, T, T, I)
|
||||
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_FAS_S(double, double, "xchgq")
|
||||
#endif
|
||||
CK_PR_FAS_S(char, char, "xchgb")
|
||||
CK_PR_FAS_S(uint, unsigned int, "xchgl")
|
||||
CK_PR_FAS_S(int, int, "xchgl")
|
||||
CK_PR_FAS_S(64, uint64_t, "xchgq")
|
||||
CK_PR_FAS_S(32, uint32_t, "xchgl")
|
||||
CK_PR_FAS_S(16, uint16_t, "xchgw")
|
||||
CK_PR_FAS_S(8, uint8_t, "xchgb")
|
||||
|
||||
#undef CK_PR_FAS_S
|
||||
#undef CK_PR_FAS
|
||||
|
||||
/*
|
||||
* Atomic load-from-memory operations.
|
||||
*/
|
||||
#define CK_PR_LOAD(S, M, T, C, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_md_load_##S(const M *target) \
|
||||
{ \
|
||||
T r; \
|
||||
__asm__ __volatile__(I " %1, %0" \
|
||||
: "=q" (r) \
|
||||
: "m" (*(const C *)target) \
|
||||
: "memory"); \
|
||||
return (r); \
|
||||
}
|
||||
|
||||
CK_PR_LOAD(ptr, void, void *, char, "movq")
|
||||
|
||||
#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I)
|
||||
|
||||
CK_PR_LOAD_S(char, char, "movb")
|
||||
CK_PR_LOAD_S(uint, unsigned int, "movl")
|
||||
CK_PR_LOAD_S(int, int, "movl")
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_LOAD_S(double, double, "movq")
|
||||
#endif
|
||||
CK_PR_LOAD_S(64, uint64_t, "movq")
|
||||
CK_PR_LOAD_S(32, uint32_t, "movl")
|
||||
CK_PR_LOAD_S(16, uint16_t, "movw")
|
||||
CK_PR_LOAD_S(8, uint8_t, "movb")
|
||||
|
||||
#undef CK_PR_LOAD_S
|
||||
#undef CK_PR_LOAD
|
||||
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_load_64_2(const uint64_t target[2], uint64_t v[2])
|
||||
{
|
||||
__asm__ __volatile__("movq %%rdx, %%rcx;"
|
||||
"movq %%rax, %%rbx;"
|
||||
CK_PR_LOCK_PREFIX "cmpxchg16b %2;"
|
||||
: "=a" (v[0]),
|
||||
"=d" (v[1])
|
||||
: "m" (*(const uint64_t *)target)
|
||||
: "rbx", "rcx", "memory", "cc");
|
||||
return;
|
||||
}
|
||||
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_load_ptr_2(const void *t, void *v)
|
||||
{
|
||||
ck_pr_load_64_2(CK_CPP_CAST(const uint64_t *, t),
|
||||
CK_CPP_CAST(uint64_t *, v));
|
||||
return;
|
||||
}
|
||||
|
||||
#define CK_PR_LOAD_2(S, W, T) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_load_##S##_##W(const T t[2], T v[2]) \
|
||||
{ \
|
||||
ck_pr_load_64_2((const uint64_t *)(const void *)t, \
|
||||
(uint64_t *)(void *)v); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_LOAD_2(char, 16, char)
|
||||
CK_PR_LOAD_2(int, 4, int)
|
||||
CK_PR_LOAD_2(uint, 4, unsigned int)
|
||||
CK_PR_LOAD_2(32, 4, uint32_t)
|
||||
CK_PR_LOAD_2(16, 8, uint16_t)
|
||||
CK_PR_LOAD_2(8, 16, uint8_t)
|
||||
|
||||
#undef CK_PR_LOAD_2
|
||||
|
||||
/*
|
||||
* Atomic store-to-memory operations.
|
||||
*/
|
||||
#define CK_PR_STORE_IMM(S, M, T, C, I, K) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I " %1, %0" \
|
||||
: "=m" (*(C *)target) \
|
||||
: K "q" (v) \
|
||||
: "memory"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
#define CK_PR_STORE(S, M, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_md_store_##S(M *target, T v) \
|
||||
{ \
|
||||
__asm__ __volatile__(I " %1, %0" \
|
||||
: "=m" (*(C *)target) \
|
||||
: "q" (v) \
|
||||
: "memory"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
CK_PR_STORE_IMM(ptr, void, const void *, char, "movq", CK_CC_IMM_U32)
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_STORE(double, double, double, double, "movq")
|
||||
#endif
|
||||
|
||||
#define CK_PR_STORE_S(S, T, I, K) CK_PR_STORE_IMM(S, T, T, T, I, K)
|
||||
|
||||
CK_PR_STORE_S(char, char, "movb", CK_CC_IMM_S32)
|
||||
CK_PR_STORE_S(int, int, "movl", CK_CC_IMM_S32)
|
||||
CK_PR_STORE_S(uint, unsigned int, "movl", CK_CC_IMM_U32)
|
||||
CK_PR_STORE_S(64, uint64_t, "movq", CK_CC_IMM_U32)
|
||||
CK_PR_STORE_S(32, uint32_t, "movl", CK_CC_IMM_U32)
|
||||
CK_PR_STORE_S(16, uint16_t, "movw", CK_CC_IMM_U32)
|
||||
CK_PR_STORE_S(8, uint8_t, "movb", CK_CC_IMM_U32)
|
||||
|
||||
#undef CK_PR_STORE_S
|
||||
#undef CK_PR_STORE_IMM
|
||||
#undef CK_PR_STORE
|
||||
|
||||
/*
|
||||
* Atomic fetch-and-add operations.
|
||||
*/
|
||||
#define CK_PR_FAA(S, M, T, C, I) \
|
||||
CK_CC_INLINE static T \
|
||||
ck_pr_faa_##S(M *target, T d) \
|
||||
{ \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \
|
||||
: "+m" (*(C *)target), \
|
||||
"+q" (d) \
|
||||
: \
|
||||
: "memory", "cc"); \
|
||||
return (d); \
|
||||
}
|
||||
|
||||
CK_PR_FAA(ptr, void, uintptr_t, char, "xaddq")
|
||||
|
||||
#define CK_PR_FAA_S(S, T, I) CK_PR_FAA(S, T, T, T, I)
|
||||
|
||||
CK_PR_FAA_S(char, char, "xaddb")
|
||||
CK_PR_FAA_S(uint, unsigned int, "xaddl")
|
||||
CK_PR_FAA_S(int, int, "xaddl")
|
||||
CK_PR_FAA_S(64, uint64_t, "xaddq")
|
||||
CK_PR_FAA_S(32, uint32_t, "xaddl")
|
||||
CK_PR_FAA_S(16, uint16_t, "xaddw")
|
||||
CK_PR_FAA_S(8, uint8_t, "xaddb")
|
||||
|
||||
#undef CK_PR_FAA_S
|
||||
#undef CK_PR_FAA
|
||||
|
||||
/*
|
||||
* Atomic store-only unary operations.
|
||||
*/
|
||||
#define CK_PR_UNARY(K, S, T, C, I) \
|
||||
CK_PR_UNARY_R(K, S, T, C, I) \
|
||||
CK_PR_UNARY_V(K, S, T, C, I)
|
||||
|
||||
#define CK_PR_UNARY_R(K, S, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##K##_##S(T *target) \
|
||||
{ \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0" \
|
||||
: "+m" (*(C *)target) \
|
||||
: \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
#define CK_PR_UNARY_V(K, S, T, C, I) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##K##_##S##_zero(T *target, bool *r) \
|
||||
{ \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \
|
||||
: "+m" (*(C *)target), \
|
||||
"=m" (*r) \
|
||||
: \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
|
||||
#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I)
|
||||
|
||||
#define CK_PR_GENERATE(K) \
|
||||
CK_PR_UNARY(K, ptr, void, char, #K "q") \
|
||||
CK_PR_UNARY_S(K, char, char, #K "b") \
|
||||
CK_PR_UNARY_S(K, int, int, #K "l") \
|
||||
CK_PR_UNARY_S(K, uint, unsigned int, #K "l") \
|
||||
CK_PR_UNARY_S(K, 64, uint64_t, #K "q") \
|
||||
CK_PR_UNARY_S(K, 32, uint32_t, #K "l") \
|
||||
CK_PR_UNARY_S(K, 16, uint16_t, #K "w") \
|
||||
CK_PR_UNARY_S(K, 8, uint8_t, #K "b")
|
||||
|
||||
CK_PR_GENERATE(inc)
|
||||
CK_PR_GENERATE(dec)
|
||||
CK_PR_GENERATE(neg)
|
||||
|
||||
/* not does not affect condition flags. */
|
||||
#undef CK_PR_UNARY_V
|
||||
#define CK_PR_UNARY_V(a, b, c, d, e)
|
||||
CK_PR_GENERATE(not)
|
||||
|
||||
#undef CK_PR_GENERATE
|
||||
#undef CK_PR_UNARY_S
|
||||
#undef CK_PR_UNARY_V
|
||||
#undef CK_PR_UNARY_R
|
||||
#undef CK_PR_UNARY
|
||||
|
||||
/*
|
||||
* Atomic store-only binary operations.
|
||||
*/
|
||||
#define CK_PR_BINARY(K, S, M, T, C, I, O) \
|
||||
CK_CC_INLINE static void \
|
||||
ck_pr_##K##_##S(M *target, T d) \
|
||||
{ \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \
|
||||
: "+m" (*(C *)target) \
|
||||
: O "q" (d) \
|
||||
: "memory", "cc"); \
|
||||
return; \
|
||||
}
|
||||
|
||||
#define CK_PR_BINARY_S(K, S, T, I, O) CK_PR_BINARY(K, S, T, T, T, I, O)
|
||||
|
||||
#define CK_PR_GENERATE(K) \
|
||||
CK_PR_BINARY(K, ptr, void, uintptr_t, char, #K "q", CK_CC_IMM_U32) \
|
||||
CK_PR_BINARY_S(K, char, char, #K "b", CK_CC_IMM_S32) \
|
||||
CK_PR_BINARY_S(K, int, int, #K "l", CK_CC_IMM_S32) \
|
||||
CK_PR_BINARY_S(K, uint, unsigned int, #K "l", CK_CC_IMM_U32) \
|
||||
CK_PR_BINARY_S(K, 64, uint64_t, #K "q", CK_CC_IMM_U32) \
|
||||
CK_PR_BINARY_S(K, 32, uint32_t, #K "l", CK_CC_IMM_U32) \
|
||||
CK_PR_BINARY_S(K, 16, uint16_t, #K "w", CK_CC_IMM_U32) \
|
||||
CK_PR_BINARY_S(K, 8, uint8_t, #K "b", CK_CC_IMM_U32)
|
||||
|
||||
CK_PR_GENERATE(add)
|
||||
CK_PR_GENERATE(sub)
|
||||
CK_PR_GENERATE(and)
|
||||
CK_PR_GENERATE(or)
|
||||
CK_PR_GENERATE(xor)
|
||||
|
||||
#undef CK_PR_GENERATE
|
||||
#undef CK_PR_BINARY_S
|
||||
#undef CK_PR_BINARY
|
||||
|
||||
/*
|
||||
* Atomic compare and swap.
|
||||
*/
|
||||
#define CK_PR_CAS(S, M, T, C, I) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##S(M *target, T compare, T set) \
|
||||
{ \
|
||||
bool z; \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %2, %0; setz %1" \
|
||||
: "+m" (*(C *)target), \
|
||||
"=a" (z) \
|
||||
: "q" (set), \
|
||||
"a" (compare) \
|
||||
: "memory", "cc"); \
|
||||
return z; \
|
||||
}
|
||||
|
||||
CK_PR_CAS(ptr, void, void *, char, "cmpxchgq")
|
||||
|
||||
#define CK_PR_CAS_S(S, T, I) CK_PR_CAS(S, T, T, T, I)
|
||||
|
||||
CK_PR_CAS_S(char, char, "cmpxchgb")
|
||||
CK_PR_CAS_S(int, int, "cmpxchgl")
|
||||
CK_PR_CAS_S(uint, unsigned int, "cmpxchgl")
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_CAS_S(double, double, "cmpxchgq")
|
||||
#endif
|
||||
CK_PR_CAS_S(64, uint64_t, "cmpxchgq")
|
||||
CK_PR_CAS_S(32, uint32_t, "cmpxchgl")
|
||||
CK_PR_CAS_S(16, uint16_t, "cmpxchgw")
|
||||
CK_PR_CAS_S(8, uint8_t, "cmpxchgb")
|
||||
|
||||
#undef CK_PR_CAS_S
|
||||
#undef CK_PR_CAS
|
||||
|
||||
/*
|
||||
* Compare and swap, set *v to old value of target.
|
||||
*/
|
||||
#define CK_PR_CAS_O(S, M, T, C, I, R) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \
|
||||
{ \
|
||||
bool z; \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \
|
||||
"mov %% " R ", %2;" \
|
||||
"setz %1;" \
|
||||
: "+m" (*(C *)target), \
|
||||
"=a" (z), \
|
||||
"=m" (*(C *)v) \
|
||||
: "q" (set), \
|
||||
"a" (compare) \
|
||||
: "memory", "cc"); \
|
||||
return z; \
|
||||
}
|
||||
|
||||
CK_PR_CAS_O(ptr, void, void *, char, "q", "rax")
|
||||
|
||||
#define CK_PR_CAS_O_S(S, T, I, R) \
|
||||
CK_PR_CAS_O(S, T, T, T, I, R)
|
||||
|
||||
CK_PR_CAS_O_S(char, char, "b", "al")
|
||||
CK_PR_CAS_O_S(int, int, "l", "eax")
|
||||
CK_PR_CAS_O_S(uint, unsigned int, "l", "eax")
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_CAS_O_S(double, double, "q", "rax")
|
||||
#endif
|
||||
CK_PR_CAS_O_S(64, uint64_t, "q", "rax")
|
||||
CK_PR_CAS_O_S(32, uint32_t, "l", "eax")
|
||||
CK_PR_CAS_O_S(16, uint16_t, "w", "ax")
|
||||
CK_PR_CAS_O_S(8, uint8_t, "b", "al")
|
||||
|
||||
#undef CK_PR_CAS_O_S
|
||||
#undef CK_PR_CAS_O
|
||||
|
||||
/*
|
||||
* Contrary to C-interface, alignment requirements are that of uint64_t[2].
|
||||
*/
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2])
|
||||
{
|
||||
bool z;
|
||||
|
||||
__asm__ __volatile__("movq 0(%4), %%rax;"
|
||||
"movq 8(%4), %%rdx;"
|
||||
CK_PR_LOCK_PREFIX "cmpxchg16b %0; setz %1"
|
||||
: "+m" (*target),
|
||||
"=q" (z)
|
||||
: "b" (set[0]),
|
||||
"c" (set[1]),
|
||||
"q" (compare)
|
||||
: "memory", "cc", "%rax", "%rdx");
|
||||
return z;
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_2(void *t, void *c, void *s)
|
||||
{
|
||||
return ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, t),
|
||||
CK_CPP_CAST(uint64_t *, c),
|
||||
CK_CPP_CAST(uint64_t *, s));
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_64_2_value(uint64_t target[2],
|
||||
uint64_t compare[2],
|
||||
uint64_t set[2],
|
||||
uint64_t v[2])
|
||||
{
|
||||
bool z;
|
||||
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg16b %0;"
|
||||
"setz %3"
|
||||
: "+m" (*target),
|
||||
"=a" (v[0]),
|
||||
"=d" (v[1]),
|
||||
"=q" (z)
|
||||
: "a" (compare[0]),
|
||||
"d" (compare[1]),
|
||||
"b" (set[0]),
|
||||
"c" (set[1])
|
||||
: "memory", "cc");
|
||||
return z;
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_cas_ptr_2_value(void *t, void *c, void *s, void *v)
|
||||
{
|
||||
return ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *,t),
|
||||
CK_CPP_CAST(uint64_t *,c),
|
||||
CK_CPP_CAST(uint64_t *,s),
|
||||
CK_CPP_CAST(uint64_t *,v));
|
||||
}
|
||||
|
||||
#define CK_PR_CAS_V(S, W, T) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##S##_##W(T t[W], T c[W], T s[W]) \
|
||||
{ \
|
||||
return ck_pr_cas_64_2((uint64_t *)(void *)t, \
|
||||
(uint64_t *)(void *)c, \
|
||||
(uint64_t *)(void *)s); \
|
||||
} \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_cas_##S##_##W##_value(T *t, T c[W], T s[W], T *v) \
|
||||
{ \
|
||||
return ck_pr_cas_64_2_value((uint64_t *)(void *)t, \
|
||||
(uint64_t *)(void *)c, \
|
||||
(uint64_t *)(void *)s, \
|
||||
(uint64_t *)(void *)v); \
|
||||
}
|
||||
|
||||
#ifndef CK_PR_DISABLE_DOUBLE
|
||||
CK_PR_CAS_V(double, 2, double)
|
||||
#endif
|
||||
CK_PR_CAS_V(char, 16, char)
|
||||
CK_PR_CAS_V(int, 4, int)
|
||||
CK_PR_CAS_V(uint, 4, unsigned int)
|
||||
CK_PR_CAS_V(32, 4, uint32_t)
|
||||
CK_PR_CAS_V(16, 8, uint16_t)
|
||||
CK_PR_CAS_V(8, 16, uint8_t)
|
||||
|
||||
#undef CK_PR_CAS_V
|
||||
|
||||
/*
|
||||
* Atomic bit test operations.
|
||||
*/
|
||||
#define CK_PR_BT(K, S, T, P, C, I) \
|
||||
CK_CC_INLINE static bool \
|
||||
ck_pr_##K##_##S(T *target, unsigned int b) \
|
||||
{ \
|
||||
bool c; \
|
||||
__asm__ __volatile__(CK_PR_LOCK_PREFIX I "; setc %1" \
|
||||
: "+m" (*(C *)target), \
|
||||
"=q" (c) \
|
||||
: "q" ((P)b) \
|
||||
: "memory", "cc"); \
|
||||
return c; \
|
||||
}
|
||||
|
||||
#define CK_PR_BT_S(K, S, T, I) CK_PR_BT(K, S, T, T, T, I)
|
||||
|
||||
#define CK_PR_GENERATE(K) \
|
||||
CK_PR_BT(K, ptr, void, uint64_t, char, #K "q %2, %0") \
|
||||
CK_PR_BT_S(K, uint, unsigned int, #K "l %2, %0") \
|
||||
CK_PR_BT_S(K, int, int, #K "l %2, %0") \
|
||||
CK_PR_BT_S(K, 64, uint64_t, #K "q %2, %0") \
|
||||
CK_PR_BT_S(K, 32, uint32_t, #K "l %2, %0") \
|
||||
CK_PR_BT_S(K, 16, uint16_t, #K "w %w2, %0")
|
||||
|
||||
CK_PR_GENERATE(btc)
|
||||
CK_PR_GENERATE(bts)
|
||||
CK_PR_GENERATE(btr)
|
||||
|
||||
#undef CK_PR_GENERATE
|
||||
#undef CK_PR_BT
|
||||
|
||||
#endif /* CK_PR_X86_64_H */
|
||||
|
||||
109
include/gcc/x86_64/ck_pr_rtm.h
Normal file
109
include/gcc/x86_64/ck_pr_rtm.h
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
/*
|
||||
* Copyright 2013-2015 Samy Al Bahra.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (c) 2012,2013 Intel Corporation
|
||||
* Author: Andi Kleen
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that: (1) source code distributions
|
||||
* retain the above copyright notice and this paragraph in its entirety, (2)
|
||||
* distributions including binary code include the above copyright notice and
|
||||
* this paragraph in its entirety in the documentation or other materials
|
||||
* provided with the distribution
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
|
||||
* WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
|
||||
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*/
|
||||
|
||||
#ifndef CK_PR_X86_64_RTM_H
|
||||
#define CK_PR_X86_64_RTM_H
|
||||
|
||||
#ifndef CK_PR_X86_64_H
|
||||
#error Do not include this file directly, use ck_pr.h
|
||||
#endif
|
||||
|
||||
#define CK_F_PR_RTM
|
||||
|
||||
#include <ck_cc.h>
|
||||
#include <ck_stdbool.h>
|
||||
|
||||
#define CK_PR_RTM_STARTED (~0U)
|
||||
#define CK_PR_RTM_EXPLICIT (1 << 0)
|
||||
#define CK_PR_RTM_RETRY (1 << 1)
|
||||
#define CK_PR_RTM_CONFLICT (1 << 2)
|
||||
#define CK_PR_RTM_CAPACITY (1 << 3)
|
||||
#define CK_PR_RTM_DEBUG (1 << 4)
|
||||
#define CK_PR_RTM_NESTED (1 << 5)
|
||||
#define CK_PR_RTM_CODE(x) (((x) >> 24) & 0xFF)
|
||||
|
||||
CK_CC_INLINE static unsigned int
|
||||
ck_pr_rtm_begin(void)
|
||||
{
|
||||
unsigned int r = CK_PR_RTM_STARTED;
|
||||
|
||||
__asm__ __volatile__(".byte 0xc7,0xf8;"
|
||||
".long 0;"
|
||||
: "+a" (r)
|
||||
:
|
||||
: "memory");
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_rtm_end(void)
|
||||
{
|
||||
|
||||
__asm__ __volatile__(".byte 0x0f,0x01,0xd5" ::: "memory");
|
||||
return;
|
||||
}
|
||||
|
||||
CK_CC_INLINE static void
|
||||
ck_pr_rtm_abort(const unsigned int status)
|
||||
{
|
||||
|
||||
__asm__ __volatile__(".byte 0xc6,0xf8,%P0" :: "i" (status) : "memory");
|
||||
return;
|
||||
}
|
||||
|
||||
CK_CC_INLINE static bool
|
||||
ck_pr_rtm_test(void)
|
||||
{
|
||||
bool r;
|
||||
|
||||
__asm__ __volatile__(".byte 0x0f,0x01,0xd6;"
|
||||
"setnz %0"
|
||||
: "=r" (r)
|
||||
:
|
||||
: "memory");
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
#endif /* CK_PR_X86_64_RTM_H */
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue