Kokkos Core Kernels Package Version of the Day
Loading...
Searching...
No Matches
Kokkos_Atomics_Desul_Wrapper.hpp
1//@HEADER
2// ************************************************************************
3//
4// Kokkos v. 4.0
5// Copyright (2022) National Technology & Engineering
6// Solutions of Sandia, LLC (NTESS).
7//
8// Under the terms of Contract DE-NA0003525 with NTESS,
9// the U.S. Government retains certain rights in this software.
10//
11// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12// See https://kokkos.org/LICENSE for license information.
13// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14//
15//@HEADER
16
17#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18#include <Kokkos_Macros.hpp>
19static_assert(false,
20 "Including non-public Kokkos header files is not allowed.");
21#endif
22#ifndef KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
23#define KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
24#include <Kokkos_Macros.hpp>
25
26#ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS
27#include <Kokkos_Atomics_Desul_Config.hpp>
28#include <desul/atomics.hpp>
29
30#include <impl/Kokkos_Atomic_Memory_Order.hpp>
31#include <impl/Kokkos_Volatile_Load.hpp>
32
33// clang-format off
34namespace Kokkos {
35
36// FIXME: These functions don't have any use/test in unit tests ...
37// ==========================================================
38inline const char* atomic_query_version() { return "KOKKOS_DESUL_ATOMICS"; }
39
40#if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
41 !defined(__CUDA_ARCH__)
42
43#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
44#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
45
46#else
47
48#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
49#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
50
51#endif
52// ============================================================
53
54#ifdef KOKKOS_INTERNAL_NOT_PARALLEL
55#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
56#else
57#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()
58#endif
59
60template<class T> KOKKOS_INLINE_FUNCTION
61T atomic_load(T* const dest) { return desul::atomic_load(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
62
63template<class T> KOKKOS_INLINE_FUNCTION
64void atomic_store(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_store(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
65
66template<class T> KOKKOS_INLINE_FUNCTION
67void atomic_assign(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { atomic_store(dest,val); }
68
69KOKKOS_INLINE_FUNCTION
70void memory_fence() {
71 desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE);
72}
73
74KOKKOS_INLINE_FUNCTION
75void load_fence() { return desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); }
76
77KOKKOS_INLINE_FUNCTION
78void store_fence() { return desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); }
79
80// atomic_fetch_op
81template<class T> KOKKOS_INLINE_FUNCTION
82T atomic_fetch_add (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
83
84template<class T> KOKKOS_INLINE_FUNCTION
85T atomic_fetch_sub (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
86
87template<class T> KOKKOS_INLINE_FUNCTION
88T atomic_fetch_max (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
89
90template<class T> KOKKOS_INLINE_FUNCTION
91T atomic_fetch_min (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
92
93template<class T> KOKKOS_INLINE_FUNCTION
94T atomic_fetch_mul (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
95
96template<class T> KOKKOS_INLINE_FUNCTION
97T atomic_fetch_div (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
98
99template<class T> KOKKOS_INLINE_FUNCTION
100T atomic_fetch_mod (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_mod (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
101
102template<class T> KOKKOS_INLINE_FUNCTION
103T atomic_fetch_and (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
104
105template<class T> KOKKOS_INLINE_FUNCTION
106T atomic_fetch_or (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
107
108template<class T> KOKKOS_INLINE_FUNCTION
109T atomic_fetch_xor (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_xor (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
110
111template<class T> KOKKOS_INLINE_FUNCTION
112T atomic_fetch_nand(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_nand(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
113
114template<class T> KOKKOS_INLINE_FUNCTION
115T atomic_fetch_lshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_lshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
116
117template<class T> KOKKOS_INLINE_FUNCTION
118T atomic_fetch_rshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_rshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
119
120template<class T> KOKKOS_INLINE_FUNCTION
121T atomic_fetch_inc(T* const dest) { return desul::atomic_fetch_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
122
123template<class T> KOKKOS_INLINE_FUNCTION
124T atomic_fetch_dec(T* const dest) { return desul::atomic_fetch_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
125
126
127// atomic_op_fetch
128template<class T> KOKKOS_INLINE_FUNCTION
129T atomic_add_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_add_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
130
131template<class T> KOKKOS_INLINE_FUNCTION
132T atomic_sub_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_sub_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
133
134template<class T> KOKKOS_INLINE_FUNCTION
135T atomic_max_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_max_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
136
137template<class T> KOKKOS_INLINE_FUNCTION
138T atomic_min_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_min_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
139
140template<class T> KOKKOS_INLINE_FUNCTION
141T atomic_mul_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mul_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
142
143template<class T> KOKKOS_INLINE_FUNCTION
144T atomic_div_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_div_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
145
146template<class T> KOKKOS_INLINE_FUNCTION
147T atomic_mod_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mod_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
148
149template<class T> KOKKOS_INLINE_FUNCTION
150T atomic_and_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_and_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
151
152template<class T> KOKKOS_INLINE_FUNCTION
153T atomic_or_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_or_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
154
155template<class T> KOKKOS_INLINE_FUNCTION
156T atomic_xor_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_xor_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
157
158template<class T> KOKKOS_INLINE_FUNCTION
159T atomic_nand_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_nand_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
160
161template<class T> KOKKOS_INLINE_FUNCTION
162T atomic_lshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_lshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
163
164template<class T> KOKKOS_INLINE_FUNCTION
165T atomic_rshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_rshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
166
167template<class T> KOKKOS_INLINE_FUNCTION
168T atomic_inc_fetch(T* const dest) { return desul::atomic_inc_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
169
170template<class T> KOKKOS_INLINE_FUNCTION
171T atomic_dec_fetch(T* const dest) { return desul::atomic_dec_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
172
173
174// atomic_op
175template<class T> KOKKOS_INLINE_FUNCTION
176void atomic_add(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
177
178template<class T> KOKKOS_INLINE_FUNCTION
179void atomic_sub(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
180
181template<class T> KOKKOS_INLINE_FUNCTION
182void atomic_mul(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
183
184template<class T> KOKKOS_INLINE_FUNCTION
185void atomic_div(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
186
187template<class T> KOKKOS_INLINE_FUNCTION
188void atomic_min(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
189
190template<class T> KOKKOS_INLINE_FUNCTION
191void atomic_max(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
192
193// FIXME: Desul doesn't have atomic_and yet so call fetch_and
194template<class T> KOKKOS_INLINE_FUNCTION
195void atomic_and(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
196
197// FIXME: Desul doesn't have atomic_or yet so call fetch_or
198template<class T> KOKKOS_INLINE_FUNCTION
199void atomic_or(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
200
201template<class T> KOKKOS_INLINE_FUNCTION
202void atomic_inc(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
203
204template<class T> KOKKOS_INLINE_FUNCTION
205void atomic_dec(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
206
207template<class T> KOKKOS_INLINE_FUNCTION
208void atomic_increment(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
209
210template<class T> KOKKOS_INLINE_FUNCTION
211void atomic_decrement(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
212
213// Exchange
214
215template<class T> KOKKOS_INLINE_FUNCTION
216T atomic_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_exchange(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
217
218template<class T> KOKKOS_INLINE_FUNCTION
219bool atomic_compare_exchange_strong(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> expected, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
220 T expected_ref = expected;
221 return desul::atomic_compare_exchange_strong(dest, expected_ref, desired,
222 desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
223}
224
225template<class T> KOKKOS_INLINE_FUNCTION
226T atomic_compare_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> compare, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
227 return desul::atomic_compare_exchange(dest, compare, desired,
228 desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
229}
230
231namespace Impl {
232
233 template<class MemoryOrder>
234 struct KokkosToDesulMemoryOrder;
235
236 template<>
237 struct KokkosToDesulMemoryOrder<memory_order_seq_cst_t> {
238 using type = desul::MemoryOrderSeqCst;
239 };
240 template<>
241 struct KokkosToDesulMemoryOrder<memory_order_acquire_t> {
242 using type = desul::MemoryOrderAcquire;
243 };
244 template<>
245 struct KokkosToDesulMemoryOrder<memory_order_release_t> {
246 using type = desul::MemoryOrderRelease;
247 };
248 template<>
249 struct KokkosToDesulMemoryOrder<memory_order_acq_rel_t> {
250 using type = desul::MemoryOrderAcqRel;
251 };
252 template<>
253 struct KokkosToDesulMemoryOrder<memory_order_relaxed_t> {
254 using type = desul::MemoryOrderRelaxed;
255 };
256 template<class T, class MemOrderSuccess, class MemOrderFailure> KOKKOS_INLINE_FUNCTION
257 bool atomic_compare_exchange_strong(T* const dest, T& expected, const T desired, MemOrderSuccess, MemOrderFailure) {
258 return desul::atomic_compare_exchange_strong(dest, expected, desired,
259 typename KokkosToDesulMemoryOrder<MemOrderSuccess>::type(),
260 typename KokkosToDesulMemoryOrder<MemOrderFailure>::type(),
261 KOKKOS_DESUL_MEM_SCOPE);
262
263 }
264 template<class T, class MemoryOrder>
265 KOKKOS_INLINE_FUNCTION
266 T atomic_load(const T* const src, MemoryOrder) {
267 return desul::atomic_load(src, typename KokkosToDesulMemoryOrder<MemoryOrder>::type(), KOKKOS_DESUL_MEM_SCOPE);
268 }
269 template<class T, class MemoryOrder>
270 KOKKOS_INLINE_FUNCTION
271 void atomic_store(T* const src, const T val, MemoryOrder) {
272 return desul::atomic_store(src, val, typename KokkosToDesulMemoryOrder<MemoryOrder>::type(), KOKKOS_DESUL_MEM_SCOPE);
273 }
274}
275
276}
277
278#undef KOKKOS_DESUL_MEM_SCOPE
279
280// clang-format on
281#endif // KOKKOS_ENABLE_IMPL_DESUL_ATOMICS
282#endif