Skip to content

Commit 2802f3b

Browse files
zhangyu-duckzhangyu-duck
committed
libc: realize atomic64 via a spinlock helper
libc/machine/arch_atomic64.c implementing atomic_*_8 on a single spinlock. All helpers are weak_function so an arch with native 64-bit support overrides at link time Signed-off-by: zhangyu117 <zhangyu117@xiaomi.com>
1 parent 5c36cc3 commit 2802f3b

5 files changed

Lines changed: 296 additions & 12 deletions

File tree

‎include/nuttx/lib/arch_atomic.h‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -255,20 +255,19 @@ static inline void atomic_unlock(irqstate_t flags)
255255
#define ARCH_HAVE_ATOMIC_1
256256
#define ARCH_HAVE_ATOMIC_2
257257
#define ARCH_HAVE_ATOMIC_4
258-
#define ARCH_HAVE_ATOMIC_8
259258

260259
/****************************************************************************
261260
* Inline Functions
262261
****************************************************************************/
263262

263+
#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN
264264
ARCH_ATOMIC_DEFINE(atomic, uint8_t, 1)
265265
ARCH_ATOMIC_DEFINE(atomic, uint16_t, 2)
266266
ARCH_ATOMIC_DEFINE(atomic, int32_t, 4)
267-
ARCH_ATOMIC_DEFINE(atomic, int64_t, 8)
268267

269268
ARCH_SYNC_DEFINE(sync, uint8_t, 1)
270269
ARCH_SYNC_DEFINE(sync, uint16_t, 2)
271270
ARCH_SYNC_DEFINE(sync, uint32_t, 4)
272-
ARCH_SYNC_DEFINE(sync, uint64_t, 8)
271+
#endif
273272

274273
#endif /* __INCLUDE_NUTTX_LIB_ARCH_ATOMIC_H */

‎libs/libc/machine/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ if(NOT CONFIG_LIBC_ATOMIC_TOOLCHAIN)
2626
target_sources(c PRIVATE arch_atomic.c)
2727
endif()
2828

29+
target_sources(c PRIVATE arch_atomic64.c)
30+
2931
if(CONFIG_MM_KASAN)
3032
target_sources(c PRIVATE arch_libc.c)
3133
endif()

‎libs/libc/machine/Make.defs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ ifneq ($(CONFIG_LIBC_ATOMIC_TOOLCHAIN),y)
2424
CSRCS += arch_atomic.c
2525
endif
2626

27+
CSRCS += arch_atomic64.c
28+
2729
ifeq ($(CONFIG_MM_KASAN),y)
2830
CSRCS += arch_libc.c
2931
endif

‎libs/libc/machine/arch_atomic.c‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626

2727
#include <nuttx/config.h>
2828

29-
#define ARCH_ATOMIC_SPECIFIER
29+
#define ARCH_ATOMIC_SPECIFIER weak_function
3030

3131
#include <nuttx/atomic.h>
3232
#include <nuttx/arch.h>
@@ -36,7 +36,7 @@
3636
****************************************************************************/
3737

3838
/****************************************************************************
39-
* Name: __atomic_*_{1,2,4,8}
39+
* Name: __atomic_*_{1,2,4}
4040
****************************************************************************/
4141

4242
#ifdef ARCH_HAVE_ATOMIC_1
@@ -48,9 +48,6 @@ ARCH_ATOMIC_DEFINE(__atomic, uint16_t, 2)
4848
#ifdef ARCH_HAVE_ATOMIC_4
4949
ARCH_ATOMIC_DEFINE(__atomic, uint32_t, 4)
5050
#endif
51-
#ifdef ARCH_HAVE_ATOMIC_8
52-
ARCH_ATOMIC_DEFINE(__atomic, uint64_t, 8)
53-
#endif
5451

5552
/* Clang define the __sync builtins, add #ifndef to avoid
5653
* redefined/redeclared problem.
@@ -59,7 +56,7 @@ ARCH_ATOMIC_DEFINE(__atomic, uint64_t, 8)
5956
#ifndef __clang__
6057

6158
/****************************************************************************
62-
* Name: __sync_*_{1,2,4,8}
59+
* Name: __sync_*_{1,2,4}
6360
****************************************************************************/
6461

6562
#ifdef ARCH_SYNC_DEFINE
@@ -72,9 +69,6 @@ ARCH_SYNC_DEFINE(__sync, uint16_t, 2)
7269
# ifdef ARCH_HAVE_ATOMIC_4
7370
ARCH_SYNC_DEFINE(__sync, uint32_t, 4)
7471
# endif
75-
# ifdef ARCH_HAVE_ATOMIC_8
76-
ARCH_SYNC_DEFINE(__sync, uint64_t, 8)
77-
# endif
7872
#endif
7973

8074
/****************************************************************************

‎libs/libc/machine/arch_atomic64.c‎

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
/****************************************************************************
2+
* libs/libc/machine/arch_atomic64.c
3+
*
4+
* SPDX-License-Identifier: Apache-2.0
5+
*
6+
* Licensed to the Apache Software Foundation (ASF) under one or more
7+
* contributor license agreements. See the NOTICE file distributed with
8+
* this work for additional information regarding copyright ownership. The
9+
* ASF licenses this file to you under the Apache License, Version 2.0 (the
10+
* "License"); you may not use this file except in compliance with the
11+
* License. You may obtain a copy of the License at
12+
*
13+
* http://www.apache.org/licenses/LICENSE-2.0
14+
*
15+
* Unless required by applicable law or agreed to in writing, software
16+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
17+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
18+
* License for the specific language governing permissions and limitations
19+
* under the License.
20+
*
21+
****************************************************************************/
22+
23+
/* 8 byte atomics are not lock free on every target. An arch may only have a
24+
* 32 bit atomic instruction (TriCore swap.w/cmpswap.w for instance) and a
25+
* toolchain without 64 bit support emits calls to the __atomic_*_8 helpers
26+
* that would otherwise come from libatomic, which NuttX does not link.
27+
*
28+
* The helpers are implemented here on top of a single spinlock. A spinlock
29+
* rather than a plain up_irq_save() is needed because disabling interrupts
30+
* only excludes the local CPU: on SMP another CPU could still enter the same
31+
* critical section and corrupt the 64 bit value. The interrupt state is
32+
* still saved (spin_lock_irqsave) so that an ISR on this CPU cannot deadlock
33+
* against a holder it interrupted.
34+
*
35+
* <arch/atomic.h> is deliberately not reused: its macros are built around
36+
* the native word size and a 64 bit access would be silently truncated. All
37+
* symbols are weak, so a toolchain or arch with a native 64 bit
38+
* implementation still wins at link time.
39+
*/
40+
41+
/****************************************************************************
42+
* Included Files
43+
****************************************************************************/
44+
45+
#include <nuttx/config.h>
46+
47+
#include <nuttx/compiler.h>
48+
#include <nuttx/spinlock.h>
49+
50+
#include <stdbool.h>
51+
#include <stdint.h>
52+
53+
/****************************************************************************
54+
* Private Data
55+
****************************************************************************/
56+
57+
/* Every 64 bit atomic serializes on this lock. The granularity is coarse,
58+
* but 64 bit atomics are rare enough that a single lock is not a bottleneck.
59+
*/
60+
61+
static spinlock_t g_atomic64_lock = SP_UNLOCKED;
62+
63+
/****************************************************************************
64+
* Private Functions
65+
****************************************************************************/
66+
67+
static inline irqstate_t atomic64_lock(void)
68+
{
69+
return spin_lock_irqsave(&g_atomic64_lock);
70+
}
71+
72+
static inline void atomic64_unlock(irqstate_t flags)
73+
{
74+
spin_unlock_irqrestore(&g_atomic64_lock, flags);
75+
}
76+
77+
/****************************************************************************
78+
* Pre-processor Definitions
79+
****************************************************************************/
80+
81+
#define ATOMIC64_STORE(func, t) \
82+
weak_function \
83+
void func(FAR volatile void *ptr, t value, int memorder) \
84+
{ \
85+
irqstate_t irqstate = atomic64_lock(); \
86+
\
87+
*(FAR t *)ptr = value; \
88+
\
89+
atomic64_unlock(irqstate); \
90+
}
91+
92+
#define ATOMIC64_LOAD(func, t) \
93+
weak_function \
94+
t func(FAR const volatile void *ptr, int memorder) \
95+
{ \
96+
irqstate_t irqstate = atomic64_lock(); \
97+
\
98+
t ret = *(FAR t *)ptr; \
99+
\
100+
atomic64_unlock(irqstate); \
101+
return ret; \
102+
}
103+
104+
#define ATOMIC64_EXCHANGE(func, t) \
105+
weak_function \
106+
t func(FAR volatile void *ptr, t value, int memorder) \
107+
{ \
108+
irqstate_t irqstate = atomic64_lock(); \
109+
FAR t *tmp = (FAR t *)ptr; \
110+
\
111+
t ret = *tmp; \
112+
*tmp = value; \
113+
\
114+
atomic64_unlock(irqstate); \
115+
return ret; \
116+
}
117+
118+
#define ATOMIC64_COMPARE_EXCHANGE(func, t) \
119+
weak_function \
120+
bool func(FAR volatile void *mem, FAR volatile void *expect, \
121+
t desired, bool weak, int success, int failure) \
122+
{ \
123+
bool ret = false; \
124+
irqstate_t irqstate = atomic64_lock(); \
125+
FAR t *tmpmem = (FAR t *)mem; \
126+
FAR t *tmpexp = (FAR t *)expect; \
127+
\
128+
if (*tmpmem == *tmpexp) \
129+
{ \
130+
ret = true; \
131+
*tmpmem = desired; \
132+
} \
133+
else \
134+
{ \
135+
*tmpexp = *tmpmem; \
136+
} \
137+
\
138+
atomic64_unlock(irqstate); \
139+
return ret; \
140+
}
141+
142+
#define ATOMIC64_FLAGS_TEST_AND_SET(func, t) \
143+
weak_function \
144+
t func(FAR volatile void *ptr, int memorder) \
145+
{ \
146+
irqstate_t irqstate = atomic64_lock(); \
147+
FAR t *tmp = (FAR t *)ptr; \
148+
t ret = *tmp; \
149+
\
150+
*tmp = 1; \
151+
\
152+
atomic64_unlock(irqstate); \
153+
return ret; \
154+
}
155+
156+
#define ATOMIC64_FETCH_OP(func, t, op) \
157+
weak_function \
158+
t func(FAR volatile void *ptr, t value, int memorder) \
159+
{ \
160+
irqstate_t irqstate = atomic64_lock(); \
161+
FAR t *tmp = (FAR t *)ptr; \
162+
t ret = *tmp; \
163+
\
164+
*tmp = *tmp op value; \
165+
\
166+
atomic64_unlock(irqstate); \
167+
return ret; \
168+
}
169+
170+
#define ATOMIC64_OP_FETCH(func, t, op) \
171+
weak_function \
172+
t func(FAR volatile void *ptr, t value) \
173+
{ \
174+
irqstate_t irqstate = atomic64_lock(); \
175+
FAR t *tmp = (FAR t *)ptr; \
176+
t ret; \
177+
\
178+
*tmp = *tmp op value; \
179+
ret = *tmp; \
180+
\
181+
atomic64_unlock(irqstate); \
182+
return ret; \
183+
}
184+
185+
#define ATOMIC64_NAND_FETCH(func, t) \
186+
weak_function \
187+
t func(FAR volatile void *ptr, t value) \
188+
{ \
189+
irqstate_t irqstate = atomic64_lock(); \
190+
FAR t *tmp = (FAR t *)ptr; \
191+
t ret; \
192+
\
193+
*tmp = ~(*tmp & value); \
194+
ret = *tmp; \
195+
\
196+
atomic64_unlock(irqstate); \
197+
return ret; \
198+
}
199+
200+
#define ATOMIC64_BOOL_CMP_SWAP(func, t) \
201+
weak_function \
202+
bool func(FAR volatile void *ptr, t oldvalue, t newvalue) \
203+
{ \
204+
bool ret = false; \
205+
irqstate_t irqstate = atomic64_lock(); \
206+
FAR t *tmp = (FAR t *)ptr; \
207+
\
208+
if (*tmp == oldvalue) \
209+
{ \
210+
ret = true; \
211+
*tmp = newvalue; \
212+
} \
213+
\
214+
atomic64_unlock(irqstate); \
215+
return ret; \
216+
}
217+
218+
#define ATOMIC64_VAL_CMP_SWAP(func, t) \
219+
weak_function \
220+
t func(FAR volatile void *ptr, t oldvalue, t newvalue) \
221+
{ \
222+
irqstate_t irqstate = atomic64_lock(); \
223+
FAR t *tmp = (FAR t *)ptr; \
224+
t ret = *tmp; \
225+
\
226+
if (*tmp == oldvalue) \
227+
{ \
228+
*tmp = newvalue; \
229+
} \
230+
\
231+
atomic64_unlock(irqstate); \
232+
return ret; \
233+
}
234+
235+
#define ATOMIC64_DEFINE(prefix, t, n) \
236+
ATOMIC64_STORE(prefix ## _store_ ## n, t) \
237+
ATOMIC64_LOAD(prefix ## _load_ ## n, t) \
238+
ATOMIC64_EXCHANGE(prefix ## _exchange_ ## n, t) \
239+
ATOMIC64_COMPARE_EXCHANGE(prefix ## _compare_exchange_ ## n, t) \
240+
ATOMIC64_FLAGS_TEST_AND_SET(prefix ## _flags_test_and_set_ ## n, t) \
241+
ATOMIC64_FETCH_OP(prefix ## _fetch_add_ ## n, t, +) \
242+
ATOMIC64_FETCH_OP(prefix ## _fetch_sub_ ## n, t, -) \
243+
ATOMIC64_FETCH_OP(prefix ## _fetch_and_ ## n, t, &) \
244+
ATOMIC64_FETCH_OP(prefix ## _fetch_or_ ## n, t, |) \
245+
ATOMIC64_FETCH_OP(prefix ## _fetch_xor_ ## n, t, ^)
246+
247+
#define SYNC64_DEFINE(prefix, t, n) \
248+
ATOMIC64_OP_FETCH(prefix ## _add_and_fetch_ ## n, t, +) \
249+
ATOMIC64_OP_FETCH(prefix ## _sub_and_fetch_ ## n, t, -) \
250+
ATOMIC64_OP_FETCH(prefix ## _or_and_fetch_ ## n, t, |) \
251+
ATOMIC64_OP_FETCH(prefix ## _and_and_fetch_ ## n, t, &) \
252+
ATOMIC64_OP_FETCH(prefix ## _xor_and_fetch_ ## n, t, ^) \
253+
ATOMIC64_NAND_FETCH(prefix ## _nand_and_fetch_ ## n, t) \
254+
ATOMIC64_BOOL_CMP_SWAP(prefix ## _bool_compare_and_swap_ ## n, t) \
255+
ATOMIC64_VAL_CMP_SWAP(prefix ## _val_compare_and_swap_ ## n, t)
256+
257+
/****************************************************************************
258+
* Public Functions
259+
****************************************************************************/
260+
261+
/****************************************************************************
262+
* Name: atomic_*_8 and __atomic_*_8
263+
****************************************************************************/
264+
265+
#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN
266+
ATOMIC64_DEFINE(atomic, int64_t, 8)
267+
#endif
268+
269+
ATOMIC64_DEFINE(__atomic, uint64_t, 8)
270+
271+
/* Clang define the __sync builtins, add #ifndef to avoid
272+
* redefined/redeclared problem.
273+
*/
274+
275+
#ifndef __clang__
276+
277+
/****************************************************************************
278+
* Name: sync_*_8 and __sync_*_8
279+
****************************************************************************/
280+
281+
#ifndef CONFIG_LIBC_ATOMIC_TOOLCHAIN
282+
SYNC64_DEFINE(sync, uint64_t, 8)
283+
#endif
284+
285+
SYNC64_DEFINE(__sync, uint64_t, 8)
286+
287+
#endif /* __clang__ */

0 commit comments

Comments
 (0)