1 From a8f2d0c75c9cc179fc9e7f7ca17ea3b3b3b5af41 Mon Sep 17 00:00:00 2001
2 From: Michael Jeanson <mjeanson@efficios.com>
3 Date: Tue, 19 Dec 2017 15:06:42 -0500
4 Subject: [PATCH] Fix: ACCESS_ONCE() removed in kernel 4.15
6 The ACCESS_ONCE() macro was removed in kernel 4.15 and should be
7 replaced by READ_ONCE and WRITE_ONCE which were introduced in kernel
10 This commit replaces all calls to ACCESS_ONCE() with the appropriate
11 READ_ONCE or WRITE_ONCE and adds compatibility macros for kernels that
14 See this upstream commit:
16 commit b03a0fe0c5e4b46dcd400d27395b124499554a71
17 Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
18 Date: Mon Oct 23 14:07:25 2017 -0700
20 locking/atomics, mm: Convert ACCESS_ONCE() to READ_ONCE()/WRITE_ONCE()
22 For several reasons, it is desirable to use {READ,WRITE}_ONCE() in
23 preference to ACCESS_ONCE(), and new code is expected to use one of the
24 former. So far, there's been no reason to change most existing uses of
25 ACCESS_ONCE(), as these aren't currently harmful.
27 However, for some features it is necessary to instrument reads and
28 writes separately, which is not possible with ACCESS_ONCE(). This
29 distinction is critical to correct operation.
31 It's possible to transform the bulk of kernel code using the Coccinelle
32 script below. However, this doesn't handle comments, leaving references
33 to ACCESS_ONCE() instances which have been removed. As a preparatory
34 step, this patch converts the mm code and comments to use
35 {READ,WRITE}_ONCE() consistently.
44 - ACCESS_ONCE(E1) = E2
55 Signed-off-by: Michael Jeanson <mjeanson@efficios.com>
56 Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
58 instrumentation/events/lttng-module/i2c.h | 4 ++--
59 lib/ringbuffer/backend.h | 2 +-
60 lib/ringbuffer/backend_internal.h | 7 ++++---
61 lib/ringbuffer/frontend.h | 2 +-
62 lib/ringbuffer/ring_buffer_frontend.c | 10 +++++-----
63 lib/ringbuffer/ring_buffer_iterator.c | 2 +-
64 lttng-clock.c | 6 +++---
65 lttng-events.c | 30 +++++++++++++++---------------
66 probes/lttng-ftrace.c | 12 ++++++------
67 probes/lttng-kprobes.c | 6 +++---
68 probes/lttng-kretprobes.c | 10 +++++-----
69 probes/lttng-tracepoint-event-impl.h | 12 ++++++------
70 wrapper/compiler.h | 13 +++++++++++++
71 wrapper/trace-clock.h | 11 ++++++-----
72 14 files changed, 71 insertions(+), 56 deletions(-)
74 diff --git a/instrumentation/events/lttng-module/i2c.h b/instrumentation/events/lttng-module/i2c.h
75 index 4088b60..dd91c9b 100644
76 --- a/instrumentation/events/lttng-module/i2c.h
77 +++ b/instrumentation/events/lttng-module/i2c.h
78 @@ -22,7 +22,7 @@ LTTNG_TRACEPOINT_EVENT_CODE(i2c_write,
81 tp_locvar->extract_sensitive_payload =
82 - ACCESS_ONCE(extract_sensitive_payload);
83 + READ_ONCE(extract_sensitive_payload);
87 @@ -77,7 +77,7 @@ LTTNG_TRACEPOINT_EVENT_CODE(i2c_reply,
90 tp_locvar->extract_sensitive_payload =
91 - ACCESS_ONCE(extract_sensitive_payload);
92 + READ_ONCE(extract_sensitive_payload);
96 diff --git a/lib/ringbuffer/backend.h b/lib/ringbuffer/backend.h
97 index 9db0095..0b75de8 100644
98 --- a/lib/ringbuffer/backend.h
99 +++ b/lib/ringbuffer/backend.h
100 @@ -169,7 +169,7 @@ size_t lib_ring_buffer_do_strcpy(const struct lib_ring_buffer_config *config,
101 * Only read source character once, in case it is
102 * modified concurrently.
104 - c = ACCESS_ONCE(src[count]);
105 + c = READ_ONCE(src[count]);
108 lib_ring_buffer_do_copy(config, &dest[count], &c, 1);
109 diff --git a/lib/ringbuffer/backend_internal.h b/lib/ringbuffer/backend_internal.h
110 index 2e59b68..dc69ecf 100644
111 --- a/lib/ringbuffer/backend_internal.h
112 +++ b/lib/ringbuffer/backend_internal.h
114 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
117 +#include <wrapper/compiler.h>
118 #include <wrapper/ringbuffer/config.h>
119 #include <wrapper/ringbuffer/backend_types.h>
120 #include <wrapper/ringbuffer/frontend_types.h>
121 @@ -171,7 +172,7 @@ void subbuffer_id_set_noref_offset(const struct lib_ring_buffer_config *config,
122 tmp |= offset << SB_ID_OFFSET_SHIFT;
123 tmp |= SB_ID_NOREF_MASK;
124 /* Volatile store, read concurrently by readers. */
125 - ACCESS_ONCE(*id) = tmp;
126 + WRITE_ONCE(*id, tmp);
130 @@ -379,7 +380,7 @@ void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config,
131 * Performing a volatile access to read the sb_pages, because we want to
132 * read a coherent version of the pointer and the associated noref flag.
134 - id = ACCESS_ONCE(bufb->buf_wsb[idx].id);
135 + id = READ_ONCE(bufb->buf_wsb[idx].id);
137 /* This check is called on the fast path for each record. */
138 if (likely(!subbuffer_id_is_noref(config, id))) {
139 @@ -448,7 +449,7 @@ int update_read_sb_index(const struct lib_ring_buffer_config *config,
140 if (config->mode == RING_BUFFER_OVERWRITE) {
142 * Exchange the target writer subbuffer with our own unused
143 - * subbuffer. No need to use ACCESS_ONCE() here to read the
144 + * subbuffer. No need to use READ_ONCE() here to read the
145 * old_wpage, because the value read will be confirmed by the
146 * following cmpxchg().
148 diff --git a/lib/ringbuffer/frontend.h b/lib/ringbuffer/frontend.h
149 index 909abc2..1450cb7 100644
150 --- a/lib/ringbuffer/frontend.h
151 +++ b/lib/ringbuffer/frontend.h
152 @@ -168,7 +168,7 @@ static inline
153 int lib_ring_buffer_is_finalized(const struct lib_ring_buffer_config *config,
154 struct lib_ring_buffer *buf)
156 - int finalized = ACCESS_ONCE(buf->finalized);
157 + int finalized = READ_ONCE(buf->finalized);
159 * Read finalized before counters.
161 diff --git a/lib/ringbuffer/ring_buffer_frontend.c b/lib/ringbuffer/ring_buffer_frontend.c
162 index abd9757..0d8279b 100644
163 --- a/lib/ringbuffer/ring_buffer_frontend.c
164 +++ b/lib/ringbuffer/ring_buffer_frontend.c
165 @@ -983,7 +983,7 @@ void *channel_destroy(struct channel *chan)
166 * Perform flush before writing to finalized.
169 - ACCESS_ONCE(buf->finalized) = 1;
170 + WRITE_ONCE(buf->finalized, 1);
171 wake_up_interruptible(&buf->read_wait);
174 @@ -997,10 +997,10 @@ void *channel_destroy(struct channel *chan)
175 * Perform flush before writing to finalized.
178 - ACCESS_ONCE(buf->finalized) = 1;
179 + WRITE_ONCE(buf->finalized, 1);
180 wake_up_interruptible(&buf->read_wait);
182 - ACCESS_ONCE(chan->finalized) = 1;
183 + WRITE_ONCE(chan->finalized, 1);
184 wake_up_interruptible(&chan->hp_wait);
185 wake_up_interruptible(&chan->read_wait);
186 priv = chan->backend.priv;
187 @@ -1077,7 +1077,7 @@ int lib_ring_buffer_snapshot(struct lib_ring_buffer *buf,
191 - finalized = ACCESS_ONCE(buf->finalized);
192 + finalized = READ_ONCE(buf->finalized);
194 * Read finalized before counters.
196 @@ -1248,7 +1248,7 @@ int lib_ring_buffer_get_subbuf(struct lib_ring_buffer *buf,
200 - finalized = ACCESS_ONCE(buf->finalized);
201 + finalized = READ_ONCE(buf->finalized);
203 * Read finalized before counters.
205 diff --git a/lib/ringbuffer/ring_buffer_iterator.c b/lib/ringbuffer/ring_buffer_iterator.c
206 index b6bec48..61eaa5b 100644
207 --- a/lib/ringbuffer/ring_buffer_iterator.c
208 +++ b/lib/ringbuffer/ring_buffer_iterator.c
209 @@ -61,7 +61,7 @@ ssize_t lib_ring_buffer_get_next_record(struct channel *chan,
210 switch (iter->state) {
211 case ITER_GET_SUBBUF:
212 ret = lib_ring_buffer_get_next_subbuf(buf);
213 - if (ret && !ACCESS_ONCE(buf->finalized)
214 + if (ret && !READ_ONCE(buf->finalized)
215 && config->alloc == RING_BUFFER_ALLOC_GLOBAL) {
217 * Use "pull" scheme for global buffers. The reader
218 diff --git a/lttng-clock.c b/lttng-clock.c
219 index a5a7eaa..48b4be5 100644
222 @@ -48,7 +48,7 @@ int lttng_clock_register_plugin(struct lttng_trace_clock *ltc,
226 - ACCESS_ONCE(lttng_trace_clock) = ltc;
227 + WRITE_ONCE(lttng_trace_clock, ltc);
228 lttng_trace_clock_mod = mod;
230 mutex_unlock(&clock_mutex);
231 @@ -66,7 +66,7 @@ void lttng_clock_unregister_plugin(struct lttng_trace_clock *ltc,
233 WARN_ON_ONCE(lttng_trace_clock_mod != mod);
235 - ACCESS_ONCE(lttng_trace_clock) = NULL;
236 + WRITE_ONCE(lttng_trace_clock, NULL);
237 lttng_trace_clock_mod = NULL;
239 mutex_unlock(&clock_mutex);
240 @@ -83,7 +83,7 @@ void lttng_clock_ref(void)
241 ret = try_module_get(lttng_trace_clock_mod);
243 printk(KERN_ERR "LTTng-clock cannot get clock plugin module\n");
244 - ACCESS_ONCE(lttng_trace_clock) = NULL;
245 + WRITE_ONCE(lttng_trace_clock, NULL);
246 lttng_trace_clock_mod = NULL;
249 diff --git a/lttng-events.c b/lttng-events.c
250 index 21c4113..75c3fb1 100644
253 @@ -186,7 +186,7 @@ void lttng_session_destroy(struct lttng_session *session)
256 mutex_lock(&sessions_mutex);
257 - ACCESS_ONCE(session->active) = 0;
258 + WRITE_ONCE(session->active, 0);
259 list_for_each_entry(chan, &session->chan, list) {
260 ret = lttng_syscalls_unregister(chan);
262 @@ -261,16 +261,16 @@ int lttng_session_enable(struct lttng_session *session)
263 lib_ring_buffer_clear_quiescent_channel(chan->chan);
266 - ACCESS_ONCE(session->active) = 1;
267 - ACCESS_ONCE(session->been_active) = 1;
268 + WRITE_ONCE(session->active, 1);
269 + WRITE_ONCE(session->been_active, 1);
270 ret = _lttng_session_metadata_statedump(session);
272 - ACCESS_ONCE(session->active) = 0;
273 + WRITE_ONCE(session->active, 0);
276 ret = lttng_statedump_start(session);
278 - ACCESS_ONCE(session->active) = 0;
279 + WRITE_ONCE(session->active, 0);
281 mutex_unlock(&sessions_mutex);
283 @@ -286,7 +286,7 @@ int lttng_session_disable(struct lttng_session *session)
287 - ACCESS_ONCE(session->active) = 0;
288 + WRITE_ONCE(session->active, 0);
290 /* Set transient enabler state to "disabled" */
292 @@ -361,7 +361,7 @@ int lttng_channel_enable(struct lttng_channel *channel)
294 lttng_session_sync_enablers(channel->session);
295 /* Set atomically the state to "enabled" */
296 - ACCESS_ONCE(channel->enabled) = 1;
297 + WRITE_ONCE(channel->enabled, 1);
299 mutex_unlock(&sessions_mutex);
301 @@ -381,7 +381,7 @@ int lttng_channel_disable(struct lttng_channel *channel)
304 /* Set atomically the state to "disabled" */
305 - ACCESS_ONCE(channel->enabled) = 0;
306 + WRITE_ONCE(channel->enabled, 0);
307 /* Set transient enabler state to "enabled" */
309 lttng_session_sync_enablers(channel->session);
310 @@ -411,7 +411,7 @@ int lttng_event_enable(struct lttng_event *event)
311 case LTTNG_KERNEL_KPROBE:
312 case LTTNG_KERNEL_FUNCTION:
313 case LTTNG_KERNEL_NOOP:
314 - ACCESS_ONCE(event->enabled) = 1;
315 + WRITE_ONCE(event->enabled, 1);
317 case LTTNG_KERNEL_KRETPROBE:
318 ret = lttng_kretprobes_event_enable_state(event, 1);
319 @@ -446,7 +446,7 @@ int lttng_event_disable(struct lttng_event *event)
320 case LTTNG_KERNEL_KPROBE:
321 case LTTNG_KERNEL_FUNCTION:
322 case LTTNG_KERNEL_NOOP:
323 - ACCESS_ONCE(event->enabled) = 0;
324 + WRITE_ONCE(event->enabled, 0);
326 case LTTNG_KERNEL_KRETPROBE:
327 ret = lttng_kretprobes_event_enable_state(event, 0);
328 @@ -1517,7 +1517,7 @@ void lttng_session_sync_enablers(struct lttng_session *session)
330 enabled = enabled && session->tstate && event->chan->tstate;
332 - ACCESS_ONCE(event->enabled) = enabled;
333 + WRITE_ONCE(event->enabled, enabled);
335 * Sync tracepoint registration with event enabled
337 @@ -1643,7 +1643,7 @@ int lttng_metadata_printf(struct lttng_session *session,
339 struct lttng_metadata_stream *stream;
341 - WARN_ON_ONCE(!ACCESS_ONCE(session->active));
342 + WARN_ON_ONCE(!READ_ONCE(session->active));
345 str = kvasprintf(GFP_KERNEL, fmt, ap);
346 @@ -2230,7 +2230,7 @@ int _lttng_event_metadata_statedump(struct lttng_session *session,
350 - if (event->metadata_dumped || !ACCESS_ONCE(session->active))
351 + if (event->metadata_dumped || !READ_ONCE(session->active))
353 if (chan->channel_type == METADATA_CHANNEL)
355 @@ -2297,7 +2297,7 @@ int _lttng_channel_metadata_statedump(struct lttng_session *session,
359 - if (chan->metadata_dumped || !ACCESS_ONCE(session->active))
360 + if (chan->metadata_dumped || !READ_ONCE(session->active))
363 if (chan->channel_type == METADATA_CHANNEL)
364 @@ -2454,7 +2454,7 @@ int _lttng_session_metadata_statedump(struct lttng_session *session)
365 struct lttng_event *event;
368 - if (!ACCESS_ONCE(session->active))
369 + if (!READ_ONCE(session->active))
371 if (session->metadata_dumped)
373 diff --git a/probes/lttng-ftrace.c b/probes/lttng-ftrace.c
374 index 9ec326e..50675a4 100644
375 --- a/probes/lttng-ftrace.c
376 +++ b/probes/lttng-ftrace.c
377 @@ -58,11 +58,11 @@ void lttng_ftrace_handler(unsigned long ip, unsigned long parent_ip,
381 - if (unlikely(!ACCESS_ONCE(chan->session->active)))
382 + if (unlikely(!READ_ONCE(chan->session->active)))
384 - if (unlikely(!ACCESS_ONCE(chan->enabled)))
385 + if (unlikely(!READ_ONCE(chan->enabled)))
387 - if (unlikely(!ACCESS_ONCE(event->enabled)))
388 + if (unlikely(!READ_ONCE(event->enabled)))
391 lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx,
392 @@ -94,11 +94,11 @@ void lttng_ftrace_handler(unsigned long ip, unsigned long parent_ip, void **data
396 - if (unlikely(!ACCESS_ONCE(chan->session->active)))
397 + if (unlikely(!READ_ONCE(chan->session->active)))
399 - if (unlikely(!ACCESS_ONCE(chan->enabled)))
400 + if (unlikely(!READ_ONCE(chan->enabled)))
402 - if (unlikely(!ACCESS_ONCE(event->enabled)))
403 + if (unlikely(!READ_ONCE(event->enabled)))
406 lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx,
407 diff --git a/probes/lttng-kprobes.c b/probes/lttng-kprobes.c
408 index daf14ce..b58a09b 100644
409 --- a/probes/lttng-kprobes.c
410 +++ b/probes/lttng-kprobes.c
411 @@ -43,11 +43,11 @@ int lttng_kprobes_handler_pre(struct kprobe *p, struct pt_regs *regs)
413 unsigned long data = (unsigned long) p->addr;
415 - if (unlikely(!ACCESS_ONCE(chan->session->active)))
416 + if (unlikely(!READ_ONCE(chan->session->active)))
418 - if (unlikely(!ACCESS_ONCE(chan->enabled)))
419 + if (unlikely(!READ_ONCE(chan->enabled)))
421 - if (unlikely(!ACCESS_ONCE(event->enabled)))
422 + if (unlikely(!READ_ONCE(event->enabled)))
425 lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(data),
426 diff --git a/probes/lttng-kretprobes.c b/probes/lttng-kretprobes.c
427 index 498df62..49b7de8 100644
428 --- a/probes/lttng-kretprobes.c
429 +++ b/probes/lttng-kretprobes.c
430 @@ -63,11 +63,11 @@ int _lttng_kretprobes_handler(struct kretprobe_instance *krpi,
431 unsigned long parent_ip;
434 - if (unlikely(!ACCESS_ONCE(chan->session->active)))
435 + if (unlikely(!READ_ONCE(chan->session->active)))
437 - if (unlikely(!ACCESS_ONCE(chan->enabled)))
438 + if (unlikely(!READ_ONCE(chan->enabled)))
440 - if (unlikely(!ACCESS_ONCE(event->enabled)))
441 + if (unlikely(!READ_ONCE(event->enabled)))
444 payload.ip = (unsigned long) krpi->rp->kp.addr;
445 @@ -304,8 +304,8 @@ int lttng_kretprobes_event_enable_state(struct lttng_event *event,
447 lttng_krp = event->u.kretprobe.lttng_krp;
448 event_return = lttng_krp->event[EVENT_RETURN];
449 - ACCESS_ONCE(event->enabled) = enable;
450 - ACCESS_ONCE(event_return->enabled) = enable;
451 + WRITE_ONCE(event->enabled, enable);
452 + WRITE_ONCE(event_return->enabled, enable);
455 EXPORT_SYMBOL_GPL(lttng_kretprobes_event_enable_state);
456 diff --git a/probes/lttng-tracepoint-event-impl.h b/probes/lttng-tracepoint-event-impl.h
457 index 7ec0d75..61f1c2d 100644
458 --- a/probes/lttng-tracepoint-event-impl.h
459 +++ b/probes/lttng-tracepoint-event-impl.h
460 @@ -1143,11 +1143,11 @@ static void __event_probe__##_name(void *__data, _proto) \
462 if (!_TP_SESSION_CHECK(session, __session)) \
464 - if (unlikely(!ACCESS_ONCE(__session->active))) \
465 + if (unlikely(!READ_ONCE(__session->active))) \
467 - if (unlikely(!ACCESS_ONCE(__chan->enabled))) \
468 + if (unlikely(!READ_ONCE(__chan->enabled))) \
470 - if (unlikely(!ACCESS_ONCE(__event->enabled))) \
471 + if (unlikely(!READ_ONCE(__event->enabled))) \
473 __lpf = lttng_rcu_dereference(__session->pid_tracker); \
474 if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->tgid))) \
475 @@ -1217,11 +1217,11 @@ static void __event_probe__##_name(void *__data) \
477 if (!_TP_SESSION_CHECK(session, __session)) \
479 - if (unlikely(!ACCESS_ONCE(__session->active))) \
480 + if (unlikely(!READ_ONCE(__session->active))) \
482 - if (unlikely(!ACCESS_ONCE(__chan->enabled))) \
483 + if (unlikely(!READ_ONCE(__chan->enabled))) \
485 - if (unlikely(!ACCESS_ONCE(__event->enabled))) \
486 + if (unlikely(!READ_ONCE(__event->enabled))) \
488 __lpf = lttng_rcu_dereference(__session->pid_tracker); \
489 if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid))) \
490 diff --git a/wrapper/compiler.h b/wrapper/compiler.h
491 index 0c01632..e4533d4 100644
492 --- a/wrapper/compiler.h
493 +++ b/wrapper/compiler.h
499 + * READ/WRITE_ONCE were introduced in kernel 3.19 and ACCESS_ONCE
500 + * was removed in 4.15. Prefer READ/WRITE but fallback to ACCESS
501 + * when they are not available.
504 +# define READ_ONCE(x) ACCESS_ONCE(x)
508 +# define WRITE_ONCE(x, val) ({ ACCESS_ONCE(x) = val; })
511 #endif /* _LTTNG_WRAPPER_COMPILER_H */
512 diff --git a/wrapper/trace-clock.h b/wrapper/trace-clock.h
513 index 7f17ccd..08f9922 100644
514 --- a/wrapper/trace-clock.h
515 +++ b/wrapper/trace-clock.h
517 #include <asm/local.h>
518 #include <lttng-kernel-version.h>
519 #include <lttng-clock.h>
520 +#include <wrapper/compiler.h>
521 #include <wrapper/percpu-defs.h>
522 #include <wrapper/random.h>
524 @@ -176,7 +177,7 @@ static inline void put_trace_clock(void)
526 static inline u64 trace_clock_read64(void)
528 - struct lttng_trace_clock *ltc = ACCESS_ONCE(lttng_trace_clock);
529 + struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
532 return trace_clock_read64_monotonic();
533 @@ -188,7 +189,7 @@ static inline u64 trace_clock_read64(void)
535 static inline u64 trace_clock_freq(void)
537 - struct lttng_trace_clock *ltc = ACCESS_ONCE(lttng_trace_clock);
538 + struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
541 return trace_clock_freq_monotonic();
542 @@ -200,7 +201,7 @@ static inline u64 trace_clock_freq(void)
544 static inline int trace_clock_uuid(char *uuid)
546 - struct lttng_trace_clock *ltc = ACCESS_ONCE(lttng_trace_clock);
547 + struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
549 read_barrier_depends(); /* load ltc before content */
550 /* Use default UUID cb when NULL */
551 @@ -213,7 +214,7 @@ static inline int trace_clock_uuid(char *uuid)
553 static inline const char *trace_clock_name(void)
555 - struct lttng_trace_clock *ltc = ACCESS_ONCE(lttng_trace_clock);
556 + struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
559 return trace_clock_name_monotonic();
560 @@ -225,7 +226,7 @@ static inline const char *trace_clock_name(void)
562 static inline const char *trace_clock_description(void)
564 - struct lttng_trace_clock *ltc = ACCESS_ONCE(lttng_trace_clock);
565 + struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
568 return trace_clock_description_monotonic();