1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * kernel/locking/mutex.c
4 *
5 * Mutexes: blocking mutual exclusion locks
6 *
7 * Started by Ingo Molnar:
8 *
9 * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
10 *
11 * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
12 * David Howells for suggestions and improvements.
13 *
14 * - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline
15 * from the -rt tree, where it was originally implemented for rtmutexes
16 * by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
17 * and Sven Dietrich.
18 *
19 * Also see Documentation/locking/mutex-design.rst.
20 */
21#include <linux/mutex.h>
22#include <linux/ww_mutex.h>
23#include <linux/sched/signal.h>
24#include <linux/sched/rt.h>
25#include <linux/sched/wake_q.h>
26#include <linux/sched/debug.h>
27#include <linux/export.h>
28#include <linux/spinlock.h>
29#include <linux/interrupt.h>
30#include <linux/debug_locks.h>
31#include <linux/osq_lock.h>
32#include <linux/hung_task.h>
33
34#define CREATE_TRACE_POINTS
35#include <trace/events/lock.h>
36
37#ifndef CONFIG_PREEMPT_RT
38#include "mutex.h"
39
40#ifdef CONFIG_DEBUG_MUTEXES
41# define MUTEX_WARN_ON(cond) DEBUG_LOCKS_WARN_ON(cond)
42#else
43# define MUTEX_WARN_ON(cond)
44#endif
45
46void
47__mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
48{
49 atomic_long_set(v: &lock->owner, i: 0);
50 raw_spin_lock_init(&lock->wait_lock);
51 INIT_LIST_HEAD(list: &lock->wait_list);
52#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
53 osq_lock_init(lock: &lock->osq);
54#endif
55
56 debug_mutex_init(lock, name, key);
57}
58EXPORT_SYMBOL(__mutex_init);
59
60static inline struct task_struct *__owner_task(unsigned long owner)
61{
62 return (struct task_struct *)(owner & ~MUTEX_FLAGS);
63}
64
65bool mutex_is_locked(struct mutex *lock)
66{
67 return __mutex_owner(lock) != NULL;
68}
69EXPORT_SYMBOL(mutex_is_locked);
70
71static inline unsigned long __owner_flags(unsigned long owner)
72{
73 return owner & MUTEX_FLAGS;
74}
75
76/* Do not use the return value as a pointer directly. */
77unsigned long mutex_get_owner(struct mutex *lock)
78{
79 unsigned long owner = atomic_long_read(v: &lock->owner);
80
81 return (unsigned long)__owner_task(owner);
82}
83
84/*
85 * Returns: __mutex_owner(lock) on failure or NULL on success.
86 */
87static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff)
88{
89 unsigned long owner, curr = (unsigned long)current;
90
91 owner = atomic_long_read(v: &lock->owner);
92 for (;;) { /* must loop, can race against a flag */
93 unsigned long flags = __owner_flags(owner);
94 unsigned long task = owner & ~MUTEX_FLAGS;
95
96 if (task) {
97 if (flags & MUTEX_FLAG_PICKUP) {
98 if (task != curr)
99 break;
100 flags &= ~MUTEX_FLAG_PICKUP;
101 } else if (handoff) {
102 if (flags & MUTEX_FLAG_HANDOFF)
103 break;
104 flags |= MUTEX_FLAG_HANDOFF;
105 } else {
106 break;
107 }
108 } else {
109 MUTEX_WARN_ON(flags & (MUTEX_FLAG_HANDOFF | MUTEX_FLAG_PICKUP));
110 task = curr;
111 }
112
113 if (atomic_long_try_cmpxchg_acquire(v: &lock->owner, old: &owner, new: task | flags)) {
114 if (task == curr)
115 return NULL;
116 break;
117 }
118 }
119
120 return __owner_task(owner);
121}
122
123/*
124 * Trylock or set HANDOFF
125 */
126static inline bool __mutex_trylock_or_handoff(struct mutex *lock, bool handoff)
127{
128 return !__mutex_trylock_common(lock, handoff);
129}
130
131/*
132 * Actual trylock that will work on any unlocked state.
133 */
134static inline bool __mutex_trylock(struct mutex *lock)
135{
136 return !__mutex_trylock_common(lock, handoff: false);
137}
138
139#ifndef CONFIG_DEBUG_LOCK_ALLOC
140/*
141 * Lockdep annotations are contained to the slow paths for simplicity.
142 * There is nothing that would stop spreading the lockdep annotations outwards
143 * except more code.
144 */
145
146/*
147 * Optimistic trylock that only works in the uncontended case. Make sure to
148 * follow with a __mutex_trylock() before failing.
149 */
150static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
151{
152 unsigned long curr = (unsigned long)current;
153 unsigned long zero = 0UL;
154
155 MUTEX_WARN_ON(lock->magic != lock);
156
157 if (atomic_long_try_cmpxchg_acquire(v: &lock->owner, old: &zero, new: curr))
158 return true;
159
160 return false;
161}
162
163static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
164{
165 unsigned long curr = (unsigned long)current;
166
167 return atomic_long_try_cmpxchg_release(v: &lock->owner, old: &curr, new: 0UL);
168}
169#endif
170
171static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag)
172{
173 atomic_long_or(i: flag, v: &lock->owner);
174}
175
176static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag)
177{
178 atomic_long_andnot(i: flag, v: &lock->owner);
179}
180
181static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)
182{
183 return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
184}
185
186/*
187 * Add @waiter to a given location in the lock wait_list and set the
188 * FLAG_WAITERS flag if it's the first waiter.
189 */
190static void
191__mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
192 struct list_head *list)
193{
194 hung_task_set_blocker(lock, BLOCKER_TYPE_MUTEX);
195 debug_mutex_add_waiter(lock, waiter, current);
196
197 list_add_tail(new: &waiter->list, head: list);
198 if (__mutex_waiter_is_first(lock, waiter))
199 __mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
200}
201
202static void
203__mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
204{
205 list_del(entry: &waiter->list);
206 if (likely(list_empty(&lock->wait_list)))
207 __mutex_clear_flag(lock, MUTEX_FLAGS);
208
209 debug_mutex_remove_waiter(lock, waiter, current);
210 hung_task_clear_blocker();
211}
212
213/*
214 * Give up ownership to a specific task, when @task = NULL, this is equivalent
215 * to a regular unlock. Sets PICKUP on a handoff, clears HANDOFF, preserves
216 * WAITERS. Provides RELEASE semantics like a regular unlock, the
217 * __mutex_trylock() provides a matching ACQUIRE semantics for the handoff.
218 */
219static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
220{
221 unsigned long owner = atomic_long_read(v: &lock->owner);
222
223 for (;;) {
224 unsigned long new;
225
226 MUTEX_WARN_ON(__owner_task(owner) != current);
227 MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
228
229 new = (owner & MUTEX_FLAG_WAITERS);
230 new |= (unsigned long)task;
231 if (task)
232 new |= MUTEX_FLAG_PICKUP;
233
234 if (atomic_long_try_cmpxchg_release(v: &lock->owner, old: &owner, new))
235 break;
236 }
237}
238
239#ifndef CONFIG_DEBUG_LOCK_ALLOC
240/*
241 * We split the mutex lock/unlock logic into separate fastpath and
242 * slowpath functions, to reduce the register pressure on the fastpath.
243 * We also put the fastpath first in the kernel image, to make sure the
244 * branch is predicted by the CPU as default-untaken.
245 */
246static void __sched __mutex_lock_slowpath(struct mutex *lock);
247
248/**
249 * mutex_lock - acquire the mutex
250 * @lock: the mutex to be acquired
251 *
252 * Lock the mutex exclusively for this task. If the mutex is not
253 * available right now, it will sleep until it can get it.
254 *
255 * The mutex must later on be released by the same task that
256 * acquired it. Recursive locking is not allowed. The task
257 * may not exit without first unlocking the mutex. Also, kernel
258 * memory where the mutex resides must not be freed with
259 * the mutex still locked. The mutex must first be initialized
260 * (or statically defined) before it can be locked. memset()-ing
261 * the mutex to 0 is not allowed.
262 *
263 * (The CONFIG_DEBUG_MUTEXES .config option turns on debugging
264 * checks that will enforce the restrictions and will also do
265 * deadlock debugging)
266 *
267 * This function is similar to (but not equivalent to) down().
268 */
269void __sched mutex_lock(struct mutex *lock)
270{
271 might_sleep();
272
273 if (!__mutex_trylock_fast(lock))
274 __mutex_lock_slowpath(lock);
275}
276EXPORT_SYMBOL(mutex_lock);
277#endif
278
279#include "ww_mutex.h"
280
281#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
282
283/*
284 * Trylock variant that returns the owning task on failure.
285 */
286static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock)
287{
288 return __mutex_trylock_common(lock, handoff: false);
289}
290
291static inline
292bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
293 struct mutex_waiter *waiter)
294{
295 struct ww_mutex *ww;
296
297 ww = container_of(lock, struct ww_mutex, base);
298
299 /*
300 * If ww->ctx is set the contents are undefined, only
301 * by acquiring wait_lock there is a guarantee that
302 * they are not invalid when reading.
303 *
304 * As such, when deadlock detection needs to be
305 * performed the optimistic spinning cannot be done.
306 *
307 * Check this in every inner iteration because we may
308 * be racing against another thread's ww_mutex_lock.
309 */
310 if (ww_ctx->acquired > 0 && READ_ONCE(ww->ctx))
311 return false;
312
313 /*
314 * If we aren't on the wait list yet, cancel the spin
315 * if there are waiters. We want to avoid stealing the
316 * lock from a waiter with an earlier stamp, since the
317 * other thread may already own a lock that we also
318 * need.
319 */
320 if (!waiter && (atomic_long_read(v: &lock->owner) & MUTEX_FLAG_WAITERS))
321 return false;
322
323 /*
324 * Similarly, stop spinning if we are no longer the
325 * first waiter.
326 */
327 if (waiter && !__mutex_waiter_is_first(lock, waiter))
328 return false;
329
330 return true;
331}
332
333/*
334 * Look out! "owner" is an entirely speculative pointer access and not
335 * reliable.
336 *
337 * "noinline" so that this function shows up on perf profiles.
338 */
339static noinline
340bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,
341 struct ww_acquire_ctx *ww_ctx, struct mutex_waiter *waiter)
342{
343 bool ret = true;
344
345 lockdep_assert_preemption_disabled();
346
347 while (__mutex_owner(lock) == owner) {
348 /*
349 * Ensure we emit the owner->on_cpu, dereference _after_
350 * checking lock->owner still matches owner. And we already
351 * disabled preemption which is equal to the RCU read-side
352 * crital section in optimistic spinning code. Thus the
353 * task_strcut structure won't go away during the spinning
354 * period
355 */
356 barrier();
357
358 /*
359 * Use vcpu_is_preempted to detect lock holder preemption issue.
360 */
361 if (!owner_on_cpu(owner) || need_resched()) {
362 ret = false;
363 break;
364 }
365
366 if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) {
367 ret = false;
368 break;
369 }
370
371 cpu_relax();
372 }
373
374 return ret;
375}
376
377/*
378 * Initial check for entering the mutex spinning loop
379 */
380static inline int mutex_can_spin_on_owner(struct mutex *lock)
381{
382 struct task_struct *owner;
383 int retval = 1;
384
385 lockdep_assert_preemption_disabled();
386
387 if (need_resched())
388 return 0;
389
390 /*
391 * We already disabled preemption which is equal to the RCU read-side
392 * crital section in optimistic spinning code. Thus the task_strcut
393 * structure won't go away during the spinning period.
394 */
395 owner = __mutex_owner(lock);
396 if (owner)
397 retval = owner_on_cpu(owner);
398
399 /*
400 * If lock->owner is not set, the mutex has been released. Return true
401 * such that we'll trylock in the spin path, which is a faster option
402 * than the blocking slow path.
403 */
404 return retval;
405}
406
407/*
408 * Optimistic spinning.
409 *
410 * We try to spin for acquisition when we find that the lock owner
411 * is currently running on a (different) CPU and while we don't
412 * need to reschedule. The rationale is that if the lock owner is
413 * running, it is likely to release the lock soon.
414 *
415 * The mutex spinners are queued up using MCS lock so that only one
416 * spinner can compete for the mutex. However, if mutex spinning isn't
417 * going to happen, there is no point in going through the lock/unlock
418 * overhead.
419 *
420 * Returns true when the lock was taken, otherwise false, indicating
421 * that we need to jump to the slowpath and sleep.
422 *
423 * The waiter flag is set to true if the spinner is a waiter in the wait
424 * queue. The waiter-spinner will spin on the lock directly and concurrently
425 * with the spinner at the head of the OSQ, if present, until the owner is
426 * changed to itself.
427 */
428static __always_inline bool
429mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
430 struct mutex_waiter *waiter)
431{
432 if (!waiter) {
433 /*
434 * The purpose of the mutex_can_spin_on_owner() function is
435 * to eliminate the overhead of osq_lock() and osq_unlock()
436 * in case spinning isn't possible. As a waiter-spinner
437 * is not going to take OSQ lock anyway, there is no need
438 * to call mutex_can_spin_on_owner().
439 */
440 if (!mutex_can_spin_on_owner(lock))
441 goto fail;
442
443 /*
444 * In order to avoid a stampede of mutex spinners trying to
445 * acquire the mutex all at once, the spinners need to take a
446 * MCS (queued) lock first before spinning on the owner field.
447 */
448 if (!osq_lock(lock: &lock->osq))
449 goto fail;
450 }
451
452 for (;;) {
453 struct task_struct *owner;
454
455 /* Try to acquire the mutex... */
456 owner = __mutex_trylock_or_owner(lock);
457 if (!owner)
458 break;
459
460 /*
461 * There's an owner, wait for it to either
462 * release the lock or go to sleep.
463 */
464 if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter))
465 goto fail_unlock;
466
467 /*
468 * The cpu_relax() call is a compiler barrier which forces
469 * everything in this loop to be re-loaded. We don't need
470 * memory barriers as we'll eventually observe the right
471 * values at the cost of a few extra spins.
472 */
473 cpu_relax();
474 }
475
476 if (!waiter)
477 osq_unlock(lock: &lock->osq);
478
479 return true;
480
481
482fail_unlock:
483 if (!waiter)
484 osq_unlock(lock: &lock->osq);
485
486fail:
487 /*
488 * If we fell out of the spin path because of need_resched(),
489 * reschedule now, before we try-lock the mutex. This avoids getting
490 * scheduled out right after we obtained the mutex.
491 */
492 if (need_resched()) {
493 /*
494 * We _should_ have TASK_RUNNING here, but just in case
495 * we do not, make it so, otherwise we might get stuck.
496 */
497 __set_current_state(TASK_RUNNING);
498 schedule_preempt_disabled();
499 }
500
501 return false;
502}
503#else
504static __always_inline bool
505mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
506 struct mutex_waiter *waiter)
507{
508 return false;
509}
510#endif
511
512static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip);
513
514/**
515 * mutex_unlock - release the mutex
516 * @lock: the mutex to be released
517 *
518 * Unlock a mutex that has been locked by this task previously.
519 *
520 * This function must not be used in interrupt context. Unlocking
521 * of a not locked mutex is not allowed.
522 *
523 * The caller must ensure that the mutex stays alive until this function has
524 * returned - mutex_unlock() can NOT directly be used to release an object such
525 * that another concurrent task can free it.
526 * Mutexes are different from spinlocks & refcounts in this aspect.
527 *
528 * This function is similar to (but not equivalent to) up().
529 */
530void __sched mutex_unlock(struct mutex *lock)
531{
532#ifndef CONFIG_DEBUG_LOCK_ALLOC
533 if (__mutex_unlock_fast(lock))
534 return;
535#endif
536 __mutex_unlock_slowpath(lock, _RET_IP_);
537}
538EXPORT_SYMBOL(mutex_unlock);
539
540/**
541 * ww_mutex_unlock - release the w/w mutex
542 * @lock: the mutex to be released
543 *
544 * Unlock a mutex that has been locked by this task previously with any of the
545 * ww_mutex_lock* functions (with or without an acquire context). It is
546 * forbidden to release the locks after releasing the acquire context.
547 *
548 * This function must not be used in interrupt context. Unlocking
549 * of a unlocked mutex is not allowed.
550 */
551void __sched ww_mutex_unlock(struct ww_mutex *lock)
552{
553 __ww_mutex_unlock(lock);
554 mutex_unlock(&lock->base);
555}
556EXPORT_SYMBOL(ww_mutex_unlock);
557
558/*
559 * Lock a mutex (possibly interruptible), slowpath:
560 */
561static __always_inline int __sched
562__mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass,
563 struct lockdep_map *nest_lock, unsigned long ip,
564 struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
565{
566 DEFINE_WAKE_Q(wake_q);
567 struct mutex_waiter waiter;
568 struct ww_mutex *ww;
569 unsigned long flags;
570 int ret;
571
572 if (!use_ww_ctx)
573 ww_ctx = NULL;
574
575 might_sleep();
576
577 MUTEX_WARN_ON(lock->magic != lock);
578
579 ww = container_of(lock, struct ww_mutex, base);
580 if (ww_ctx) {
581 if (unlikely(ww_ctx == READ_ONCE(ww->ctx)))
582 return -EALREADY;
583
584 /*
585 * Reset the wounded flag after a kill. No other process can
586 * race and wound us here since they can't have a valid owner
587 * pointer if we don't have any locks held.
588 */
589 if (ww_ctx->acquired == 0)
590 ww_ctx->wounded = 0;
591
592#ifdef CONFIG_DEBUG_LOCK_ALLOC
593 nest_lock = &ww_ctx->dep_map;
594#endif
595 }
596
597 preempt_disable();
598 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
599
600 trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
601 if (__mutex_trylock(lock) ||
602 mutex_optimistic_spin(lock, ww_ctx, NULL)) {
603 /* got the lock, yay! */
604 lock_acquired(&lock->dep_map, ip);
605 if (ww_ctx)
606 ww_mutex_set_context_fastpath(lock: ww, ctx: ww_ctx);
607 trace_contention_end(lock, ret: 0);
608 preempt_enable();
609 return 0;
610 }
611
612 raw_spin_lock_irqsave(&lock->wait_lock, flags);
613 /*
614 * After waiting to acquire the wait_lock, try again.
615 */
616 if (__mutex_trylock(lock)) {
617 if (ww_ctx)
618 __ww_mutex_check_waiters(lock, ww_ctx, wake_q: &wake_q);
619
620 goto skip_wait;
621 }
622
623 debug_mutex_lock_common(lock, &waiter);
624 waiter.task = current;
625 if (use_ww_ctx)
626 waiter.ww_ctx = ww_ctx;
627
628 lock_contended(&lock->dep_map, ip);
629
630 if (!use_ww_ctx) {
631 /* add waiting tasks to the end of the waitqueue (FIFO): */
632 __mutex_add_waiter(lock, waiter: &waiter, list: &lock->wait_list);
633 } else {
634 /*
635 * Add in stamp order, waking up waiters that must kill
636 * themselves.
637 */
638 ret = __ww_mutex_add_waiter(waiter: &waiter, lock, ww_ctx, wake_q: &wake_q);
639 if (ret)
640 goto err_early_kill;
641 }
642
643 __set_task_blocked_on(current, m: lock);
644 set_current_state(state);
645 trace_contention_begin(lock, LCB_F_MUTEX);
646 for (;;) {
647 bool first;
648
649 /*
650 * Once we hold wait_lock, we're serialized against
651 * mutex_unlock() handing the lock off to us, do a trylock
652 * before testing the error conditions to make sure we pick up
653 * the handoff.
654 */
655 if (__mutex_trylock(lock))
656 goto acquired;
657
658 /*
659 * Check for signals and kill conditions while holding
660 * wait_lock. This ensures the lock cancellation is ordered
661 * against mutex_unlock() and wake-ups do not go missing.
662 */
663 if (signal_pending_state(state, current)) {
664 ret = -EINTR;
665 goto err;
666 }
667
668 if (ww_ctx) {
669 ret = __ww_mutex_check_kill(lock, waiter: &waiter, ctx: ww_ctx);
670 if (ret)
671 goto err;
672 }
673
674 raw_spin_unlock_irqrestore_wake(lock: &lock->wait_lock, flags, wake_q: &wake_q);
675
676 schedule_preempt_disabled();
677
678 first = __mutex_waiter_is_first(lock, waiter: &waiter);
679
680 /*
681 * As we likely have been woken up by task
682 * that has cleared our blocked_on state, re-set
683 * it to the lock we are trying to acquire.
684 */
685 set_task_blocked_on(current, m: lock);
686 set_current_state(state);
687 /*
688 * Here we order against unlock; we must either see it change
689 * state back to RUNNING and fall through the next schedule(),
690 * or we must see its unlock and acquire.
691 */
692 if (__mutex_trylock_or_handoff(lock, handoff: first))
693 break;
694
695 if (first) {
696 trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
697 /*
698 * mutex_optimistic_spin() can call schedule(), so
699 * clear blocked on so we don't become unselectable
700 * to run.
701 */
702 clear_task_blocked_on(current, m: lock);
703 if (mutex_optimistic_spin(lock, ww_ctx, waiter: &waiter))
704 break;
705 set_task_blocked_on(current, m: lock);
706 trace_contention_begin(lock, LCB_F_MUTEX);
707 }
708
709 raw_spin_lock_irqsave(&lock->wait_lock, flags);
710 }
711 raw_spin_lock_irqsave(&lock->wait_lock, flags);
712acquired:
713 __clear_task_blocked_on(current, m: lock);
714 __set_current_state(TASK_RUNNING);
715
716 if (ww_ctx) {
717 /*
718 * Wound-Wait; we stole the lock (!first_waiter), check the
719 * waiters as anyone might want to wound us.
720 */
721 if (!ww_ctx->is_wait_die &&
722 !__mutex_waiter_is_first(lock, waiter: &waiter))
723 __ww_mutex_check_waiters(lock, ww_ctx, wake_q: &wake_q);
724 }
725
726 __mutex_remove_waiter(lock, waiter: &waiter);
727
728 debug_mutex_free_waiter(&waiter);
729
730skip_wait:
731 /* got the lock - cleanup and rejoice! */
732 lock_acquired(&lock->dep_map, ip);
733 trace_contention_end(lock, ret: 0);
734
735 if (ww_ctx)
736 ww_mutex_lock_acquired(ww, ww_ctx);
737
738 raw_spin_unlock_irqrestore_wake(lock: &lock->wait_lock, flags, wake_q: &wake_q);
739 preempt_enable();
740 return 0;
741
742err:
743 __clear_task_blocked_on(current, m: lock);
744 __set_current_state(TASK_RUNNING);
745 __mutex_remove_waiter(lock, waiter: &waiter);
746err_early_kill:
747 WARN_ON(__get_task_blocked_on(current));
748 trace_contention_end(lock, ret);
749 raw_spin_unlock_irqrestore_wake(lock: &lock->wait_lock, flags, wake_q: &wake_q);
750 debug_mutex_free_waiter(&waiter);
751 mutex_release(&lock->dep_map, ip);
752 preempt_enable();
753 return ret;
754}
755
756static int __sched
757__mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
758 struct lockdep_map *nest_lock, unsigned long ip)
759{
760 return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, use_ww_ctx: false);
761}
762
763static int __sched
764__ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,
765 unsigned long ip, struct ww_acquire_ctx *ww_ctx)
766{
767 return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, use_ww_ctx: true);
768}
769
770/**
771 * ww_mutex_trylock - tries to acquire the w/w mutex with optional acquire context
772 * @ww: mutex to lock
773 * @ww_ctx: optional w/w acquire context
774 *
775 * Trylocks a mutex with the optional acquire context; no deadlock detection is
776 * possible. Returns 1 if the mutex has been acquired successfully, 0 otherwise.
777 *
778 * Unlike ww_mutex_lock, no deadlock handling is performed. However, if a @ctx is
779 * specified, -EALREADY handling may happen in calls to ww_mutex_trylock.
780 *
781 * A mutex acquired with this function must be released with ww_mutex_unlock.
782 */
783int ww_mutex_trylock(struct ww_mutex *ww, struct ww_acquire_ctx *ww_ctx)
784{
785 if (!ww_ctx)
786 return mutex_trylock(lock: &ww->base);
787
788 MUTEX_WARN_ON(ww->base.magic != &ww->base);
789
790 /*
791 * Reset the wounded flag after a kill. No other process can
792 * race and wound us here, since they can't have a valid owner
793 * pointer if we don't have any locks held.
794 */
795 if (ww_ctx->acquired == 0)
796 ww_ctx->wounded = 0;
797
798 if (__mutex_trylock(lock: &ww->base)) {
799 ww_mutex_set_context_fastpath(lock: ww, ctx: ww_ctx);
800 mutex_acquire_nest(&ww->base.dep_map, 0, 1, &ww_ctx->dep_map, _RET_IP_);
801 return 1;
802 }
803
804 return 0;
805}
806EXPORT_SYMBOL(ww_mutex_trylock);
807
808#ifdef CONFIG_DEBUG_LOCK_ALLOC
809void __sched
810mutex_lock_nested(struct mutex *lock, unsigned int subclass)
811{
812 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
813}
814
815EXPORT_SYMBOL_GPL(mutex_lock_nested);
816
817void __sched
818_mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
819{
820 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);
821}
822EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
823
824int __sched
825_mutex_lock_killable(struct mutex *lock, unsigned int subclass,
826 struct lockdep_map *nest)
827{
828 return __mutex_lock(lock, TASK_KILLABLE, subclass, nest, _RET_IP_);
829}
830EXPORT_SYMBOL_GPL(_mutex_lock_killable);
831
832int __sched
833mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
834{
835 return __mutex_lock(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);
836}
837EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
838
839void __sched
840mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)
841{
842 int token;
843
844 might_sleep();
845
846 token = io_schedule_prepare();
847 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
848 subclass, NULL, _RET_IP_, NULL, 0);
849 io_schedule_finish(token);
850}
851EXPORT_SYMBOL_GPL(mutex_lock_io_nested);
852
853static inline int
854ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
855{
856#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
857 unsigned tmp;
858
859 if (ctx->deadlock_inject_countdown-- == 0) {
860 tmp = ctx->deadlock_inject_interval;
861 if (tmp > UINT_MAX/4)
862 tmp = UINT_MAX;
863 else
864 tmp = tmp*2 + tmp + tmp/2;
865
866 ctx->deadlock_inject_interval = tmp;
867 ctx->deadlock_inject_countdown = tmp;
868 ctx->contending_lock = lock;
869
870 ww_mutex_unlock(lock);
871
872 return -EDEADLK;
873 }
874#endif
875
876 return 0;
877}
878
879int __sched
880ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
881{
882 int ret;
883
884 might_sleep();
885 ret = __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE,
886 0, _RET_IP_, ctx);
887 if (!ret && ctx && ctx->acquired > 1)
888 return ww_mutex_deadlock_injection(lock, ctx);
889
890 return ret;
891}
892EXPORT_SYMBOL_GPL(ww_mutex_lock);
893
894int __sched
895ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
896{
897 int ret;
898
899 might_sleep();
900 ret = __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE,
901 0, _RET_IP_, ctx);
902
903 if (!ret && ctx && ctx->acquired > 1)
904 return ww_mutex_deadlock_injection(lock, ctx);
905
906 return ret;
907}
908EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible);
909
910#endif
911
912/*
913 * Release the lock, slowpath:
914 */
915static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
916{
917 struct task_struct *next = NULL;
918 DEFINE_WAKE_Q(wake_q);
919 unsigned long owner;
920 unsigned long flags;
921
922 mutex_release(&lock->dep_map, ip);
923
924 /*
925 * Release the lock before (potentially) taking the spinlock such that
926 * other contenders can get on with things ASAP.
927 *
928 * Except when HANDOFF, in that case we must not clear the owner field,
929 * but instead set it to the top waiter.
930 */
931 owner = atomic_long_read(v: &lock->owner);
932 for (;;) {
933 MUTEX_WARN_ON(__owner_task(owner) != current);
934 MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
935
936 if (owner & MUTEX_FLAG_HANDOFF)
937 break;
938
939 if (atomic_long_try_cmpxchg_release(v: &lock->owner, old: &owner, new: __owner_flags(owner))) {
940 if (owner & MUTEX_FLAG_WAITERS)
941 break;
942
943 return;
944 }
945 }
946
947 raw_spin_lock_irqsave(&lock->wait_lock, flags);
948 debug_mutex_unlock(lock);
949 if (!list_empty(head: &lock->wait_list)) {
950 /* get the first entry from the wait-list: */
951 struct mutex_waiter *waiter =
952 list_first_entry(&lock->wait_list,
953 struct mutex_waiter, list);
954
955 next = waiter->task;
956
957 debug_mutex_wake_waiter(lock, waiter);
958 __clear_task_blocked_on(p: next, m: lock);
959 wake_q_add(head: &wake_q, task: next);
960 }
961
962 if (owner & MUTEX_FLAG_HANDOFF)
963 __mutex_handoff(lock, task: next);
964
965 raw_spin_unlock_irqrestore_wake(lock: &lock->wait_lock, flags, wake_q: &wake_q);
966}
967
968#ifndef CONFIG_DEBUG_LOCK_ALLOC
969/*
970 * Here come the less common (and hence less performance-critical) APIs:
971 * mutex_lock_interruptible() and mutex_trylock().
972 */
973static noinline int __sched
974__mutex_lock_killable_slowpath(struct mutex *lock);
975
976static noinline int __sched
977__mutex_lock_interruptible_slowpath(struct mutex *lock);
978
979/**
980 * mutex_lock_interruptible() - Acquire the mutex, interruptible by signals.
981 * @lock: The mutex to be acquired.
982 *
983 * Lock the mutex like mutex_lock(). If a signal is delivered while the
984 * process is sleeping, this function will return without acquiring the
985 * mutex.
986 *
987 * Context: Process context.
988 * Return: 0 if the lock was successfully acquired or %-EINTR if a
989 * signal arrived.
990 */
991int __sched mutex_lock_interruptible(struct mutex *lock)
992{
993 might_sleep();
994
995 if (__mutex_trylock_fast(lock))
996 return 0;
997
998 return __mutex_lock_interruptible_slowpath(lock);
999}
1000
1001EXPORT_SYMBOL(mutex_lock_interruptible);
1002
1003/**
1004 * mutex_lock_killable() - Acquire the mutex, interruptible by fatal signals.
1005 * @lock: The mutex to be acquired.
1006 *
1007 * Lock the mutex like mutex_lock(). If a signal which will be fatal to
1008 * the current process is delivered while the process is sleeping, this
1009 * function will return without acquiring the mutex.
1010 *
1011 * Context: Process context.
1012 * Return: 0 if the lock was successfully acquired or %-EINTR if a
1013 * fatal signal arrived.
1014 */
1015int __sched mutex_lock_killable(struct mutex *lock)
1016{
1017 might_sleep();
1018
1019 if (__mutex_trylock_fast(lock))
1020 return 0;
1021
1022 return __mutex_lock_killable_slowpath(lock);
1023}
1024EXPORT_SYMBOL(mutex_lock_killable);
1025
1026/**
1027 * mutex_lock_io() - Acquire the mutex and mark the process as waiting for I/O
1028 * @lock: The mutex to be acquired.
1029 *
1030 * Lock the mutex like mutex_lock(). While the task is waiting for this
1031 * mutex, it will be accounted as being in the IO wait state by the
1032 * scheduler.
1033 *
1034 * Context: Process context.
1035 */
1036void __sched mutex_lock_io(struct mutex *lock)
1037{
1038 int token;
1039
1040 token = io_schedule_prepare();
1041 mutex_lock(lock);
1042 io_schedule_finish(token);
1043}
1044EXPORT_SYMBOL_GPL(mutex_lock_io);
1045
1046static noinline void __sched
1047__mutex_lock_slowpath(struct mutex *lock)
1048{
1049 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass: 0, NULL, _RET_IP_);
1050}
1051
1052static noinline int __sched
1053__mutex_lock_killable_slowpath(struct mutex *lock)
1054{
1055 return __mutex_lock(lock, TASK_KILLABLE, subclass: 0, NULL, _RET_IP_);
1056}
1057
1058static noinline int __sched
1059__mutex_lock_interruptible_slowpath(struct mutex *lock)
1060{
1061 return __mutex_lock(lock, TASK_INTERRUPTIBLE, subclass: 0, NULL, _RET_IP_);
1062}
1063
1064static noinline int __sched
1065__ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1066{
1067 return __ww_mutex_lock(lock: &lock->base, TASK_UNINTERRUPTIBLE, subclass: 0,
1068 _RET_IP_, ww_ctx: ctx);
1069}
1070
1071static noinline int __sched
1072__ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
1073 struct ww_acquire_ctx *ctx)
1074{
1075 return __ww_mutex_lock(lock: &lock->base, TASK_INTERRUPTIBLE, subclass: 0,
1076 _RET_IP_, ww_ctx: ctx);
1077}
1078
1079#endif
1080
1081#ifndef CONFIG_DEBUG_LOCK_ALLOC
1082/**
1083 * mutex_trylock - try to acquire the mutex, without waiting
1084 * @lock: the mutex to be acquired
1085 *
1086 * Try to acquire the mutex atomically. Returns 1 if the mutex
1087 * has been acquired successfully, and 0 on contention.
1088 *
1089 * NOTE: this function follows the spin_trylock() convention, so
1090 * it is negated from the down_trylock() return values! Be careful
1091 * about this when converting semaphore users to mutexes.
1092 *
1093 * This function must not be used in interrupt context. The
1094 * mutex must be released by the same task that acquired it.
1095 */
1096int __sched mutex_trylock(struct mutex *lock)
1097{
1098 MUTEX_WARN_ON(lock->magic != lock);
1099 return __mutex_trylock(lock);
1100}
1101EXPORT_SYMBOL(mutex_trylock);
1102#else
1103int __sched _mutex_trylock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock)
1104{
1105 bool locked;
1106
1107 MUTEX_WARN_ON(lock->magic != lock);
1108 locked = __mutex_trylock(lock);
1109 if (locked)
1110 mutex_acquire_nest(&lock->dep_map, 0, 1, nest_lock, _RET_IP_);
1111
1112 return locked;
1113}
1114EXPORT_SYMBOL(_mutex_trylock_nest_lock);
1115#endif
1116
1117#ifndef CONFIG_DEBUG_LOCK_ALLOC
1118int __sched
1119ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1120{
1121 might_sleep();
1122
1123 if (__mutex_trylock_fast(lock: &lock->base)) {
1124 if (ctx)
1125 ww_mutex_set_context_fastpath(lock, ctx);
1126 return 0;
1127 }
1128
1129 return __ww_mutex_lock_slowpath(lock, ctx);
1130}
1131EXPORT_SYMBOL(ww_mutex_lock);
1132
1133int __sched
1134ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
1135{
1136 might_sleep();
1137
1138 if (__mutex_trylock_fast(lock: &lock->base)) {
1139 if (ctx)
1140 ww_mutex_set_context_fastpath(lock, ctx);
1141 return 0;
1142 }
1143
1144 return __ww_mutex_lock_interruptible_slowpath(lock, ctx);
1145}
1146EXPORT_SYMBOL(ww_mutex_lock_interruptible);
1147
1148#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
1149#endif /* !CONFIG_PREEMPT_RT */
1150
1151EXPORT_TRACEPOINT_SYMBOL_GPL(contention_begin);
1152EXPORT_TRACEPOINT_SYMBOL_GPL(contention_end);
1153
1154/**
1155 * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
1156 * @cnt: the atomic which we are to dec
1157 * @lock: the mutex to return holding if we dec to 0
1158 *
1159 * return true and hold lock if we dec to 0, return false otherwise
1160 */
1161int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
1162{
1163 /* dec if we can't possibly hit 0 */
1164 if (atomic_add_unless(v: cnt, a: -1, u: 1))
1165 return 0;
1166 /* we might hit 0, so take the lock */
1167 mutex_lock(lock);
1168 if (!atomic_dec_and_test(v: cnt)) {
1169 /* when we actually did the dec, we didn't hit 0 */
1170 mutex_unlock(lock);
1171 return 0;
1172 }
1173 /* we hit 0, and we hold the lock */
1174 return 1;
1175}
1176EXPORT_SYMBOL(atomic_dec_and_mutex_lock);
1177