Lines Matching +full:over +full:- +full:current
1 // SPDX-License-Identifier: GPL-2.0-only
33 * - The 'prio' field contains the priority of the context that owns the
34 * console. Only higher priority contexts are allowed to take over the
37 * - The 'cpu' field denotes on which CPU the console is locked. It is used
40 * taken over by a higher priority context, released, and taken on another
45 * - The 'req_prio' field is used by the handover approach to make the
46 * current owner aware that there is a context with a higher priority
49 * - The 'unsafe' field allows to take over the console in a safe way in the
55 * - The 'unsafe_takeover' field is set when a hostile takeover took the
57 * until re-initialized.
65 * when the current owner has lower priority and the console is in an
84 * 3) Unsafe hostile takeover allows to take over the lock even when the
99 * - Preference for higher priority contexts.
100 * - Protection of the panic CPU.
104 * - What is marked as an unsafe section.
105 * - Whether to spin-wait if there is already an owner and the console is
107 * - Whether to attempt an unsafe hostile takeover.
121 * nbcon_state_set - Helper function to set the console state
130 atomic_set(&ACCESS_PRIVATE(con, nbcon_state), new->atom); in nbcon_state_set()
134 * nbcon_state_read - Helper function to read the console state
140 state->atom = atomic_read(&ACCESS_PRIVATE(con, nbcon_state)); in nbcon_state_read()
144 * nbcon_state_try_cmpxchg() - Helper function for atomic_try_cmpxchg() on console state
154 return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom); in nbcon_state_try_cmpxchg()
158 * nbcon_seq_read - Read the current console sequence
171 * nbcon_seq_force - Force console sequence to a specific value
192 * nbcon_seq_try_update - Try to update the console sequence number
197 * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to
199 * nbcon_seq_force() was used or the current context no longer owns the
204 unsigned long nbcon_seq = __u64seq_to_ulseq(ctxt->seq); in nbcon_seq_try_update()
205 struct console *con = ctxt->console; in nbcon_seq_try_update()
209 ctxt->seq = new_seq; in nbcon_seq_try_update()
211 ctxt->seq = nbcon_seq_read(con); in nbcon_seq_try_update()
216 * nbcon_context_try_acquire_direct - Try to acquire directly
218 * @cur: The current console state
221 * the current owner has a lower priority and the console is in a safe state.
228 * -EPERM: A panic is in progress and this is not the panic CPU.
229 * Or the current owner or waiter has the same or higher
233 * -EBUSY: The current owner has a lower priority but the console
241 struct console *con = ctxt->console; in nbcon_context_try_acquire_direct()
247 * is critical that non-panic CPUs during panic are unable to in nbcon_context_try_acquire_direct()
253 return -EPERM; in nbcon_context_try_acquire_direct()
255 if (ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio) in nbcon_context_try_acquire_direct()
256 return -EPERM; in nbcon_context_try_acquire_direct()
258 if (cur->unsafe) in nbcon_context_try_acquire_direct()
259 return -EBUSY; in nbcon_context_try_acquire_direct()
265 WARN_ON_ONCE(cur->unsafe_takeover); in nbcon_context_try_acquire_direct()
267 new.atom = cur->atom; in nbcon_context_try_acquire_direct()
268 new.prio = ctxt->prio; in nbcon_context_try_acquire_direct()
270 new.unsafe = cur->unsafe_takeover; in nbcon_context_try_acquire_direct()
283 * - Only a context with a priority higher than the owner can become in nbcon_waiter_matches()
285 * - Only a context with a priority higher than the waiter can in nbcon_waiter_matches()
286 * directly take over the request. in nbcon_waiter_matches()
287 * - There are only three priorities. in nbcon_waiter_matches()
288 * - Only one CPU is allowed to request PANIC priority. in nbcon_waiter_matches()
289 * - Lower priorities are ignored during panic() until reboot. in nbcon_waiter_matches()
308 return (cur->req_prio == expected_prio); in nbcon_waiter_matches()
312 * nbcon_context_try_acquire_requested - Try to acquire after having
315 * @cur: The current console state
318 * It is called when the console is in an unsafe state. The current
326 * -EPERM: A panic is in progress and this is not the panic CPU
329 * -EBUSY: The console is still locked. The caller should
339 struct console *con = ctxt->console; in nbcon_context_try_acquire_requested()
344 return -EPERM; in nbcon_context_try_acquire_requested()
350 if (!nbcon_waiter_matches(cur, ctxt->prio)) in nbcon_context_try_acquire_requested()
351 return -EPERM; in nbcon_context_try_acquire_requested()
354 if (cur->prio != NBCON_PRIO_NONE) in nbcon_context_try_acquire_requested()
355 return -EBUSY; in nbcon_context_try_acquire_requested()
361 WARN_ON_ONCE(cur->unsafe); in nbcon_context_try_acquire_requested()
363 new.atom = cur->atom; in nbcon_context_try_acquire_requested()
364 new.prio = ctxt->prio; in nbcon_context_try_acquire_requested()
366 new.unsafe = cur->unsafe_takeover; in nbcon_context_try_acquire_requested()
372 * over by a higher priority context. in nbcon_context_try_acquire_requested()
374 WARN_ON_ONCE(nbcon_waiter_matches(cur, ctxt->prio)); in nbcon_context_try_acquire_requested()
375 return -EPERM; in nbcon_context_try_acquire_requested()
383 * nbcon_context_try_acquire_handover - Try to acquire via handover
385 * @cur: The current console state
388 * than the current owner and the console is in an unsafe state.
389 * It is the case when nbcon_context_try_acquire_direct() returns -EBUSY.
391 * The function sets "req_prio" field to make the current owner aware of
392 * the request. Then it waits until the current owner releases the console,
393 * or an even higher context takes over the request, or timeout expires.
395 * The current owner checks the "req_prio" field on exit from the unsafe
404 * -EPERM: A panic is in progress and this is not the panic CPU.
405 * Or a higher priority context has taken over the
408 * -EBUSY: The current owner is on the same CPU so that the hand
409 * shake could not work. Or the current owner is not
415 * -EAGAIN: @cur has changed when creating the handover request.
422 struct console *con = ctxt->console; in nbcon_context_try_acquire_handover()
425 int request_err = -EBUSY; in nbcon_context_try_acquire_handover()
429 * with -EBUSY. in nbcon_context_try_acquire_handover()
431 WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio); in nbcon_context_try_acquire_handover()
432 WARN_ON_ONCE(!cur->unsafe); in nbcon_context_try_acquire_handover()
435 if (cur->cpu == cpu) in nbcon_context_try_acquire_handover()
436 return -EBUSY; in nbcon_context_try_acquire_handover()
439 * Console stays unsafe after an unsafe takeover until re-initialized. in nbcon_context_try_acquire_handover()
442 if (cur->unsafe_takeover) in nbcon_context_try_acquire_handover()
443 return -EBUSY; in nbcon_context_try_acquire_handover()
446 if (ctxt->spinwait_max_us == 0) in nbcon_context_try_acquire_handover()
447 return -EBUSY; in nbcon_context_try_acquire_handover()
451 * the console directly when the current state has been modified. in nbcon_context_try_acquire_handover()
453 new.atom = cur->atom; in nbcon_context_try_acquire_handover()
454 new.req_prio = ctxt->prio; in nbcon_context_try_acquire_handover()
456 return -EAGAIN; in nbcon_context_try_acquire_handover()
458 cur->atom = new.atom; in nbcon_context_try_acquire_handover()
461 for (timeout = ctxt->spinwait_max_us; timeout >= 0; timeout--) { in nbcon_context_try_acquire_handover()
471 if (request_err == -EPERM) in nbcon_context_try_acquire_handover()
476 /* Re-read the state because some time has passed. */ in nbcon_context_try_acquire_handover()
484 * can only happen if a higher priority context has taken over in nbcon_context_try_acquire_handover()
487 if (!nbcon_waiter_matches(cur, ctxt->prio)) in nbcon_context_try_acquire_handover()
488 return -EPERM; in nbcon_context_try_acquire_handover()
491 new.atom = cur->atom; in nbcon_context_try_acquire_handover()
498 cur->atom = new.atom; in nbcon_context_try_acquire_handover()
513 * nbcon_context_try_acquire_hostile - Acquire via unsafe hostile takeover
515 * @cur: The current console state
522 * Return: 0 on success. -EPERM when not allowed by the context.
528 struct console *con = ctxt->console; in nbcon_context_try_acquire_hostile()
531 if (!ctxt->allow_unsafe_takeover) in nbcon_context_try_acquire_hostile()
532 return -EPERM; in nbcon_context_try_acquire_hostile()
535 if (WARN_ON_ONCE(ctxt->prio != NBCON_PRIO_PANIC)) in nbcon_context_try_acquire_hostile()
536 return -EPERM; in nbcon_context_try_acquire_hostile()
540 * -EBUSY in the right situation. in nbcon_context_try_acquire_hostile()
542 WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio); in nbcon_context_try_acquire_hostile()
543 WARN_ON_ONCE(cur->unsafe != true); in nbcon_context_try_acquire_hostile()
546 new.atom = cur->atom; in nbcon_context_try_acquire_hostile()
548 new.prio = ctxt->prio; in nbcon_context_try_acquire_hostile()
549 new.unsafe |= cur->unsafe_takeover; in nbcon_context_try_acquire_hostile()
550 new.unsafe_takeover |= cur->unsafe; in nbcon_context_try_acquire_hostile()
560 * nbcon_context_try_acquire - Try to acquire nbcon console
563 * Context: Under @ctxt->con->device_lock() or local_irq_save().
567 * caller should check the current console state to see if it is
574 struct console *con = ctxt->console; in nbcon_context_try_acquire()
581 if (err != -EBUSY) in nbcon_context_try_acquire()
585 if (err == -EAGAIN) in nbcon_context_try_acquire()
587 if (err != -EBUSY) in nbcon_context_try_acquire()
599 ctxt->pbufs = &panic_nbcon_pbufs; in nbcon_context_try_acquire()
601 ctxt->pbufs = con->pbufs; in nbcon_context_try_acquire()
604 ctxt->seq = nbcon_seq_read(ctxt->console); in nbcon_context_try_acquire()
623 * the driver-specific locking mechanism used in device_lock() in nbcon_owner_matches()
629 * 2. Another context takes over the lock with NBCON_PRIO_EMERGENCY in nbcon_owner_matches()
638 if (cur->prio != expected_prio) in nbcon_owner_matches()
641 if (cur->cpu != expected_cpu) in nbcon_owner_matches()
648 * nbcon_context_release - Release the console
654 struct console *con = ctxt->console; in nbcon_context_release()
661 if (!nbcon_owner_matches(&cur, cpu, ctxt->prio)) in nbcon_context_release()
675 ctxt->pbufs = NULL; in nbcon_context_release()
679 * nbcon_context_can_proceed - Check whether ownership can proceed
681 * @cur: The current console state
684 * ownership was handed over or taken.
709 if (!nbcon_owner_matches(cur, cpu, ctxt->prio)) in nbcon_context_can_proceed()
713 if (cur->req_prio == NBCON_PRIO_NONE) in nbcon_context_can_proceed()
722 if (cur->unsafe) in nbcon_context_can_proceed()
726 WARN_ON_ONCE(cur->req_prio <= cur->prio); in nbcon_context_can_proceed()
729 * Having a safe point for take over and eventually a few in nbcon_context_can_proceed()
732 * Release and hand over. in nbcon_context_can_proceed()
737 * It is not clear whether the waiter really took over ownership. The in nbcon_context_can_proceed()
750 * nbcon_can_proceed - Check whether ownership can proceed
754 * ownership was handed over or taken.
775 struct console *con = ctxt->console; in nbcon_can_proceed()
788 * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state
794 * over or taken.
808 struct console *con = ctxt->console; in __nbcon_context_update_unsafe()
838 struct console *con = ctxt->console; in nbcon_write_context_set_buf()
841 wctxt->outbuf = buf; in nbcon_write_context_set_buf()
842 wctxt->len = len; in nbcon_write_context_set_buf()
844 wctxt->unsafe_takeover = cur.unsafe_takeover; in nbcon_write_context_set_buf()
848 * nbcon_enter_unsafe - Enter an unsafe region in the driver
852 * ownership was handed over or taken.
872 * nbcon_exit_unsafe - Exit an unsafe region in the driver
876 * ownership was handed over or taken.
896 * nbcon_reacquire_nobuf - Reacquire a console after losing ownership
924 * nbcon_emit_next_record - Emit a record in the acquired context
929 * ownership was handed over or taken.
937 * When true is returned, @wctxt->ctxt.backlog indicates whether there are
943 struct console *con = ctxt->console; in nbcon_emit_next_record()
946 .pbufs = ctxt->pbufs, in nbcon_emit_next_record()
962 if (WARN_ON_ONCE((use_atomic && !con->write_atomic) || in nbcon_emit_next_record()
977 ctxt->backlog = printk_get_next_message(&pmsg, ctxt->seq, is_extended, true); in nbcon_emit_next_record()
978 if (!ctxt->backlog) in nbcon_emit_next_record()
982 * @con->dropped is not protected in case of an unsafe hostile in nbcon_emit_next_record()
986 con_dropped = data_race(READ_ONCE(con->dropped)); in nbcon_emit_next_record()
994 * has taken over ownership and is replaying the record. Prepend a in nbcon_emit_next_record()
1005 * value from the context that took over ownership. in nbcon_emit_next_record()
1023 nbcon_write_context_set_buf(wctxt, &pmsg.pbufs->outbuf[0], pmsg.outbuf_len); in nbcon_emit_next_record()
1026 con->write_atomic(con, wctxt); in nbcon_emit_next_record()
1028 con->write_thread(con, wctxt); in nbcon_emit_next_record()
1030 if (!wctxt->outbuf) { in nbcon_emit_next_record()
1062 WRITE_ONCE(con->dropped, dropped); in nbcon_emit_next_record()
1071 * nbcon_emit_one - Print one record for an nbcon console using the
1090 struct console *con = ctxt->console; in nbcon_emit_one()
1095 con->device_lock(con, &flags); in nbcon_emit_one()
1109 * handed over or taken over. In both cases the context is no in nbcon_emit_one()
1112 * The higher priority printing context takes over responsibility in nbcon_emit_one()
1120 ret = ctxt->backlog; in nbcon_emit_one()
1123 con->device_unlock(con, flags); in nbcon_emit_one()
1128 * nbcon_kthread_should_wakeup - Check whether a printer thread should wakeup
1152 ctxt->seq = nbcon_seq_read(con); in nbcon_kthread_should_wakeup()
1154 ret = prb_read_valid(prb, ctxt->seq, NULL); in nbcon_kthread_should_wakeup()
1162 * nbcon_kthread_func - The printer thread function
1190 rcuwait_wait_event(&con->rcuwait, in nbcon_kthread_func()
1222 * nbcon_irq_work - irq work to wake console printer thread
1250 * nbcon_kthreads_wake - Wake up printing threads using irq_work
1270 if (rcuwait_has_sleeper(&con->rcuwait)) in nbcon_kthreads_wake()
1271 irq_work_queue(&con->irq_work); in nbcon_kthreads_wake()
1277 * nbcon_kthread_stop - Stop a console printer thread
1284 if (!con->kthread) in nbcon_kthread_stop()
1287 kthread_stop(con->kthread); in nbcon_kthread_stop()
1288 con->kthread = NULL; in nbcon_kthread_stop()
1292 * nbcon_kthread_create - Create a console printer thread
1313 if (con->kthread) in nbcon_kthread_create()
1316 kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index); in nbcon_kthread_create()
1322 con->kthread = kt; in nbcon_kthread_create()
1328 sched_set_normal(con->kthread, -20); in nbcon_kthread_create()
1338 * nbcon_get_cpu_emergency_nesting - Get the per CPU emergency nesting pointer
1343 * the current CPU or to the init data during early boot.
1345 * The function is safe for reading per-CPU variables in any context because
1346 * preemption is disabled if the current CPU is in the emergency state. See
1363 * nbcon_get_default_prio - The appropriate nbcon priority to use for nbcon
1364 * printing on the current CPU
1370 * The function is safe for reading per-CPU data in any context because
1371 * preemption is disabled if the current CPU is in the emergency or panic
1389 * nbcon_legacy_emit_next_record - Print one record for an nbcon console
1392 * @handover: Will be set to true if a printk waiter has taken over the
1426 ctxt->console = con; in nbcon_legacy_emit_next_record()
1427 ctxt->prio = nbcon_get_default_prio(); in nbcon_legacy_emit_next_record()
1446 /* Non-atomic does not perform legacy spinning handovers. */ in nbcon_legacy_emit_next_record()
1454 * __nbcon_atomic_flush_pending_con - Flush specified nbcon console using its
1465 * -EPERM: Unable to acquire console ownership.
1467 * -EAGAIN: Another context took over ownership while printing.
1469 * -ENOENT: A record before @stop_seq is not available.
1472 * caller to try again when -EAGAIN was returned. When -EPERM is returned,
1473 * this context is not allowed to acquire the console. When -ENOENT is
1484 ctxt->console = con; in __nbcon_atomic_flush_pending_con()
1485 ctxt->spinwait_max_us = 2000; in __nbcon_atomic_flush_pending_con()
1486 ctxt->prio = nbcon_get_default_prio(); in __nbcon_atomic_flush_pending_con()
1487 ctxt->allow_unsafe_takeover = allow_unsafe_takeover; in __nbcon_atomic_flush_pending_con()
1490 return -EPERM; in __nbcon_atomic_flush_pending_con()
1495 * handed over or taken over. In both cases the context is no in __nbcon_atomic_flush_pending_con()
1499 return -EAGAIN; in __nbcon_atomic_flush_pending_con()
1501 if (!ctxt->backlog) { in __nbcon_atomic_flush_pending_con()
1504 err = -ENOENT; in __nbcon_atomic_flush_pending_con()
1514 * nbcon_atomic_flush_pending_con - Flush specified nbcon console using its
1546 * If there was a new owner (-EPERM, -EAGAIN), that context is in nbcon_atomic_flush_pending_con()
1549 * Do not wait for records not yet finalized (-ENOENT) to avoid a in nbcon_atomic_flush_pending_con()
1570 * __nbcon_atomic_flush_pending - Flush all nbcon consoles using their
1599 * nbcon_atomic_flush_pending - Flush all nbcon consoles using their
1613 * nbcon_atomic_flush_unsafe - Flush all nbcon consoles using their
1625 * nbcon_cpu_emergency_enter - Enter an emergency section where printk()
1644 * nbcon_cpu_emergency_exit - Exit an emergency section
1655 (*cpu_emergency_nesting)--; in nbcon_cpu_emergency_exit()
1661 * nbcon_alloc - Allocate and init the nbcon console specific data
1675 if (WARN_ON(!con->write_thread)) in nbcon_alloc()
1678 rcuwait_init(&con->rcuwait); in nbcon_alloc()
1679 init_irq_work(&con->irq_work, nbcon_irq_work); in nbcon_alloc()
1680 atomic_long_set(&ACCESS_PRIVATE(con, nbcon_prev_seq), -1UL); in nbcon_alloc()
1690 if (con->flags & CON_BOOT) { in nbcon_alloc()
1696 con->pbufs = &printk_shared_pbufs; in nbcon_alloc()
1698 con->pbufs = kmalloc(sizeof(*con->pbufs), GFP_KERNEL); in nbcon_alloc()
1699 if (!con->pbufs) { in nbcon_alloc()
1706 kfree(con->pbufs); in nbcon_alloc()
1707 con->pbufs = NULL; in nbcon_alloc()
1717 * nbcon_free - Free and cleanup the nbcon console specific data
1730 if (!(con->flags & CON_BOOT)) in nbcon_free()
1731 kfree(con->pbufs); in nbcon_free()
1733 con->pbufs = NULL; in nbcon_free()
1737 * nbcon_device_try_acquire - Try to acquire nbcon console and enter unsafe
1742 * @con->device_lock() including disabling migration.
1746 * mechasism to synchronize between console printing and non-printing
1749 * performing non-printing activities in order to synchronize against their
1762 ctxt->console = con; in nbcon_device_try_acquire()
1763 ctxt->prio = NBCON_PRIO_NORMAL; in nbcon_device_try_acquire()
1776 * nbcon_device_release - Exit unsafe section and release the nbcon console