1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* drivers/base/power/runtime.c - Helper functions for device runtime PM
4	*
5	* Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
6	* Copyright (C) 2010 Alan Stern <stern@rowland.harvard.edu>
7	*/
8	#include <linux/sched/mm.h>
9	#include <linux/ktime.h>
10	#include <linux/hrtimer.h>
11	#include <linux/export.h>
12	#include <linux/pm_runtime.h>
13	#include <linux/pm_wakeirq.h>
14	#include <linux/rculist.h>
15	#include <trace/events/rpm.h>
16
17	#include "../base.h"
18	#include "power.h"
19
20	typedef int (*pm_callback_t)(struct device *);
21
22	static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset)
23	{
24	pm_callback_t cb;
25	const struct dev_pm_ops *ops;
26
27	if (dev->pm_domain)
28	ops = &dev->pm_domain->ops;
29	else if (dev->type && dev->type->pm)
30	ops = dev->type->pm;
31	else if (dev->class && dev->class->pm)
32	ops = dev->class->pm;
33	else if (dev->bus && dev->bus->pm)
34	ops = dev->bus->pm;
35	else
36	ops = NULL;
37
38	if (ops)
39	cb = *(pm_callback_t *)((void *)ops + cb_offset);
40	else
41	cb = NULL;
42
43	if (!cb && dev->driver && dev->driver->pm)
44	cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset);
45
46	return cb;
47	}
48
49	#define RPM_GET_CALLBACK(dev, callback) \
50	__rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback))
51
52	static int rpm_resume(struct device *dev, int rpmflags);
53	static int rpm_suspend(struct device *dev, int rpmflags);
54
55	/**
56	* update_pm_runtime_accounting - Update the time accounting of power states
57	* @dev: Device to update the accounting for
58	*
59	* In order to be able to have time accounting of the various power states
60	* (as used by programs such as PowerTOP to show the effectiveness of runtime
61	* PM), we need to track the time spent in each state.
62	* update_pm_runtime_accounting must be called each time before the
63	* runtime_status field is updated, to account the time in the old state
64	* correctly.
65	*/
66	static void update_pm_runtime_accounting(struct device *dev)
67	{
68	u64 now, last, delta;
69
70	if (dev->power.disable_depth > 0)
71	return;
72
73	last = dev->power.accounting_timestamp;
74
75	now = ktime_get_mono_fast_ns();
76	dev->power.accounting_timestamp = now;
77
78	/*
79	* Because ktime_get_mono_fast_ns() is not monotonic during
80	* timekeeping updates, ensure that 'now' is after the last saved
81	* timesptamp.
82	*/
83	if (now < last)
84	return;
85
86	delta = now - last;
87
88	if (dev->power.runtime_status == RPM_SUSPENDED)
89	dev->power.suspended_time += delta;
90	else
91	dev->power.active_time += delta;
92	}
93
94	static void __update_runtime_status(struct device *dev, enum rpm_status status)
95	{
96	update_pm_runtime_accounting(dev);
97	trace_rpm_status(dev, status);
98	dev->power.runtime_status = status;
99	}
100
101	static u64 rpm_get_accounted_time(struct device *dev, bool suspended)
102	{
103	u64 time;
104	unsigned long flags;
105
106	spin_lock_irqsave(&dev->power.lock, flags);
107
108	update_pm_runtime_accounting(dev);
109	time = suspended ? dev->power.suspended_time : dev->power.active_time;
110
111	spin_unlock_irqrestore(lock: &dev->power.lock, flags);
112
113	return time;
114	}
115
116	u64 pm_runtime_active_time(struct device *dev)
117	{
118	return rpm_get_accounted_time(dev, suspended: false);
119	}
120
121	u64 pm_runtime_suspended_time(struct device *dev)
122	{
123	return rpm_get_accounted_time(dev, suspended: true);
124	}
125	EXPORT_SYMBOL_GPL(pm_runtime_suspended_time);
126
127	/**
128	* pm_runtime_deactivate_timer - Deactivate given device's suspend timer.
129	* @dev: Device to handle.
130	*/
131	static void pm_runtime_deactivate_timer(struct device *dev)
132	{
133	if (dev->power.timer_expires > 0) {
134	hrtimer_try_to_cancel(timer: &dev->power.suspend_timer);
135	dev->power.timer_expires = 0;
136	}
137	}
138
139	/**
140	* pm_runtime_cancel_pending - Deactivate suspend timer and cancel requests.
141	* @dev: Device to handle.
142	*/
143	static void pm_runtime_cancel_pending(struct device *dev)
144	{
145	pm_runtime_deactivate_timer(dev);
146	/*
147	* In case there's a request pending, make sure its work function will
148	* return without doing anything.
149	*/
150	dev->power.request = RPM_REQ_NONE;
151	}
152
153	/*
154	* pm_runtime_autosuspend_expiration - Get a device's autosuspend-delay expiration time.
155	* @dev: Device to handle.
156	*
157	* Compute the autosuspend-delay expiration time based on the device's
158	* power.last_busy time. If the delay has already expired or is disabled
159	* (negative) or the power.use_autosuspend flag isn't set, return 0.
160	* Otherwise return the expiration time in nanoseconds (adjusted to be nonzero).
161	*
162	* This function may be called either with or without dev->power.lock held.
163	* Either way it can be racy, since power.last_busy may be updated at any time.
164	*/
165	u64 pm_runtime_autosuspend_expiration(struct device *dev)
166	{
167	int autosuspend_delay;
168	u64 expires;
169
170	if (!dev->power.use_autosuspend)
171	return 0;
172
173	autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay);
174	if (autosuspend_delay < 0)
175	return 0;
176
177	expires = READ_ONCE(dev->power.last_busy);
178	expires += (u64)autosuspend_delay * NSEC_PER_MSEC;
179	if (expires > ktime_get_mono_fast_ns())
180	return expires; /* Expires in the future */
181
182	return 0;
183	}
184	EXPORT_SYMBOL_GPL(pm_runtime_autosuspend_expiration);
185
186	static int dev_memalloc_noio(struct device *dev, void *data)
187	{
188	return dev->power.memalloc_noio;
189	}
190
191	/*
192	* pm_runtime_set_memalloc_noio - Set a device's memalloc_noio flag.
193	* @dev: Device to handle.
194	* @enable: True for setting the flag and False for clearing the flag.
195	*
196	* Set the flag for all devices in the path from the device to the
197	* root device in the device tree if @enable is true, otherwise clear
198	* the flag for devices in the path whose siblings don't set the flag.
199	*
200	* The function should only be called by block device, or network
201	* device driver for solving the deadlock problem during runtime
202	* resume/suspend:
203	*
204	* If memory allocation with GFP_KERNEL is called inside runtime
205	* resume/suspend callback of any one of its ancestors(or the
206	* block device itself), the deadlock may be triggered inside the
207	* memory allocation since it might not complete until the block
208	* device becomes active and the involed page I/O finishes. The
209	* situation is pointed out first by Alan Stern. Network device
210	* are involved in iSCSI kind of situation.
211	*
212	* The lock of dev_hotplug_mutex is held in the function for handling
213	* hotplug race because pm_runtime_set_memalloc_noio() may be called
214	* in async probe().
215	*
216	* The function should be called between device_add() and device_del()
217	* on the affected device(block/network device).
218	*/
219	void pm_runtime_set_memalloc_noio(struct device *dev, bool enable)
220	{
221	static DEFINE_MUTEX(dev_hotplug_mutex);
222
223	mutex_lock(&dev_hotplug_mutex);
224	for (;;) {
225	bool enabled;
226
227	/* hold power lock since bitfield is not SMP-safe. */
228	spin_lock_irq(lock: &dev->power.lock);
229	enabled = dev->power.memalloc_noio;
230	dev->power.memalloc_noio = enable;
231	spin_unlock_irq(lock: &dev->power.lock);
232
233	/*
234	* not need to enable ancestors any more if the device
235	* has been enabled.
236	*/
237	if (enabled && enable)
238	break;
239
240	dev = dev->parent;
241
242	/*
243	* clear flag of the parent device only if all the
244	* children don't set the flag because ancestor's
245	* flag was set by any one of the descendants.
246	*/
247	if (!dev || (!enable &&
248	device_for_each_child(parent: dev, NULL, fn: dev_memalloc_noio)))
249	break;
250	}
251	mutex_unlock(lock: &dev_hotplug_mutex);
252	}
253	EXPORT_SYMBOL_GPL(pm_runtime_set_memalloc_noio);
254
255	/**
256	* rpm_check_suspend_allowed - Test whether a device may be suspended.
257	* @dev: Device to test.
258	*/
259	static int rpm_check_suspend_allowed(struct device *dev)
260	{
261	int retval = 0;
262
263	if (dev->power.runtime_error)
264	retval = -EINVAL;
265	else if (dev->power.disable_depth > 0)
266	retval = -EACCES;
267	else if (atomic_read(v: &dev->power.usage_count))
268	retval = -EAGAIN;
269	else if (!dev->power.ignore_children && atomic_read(v: &dev->power.child_count))
270	retval = -EBUSY;
271
272	/* Pending resume requests take precedence over suspends. */
273	else if ((dev->power.deferred_resume &&
274	dev->power.runtime_status == RPM_SUSPENDING) ||
275	(dev->power.request_pending && dev->power.request == RPM_REQ_RESUME))
276	retval = -EAGAIN;
277	else if (__dev_pm_qos_resume_latency(dev) == 0)
278	retval = -EPERM;
279	else if (dev->power.runtime_status == RPM_SUSPENDED)
280	retval = 1;
281
282	return retval;
283	}
284
285	static int rpm_get_suppliers(struct device *dev)
286	{
287	struct device_link *link;
288
289	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
290	device_links_read_lock_held()) {
291	int retval;
292
293	if (!(link->flags & DL_FLAG_PM_RUNTIME))
294	continue;
295
296	retval = pm_runtime_get_sync(dev: link->supplier);
297	/* Ignore suppliers with disabled runtime PM. */
298	if (retval < 0 && retval != -EACCES) {
299	pm_runtime_put_noidle(dev: link->supplier);
300	return retval;
301	}
302	refcount_inc(r: &link->rpm_active);
303	}
304	return 0;
305	}
306
307	/**
308	* pm_runtime_release_supplier - Drop references to device link's supplier.
309	* @link: Target device link.
310	*
311	* Drop all runtime PM references associated with @link to its supplier device.
312	*/
313	void pm_runtime_release_supplier(struct device_link *link)
314	{
315	struct device *supplier = link->supplier;
316
317	/*
318	* The additional power.usage_count check is a safety net in case
319	* the rpm_active refcount becomes saturated, in which case
320	* refcount_dec_not_one() would return true forever, but it is not
321	* strictly necessary.
322	*/
323	while (refcount_dec_not_one(r: &link->rpm_active) &&
324	atomic_read(v: &supplier->power.usage_count) > 0)
325	pm_runtime_put_noidle(dev: supplier);
326	}
327
328	static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend)
329	{
330	struct device_link *link;
331
332	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
333	device_links_read_lock_held()) {
334	pm_runtime_release_supplier(link);
335	if (try_to_suspend)
336	pm_request_idle(dev: link->supplier);
337	}
338	}
339
340	static void rpm_put_suppliers(struct device *dev)
341	{
342	__rpm_put_suppliers(dev, try_to_suspend: true);
343	}
344
345	static void rpm_suspend_suppliers(struct device *dev)
346	{
347	struct device_link *link;
348	int idx = device_links_read_lock();
349
350	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
351	device_links_read_lock_held())
352	pm_request_idle(dev: link->supplier);
353
354	device_links_read_unlock(idx);
355	}
356
357	/**
358	* __rpm_callback - Run a given runtime PM callback for a given device.
359	* @cb: Runtime PM callback to run.
360	* @dev: Device to run the callback for.
361	*/
362	static int __rpm_callback(int (*cb)(struct device *), struct device *dev)
363	__releases(&dev->power.lock) __acquires(&dev->power.lock)
364	{
365	int retval = 0, idx;
366	bool use_links = dev->power.links_count > 0;
367
368	if (dev->power.irq_safe) {
369	spin_unlock(lock: &dev->power.lock);
370	} else {
371	spin_unlock_irq(lock: &dev->power.lock);
372
373	/*
374	* Resume suppliers if necessary.
375	*
376	* The device's runtime PM status cannot change until this
377	* routine returns, so it is safe to read the status outside of
378	* the lock.
379	*/
380	if (use_links && dev->power.runtime_status == RPM_RESUMING) {
381	idx = device_links_read_lock();
382
383	retval = rpm_get_suppliers(dev);
384	if (retval) {
385	rpm_put_suppliers(dev);
386	goto fail;
387	}
388
389	device_links_read_unlock(idx);
390	}
391	}
392
393	if (cb)
394	retval = cb(dev);
395
396	if (dev->power.irq_safe) {
397	spin_lock(lock: &dev->power.lock);
398	} else {
399	/*
400	* If the device is suspending and the callback has returned
401	* success, drop the usage counters of the suppliers that have
402	* been reference counted on its resume.
403	*
404	* Do that if resume fails too.
405	*/
406	if (use_links &&
407	((dev->power.runtime_status == RPM_SUSPENDING && !retval) ||
408	(dev->power.runtime_status == RPM_RESUMING && retval))) {
409	idx = device_links_read_lock();
410
411	__rpm_put_suppliers(dev, try_to_suspend: false);
412
413	fail:
414	device_links_read_unlock(idx);
415	}
416
417	spin_lock_irq(lock: &dev->power.lock);
418	}
419
420	return retval;
421	}
422
423	/**
424	* rpm_callback - Run a given runtime PM callback for a given device.
425	* @cb: Runtime PM callback to run.
426	* @dev: Device to run the callback for.
427	*/
428	static int rpm_callback(int (*cb)(struct device *), struct device *dev)
429	{
430	int retval;
431
432	if (dev->power.memalloc_noio) {
433	unsigned int noio_flag;
434
435	/*
436	* Deadlock might be caused if memory allocation with
437	* GFP_KERNEL happens inside runtime_suspend and
438	* runtime_resume callbacks of one block device's
439	* ancestor or the block device itself. Network
440	* device might be thought as part of iSCSI block
441	* device, so network device and its ancestor should
442	* be marked as memalloc_noio too.
443	*/
444	noio_flag = memalloc_noio_save();
445	retval = __rpm_callback(cb, dev);
446	memalloc_noio_restore(flags: noio_flag);
447	} else {
448	retval = __rpm_callback(cb, dev);
449	}
450
451	/*
452	* Since -EACCES means that runtime PM is disabled for the given device,
453	* it should not be returned by runtime PM callbacks. If it is returned
454	* nevertheless, assume it to be a transient error and convert it to
455	* -EAGAIN.
456	*/
457	if (retval == -EACCES)
458	retval = -EAGAIN;
459
460	if (retval != -EAGAIN && retval != -EBUSY)
461	dev->power.runtime_error = retval;
462
463	return retval;
464	}
465
466	/**
467	* rpm_idle - Notify device bus type if the device can be suspended.
468	* @dev: Device to notify the bus type about.
469	* @rpmflags: Flag bits.
470	*
471	* Check if the device's runtime PM status allows it to be suspended. If
472	* another idle notification has been started earlier, return immediately. If
473	* the RPM_ASYNC flag is set then queue an idle-notification request; otherwise
474	* run the ->runtime_idle() callback directly. If the ->runtime_idle callback
475	* doesn't exist or if it returns 0, call rpm_suspend with the RPM_AUTO flag.
476	*
477	* This function must be called under dev->power.lock with interrupts disabled.
478	*/
479	static int rpm_idle(struct device *dev, int rpmflags)
480	{
481	int (*callback)(struct device *);
482	int retval;
483
484	trace_rpm_idle(dev, flags: rpmflags);
485	retval = rpm_check_suspend_allowed(dev);
486	if (retval < 0)
487	; /* Conditions are wrong. */
488
489	/* Idle notifications are allowed only in the RPM_ACTIVE state. */
490	else if (dev->power.runtime_status != RPM_ACTIVE)
491	retval = -EAGAIN;
492
493	/*
494	* Any pending request other than an idle notification takes
495	* precedence over us, except that the timer may be running.
496	*/
497	else if (dev->power.request_pending &&
498	dev->power.request > RPM_REQ_IDLE)
499	retval = -EAGAIN;
500
501	/* Act as though RPM_NOWAIT is always set. */
502	else if (dev->power.idle_notification)
503	retval = -EINPROGRESS;
504
505	if (retval)
506	goto out;
507
508	/* Pending requests need to be canceled. */
509	dev->power.request = RPM_REQ_NONE;
510
511	callback = RPM_GET_CALLBACK(dev, runtime_idle);
512
513	/* If no callback assume success. */
514	if (!callback || dev->power.no_callbacks)
515	goto out;
516
517	/* Carry out an asynchronous or a synchronous idle notification. */
518	if (rpmflags & RPM_ASYNC) {
519	dev->power.request = RPM_REQ_IDLE;
520	if (!dev->power.request_pending) {
521	dev->power.request_pending = true;
522	queue_work(wq: pm_wq, work: &dev->power.work);
523	}
524	trace_rpm_return_int(dev, _THIS_IP_, ret: 0);
525	return 0;
526	}
527
528	dev->power.idle_notification = true;
529
530	if (dev->power.irq_safe)
531	spin_unlock(lock: &dev->power.lock);
532	else
533	spin_unlock_irq(lock: &dev->power.lock);
534
535	retval = callback(dev);
536
537	if (dev->power.irq_safe)
538	spin_lock(lock: &dev->power.lock);
539	else
540	spin_lock_irq(lock: &dev->power.lock);
541
542	dev->power.idle_notification = false;
543	wake_up_all(&dev->power.wait_queue);
544
545	out:
546	trace_rpm_return_int(dev, _THIS_IP_, ret: retval);
547	return retval ? retval : rpm_suspend(dev, rpmflags: rpmflags | RPM_AUTO);
548	}
549
550	/**
551	* rpm_suspend - Carry out runtime suspend of given device.
552	* @dev: Device to suspend.
553	* @rpmflags: Flag bits.
554	*
555	* Check if the device's runtime PM status allows it to be suspended.
556	* Cancel a pending idle notification, autosuspend or suspend. If
557	* another suspend has been started earlier, either return immediately
558	* or wait for it to finish, depending on the RPM_NOWAIT and RPM_ASYNC
559	* flags. If the RPM_ASYNC flag is set then queue a suspend request;
560	* otherwise run the ->runtime_suspend() callback directly. When
561	* ->runtime_suspend succeeded, if a deferred resume was requested while
562	* the callback was running then carry it out, otherwise send an idle
563	* notification for its parent (if the suspend succeeded and both
564	* ignore_children of parent->power and irq_safe of dev->power are not set).
565	* If ->runtime_suspend failed with -EAGAIN or -EBUSY, and if the RPM_AUTO
566	* flag is set and the next autosuspend-delay expiration time is in the
567	* future, schedule another autosuspend attempt.
568	*
569	* This function must be called under dev->power.lock with interrupts disabled.
570	*/
571	static int rpm_suspend(struct device *dev, int rpmflags)
572	__releases(&dev->power.lock) __acquires(&dev->power.lock)
573	{
574	int (*callback)(struct device *);
575	struct device *parent = NULL;
576	int retval;
577
578	trace_rpm_suspend(dev, flags: rpmflags);
579
580	repeat:
581	retval = rpm_check_suspend_allowed(dev);
582	if (retval < 0)
583	goto out; /* Conditions are wrong. */
584
585	/* Synchronous suspends are not allowed in the RPM_RESUMING state. */
586	if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC))
587	retval = -EAGAIN;
588
589	if (retval)
590	goto out;
591
592	/* If the autosuspend_delay time hasn't expired yet, reschedule. */
593	if ((rpmflags & RPM_AUTO) && dev->power.runtime_status != RPM_SUSPENDING) {
594	u64 expires = pm_runtime_autosuspend_expiration(dev);
595
596	if (expires != 0) {
597	/* Pending requests need to be canceled. */
598	dev->power.request = RPM_REQ_NONE;
599
600	/*
601	* Optimization: If the timer is already running and is
602	* set to expire at or before the autosuspend delay,
603	* avoid the overhead of resetting it. Just let it
604	* expire; pm_suspend_timer_fn() will take care of the
605	* rest.
606	*/
607	if (!(dev->power.timer_expires &&
608	dev->power.timer_expires <= expires)) {
609	/*
610	* We add a slack of 25% to gather wakeups
611	* without sacrificing the granularity.
612	*/
613	u64 slack = (u64)READ_ONCE(dev->power.autosuspend_delay) *
614	(NSEC_PER_MSEC >> 2);
615
616	dev->power.timer_expires = expires;
617	hrtimer_start_range_ns(timer: &dev->power.suspend_timer,
618	tim: ns_to_ktime(ns: expires),
619	range_ns: slack,
620	mode: HRTIMER_MODE_ABS);
621	}
622	dev->power.timer_autosuspends = 1;
623	goto out;
624	}
625	}
626
627	/* Other scheduled or pending requests need to be canceled. */
628	pm_runtime_cancel_pending(dev);
629
630	if (dev->power.runtime_status == RPM_SUSPENDING) {
631	DEFINE_WAIT(wait);
632
633	if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
634	retval = -EINPROGRESS;
635	goto out;
636	}
637
638	if (dev->power.irq_safe) {
639	spin_unlock(lock: &dev->power.lock);
640
641	cpu_relax();
642
643	spin_lock(lock: &dev->power.lock);
644	goto repeat;
645	}
646
647	/* Wait for the other suspend running in parallel with us. */
648	for (;;) {
649	prepare_to_wait(wq_head: &dev->power.wait_queue, wq_entry: &wait,
650	TASK_UNINTERRUPTIBLE);
651	if (dev->power.runtime_status != RPM_SUSPENDING)
652	break;
653
654	spin_unlock_irq(lock: &dev->power.lock);
655
656	schedule();
657
658	spin_lock_irq(lock: &dev->power.lock);
659	}
660	finish_wait(wq_head: &dev->power.wait_queue, wq_entry: &wait);
661	goto repeat;
662	}
663
664	if (dev->power.no_callbacks)
665	goto no_callback; /* Assume success. */
666
667	/* Carry out an asynchronous or a synchronous suspend. */
668	if (rpmflags & RPM_ASYNC) {
669	dev->power.request = (rpmflags & RPM_AUTO) ?
670	RPM_REQ_AUTOSUSPEND : RPM_REQ_SUSPEND;
671	if (!dev->power.request_pending) {
672	dev->power.request_pending = true;
673	queue_work(wq: pm_wq, work: &dev->power.work);
674	}
675	goto out;
676	}
677
678	__update_runtime_status(dev, status: RPM_SUSPENDING);
679
680	callback = RPM_GET_CALLBACK(dev, runtime_suspend);
681
682	dev_pm_enable_wake_irq_check(dev, can_change_status: true);
683	retval = rpm_callback(cb: callback, dev);
684	if (retval)
685	goto fail;
686
687	dev_pm_enable_wake_irq_complete(dev);
688
689	no_callback:
690	__update_runtime_status(dev, status: RPM_SUSPENDED);
691	pm_runtime_deactivate_timer(dev);
692
693	if (dev->parent) {
694	parent = dev->parent;
695	atomic_add_unless(v: &parent->power.child_count, a: -1, u: 0);
696	}
697	wake_up_all(&dev->power.wait_queue);
698
699	if (dev->power.deferred_resume) {
700	dev->power.deferred_resume = false;
701	rpm_resume(dev, rpmflags: 0);
702	retval = -EAGAIN;
703	goto out;
704	}
705
706	if (dev->power.irq_safe)
707	goto out;
708
709	/* Maybe the parent is now able to suspend. */
710	if (parent && !parent->power.ignore_children) {
711	spin_unlock(lock: &dev->power.lock);
712
713	spin_lock(lock: &parent->power.lock);
714	rpm_idle(dev: parent, RPM_ASYNC);
715	spin_unlock(lock: &parent->power.lock);
716
717	spin_lock(lock: &dev->power.lock);
718	}
719	/* Maybe the suppliers are now able to suspend. */
720	if (dev->power.links_count > 0) {
721	spin_unlock_irq(lock: &dev->power.lock);
722
723	rpm_suspend_suppliers(dev);
724
725	spin_lock_irq(lock: &dev->power.lock);
726	}
727
728	out:
729	trace_rpm_return_int(dev, _THIS_IP_, ret: retval);
730
731	return retval;
732
733	fail:
734	dev_pm_disable_wake_irq_check(dev, cond_disable: true);
735	__update_runtime_status(dev, status: RPM_ACTIVE);
736	dev->power.deferred_resume = false;
737	wake_up_all(&dev->power.wait_queue);
738
739	/*
740	* On transient errors, if the callback routine failed an autosuspend,
741	* and if the last_busy time has been updated so that there is a new
742	* autosuspend expiration time, automatically reschedule another
743	* autosuspend.
744	*/
745	if (!dev->power.runtime_error && (rpmflags & RPM_AUTO) &&
746	pm_runtime_autosuspend_expiration(dev) != 0)
747	goto repeat;
748
749	pm_runtime_cancel_pending(dev);
750
751	goto out;
752	}
753
754	/**
755	* rpm_resume - Carry out runtime resume of given device.
756	* @dev: Device to resume.
757	* @rpmflags: Flag bits.
758	*
759	* Check if the device's runtime PM status allows it to be resumed. Cancel
760	* any scheduled or pending requests. If another resume has been started
761	* earlier, either return immediately or wait for it to finish, depending on the
762	* RPM_NOWAIT and RPM_ASYNC flags. Similarly, if there's a suspend running in
763	* parallel with this function, either tell the other process to resume after
764	* suspending (deferred_resume) or wait for it to finish. If the RPM_ASYNC
765	* flag is set then queue a resume request; otherwise run the
766	* ->runtime_resume() callback directly. Queue an idle notification for the
767	* device if the resume succeeded.
768	*
769	* This function must be called under dev->power.lock with interrupts disabled.
770	*/
771	static int rpm_resume(struct device *dev, int rpmflags)
772	__releases(&dev->power.lock) __acquires(&dev->power.lock)
773	{
774	int (*callback)(struct device *);
775	struct device *parent = NULL;
776	int retval = 0;
777
778	trace_rpm_resume(dev, flags: rpmflags);
779
780	repeat:
781	if (dev->power.runtime_error) {
782	retval = -EINVAL;
783	} else if (dev->power.disable_depth > 0) {
784	if (dev->power.runtime_status == RPM_ACTIVE &&
785	dev->power.last_status == RPM_ACTIVE)
786	retval = 1;
787	else
788	retval = -EACCES;
789	}
790	if (retval)
791	goto out;
792
793	/*
794	* Other scheduled or pending requests need to be canceled. Small
795	* optimization: If an autosuspend timer is running, leave it running
796	* rather than cancelling it now only to restart it again in the near
797	* future.
798	*/
799	dev->power.request = RPM_REQ_NONE;
800	if (!dev->power.timer_autosuspends)
801	pm_runtime_deactivate_timer(dev);
802
803	if (dev->power.runtime_status == RPM_ACTIVE) {
804	retval = 1;
805	goto out;
806	}
807
808	if (dev->power.runtime_status == RPM_RESUMING ||
809	dev->power.runtime_status == RPM_SUSPENDING) {
810	DEFINE_WAIT(wait);
811
812	if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
813	if (dev->power.runtime_status == RPM_SUSPENDING) {
814	dev->power.deferred_resume = true;
815	if (rpmflags & RPM_NOWAIT)
816	retval = -EINPROGRESS;
817	} else {
818	retval = -EINPROGRESS;
819	}
820	goto out;
821	}
822
823	if (dev->power.irq_safe) {
824	spin_unlock(lock: &dev->power.lock);
825
826	cpu_relax();
827
828	spin_lock(lock: &dev->power.lock);
829	goto repeat;
830	}
831
832	/* Wait for the operation carried out in parallel with us. */
833	for (;;) {
834	prepare_to_wait(wq_head: &dev->power.wait_queue, wq_entry: &wait,
835	TASK_UNINTERRUPTIBLE);
836	if (dev->power.runtime_status != RPM_RESUMING &&
837	dev->power.runtime_status != RPM_SUSPENDING)
838	break;
839
840	spin_unlock_irq(lock: &dev->power.lock);
841
842	schedule();
843
844	spin_lock_irq(lock: &dev->power.lock);
845	}
846	finish_wait(wq_head: &dev->power.wait_queue, wq_entry: &wait);
847	goto repeat;
848	}
849
850	/*
851	* See if we can skip waking up the parent. This is safe only if
852	* power.no_callbacks is set, because otherwise we don't know whether
853	* the resume will actually succeed.
854	*/
855	if (dev->power.no_callbacks && !parent && dev->parent) {
856	spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);
857	if (dev->parent->power.disable_depth > 0 ||
858	dev->parent->power.ignore_children ||
859	dev->parent->power.runtime_status == RPM_ACTIVE) {
860	atomic_inc(v: &dev->parent->power.child_count);
861	spin_unlock(lock: &dev->parent->power.lock);
862	retval = 1;
863	goto no_callback; /* Assume success. */
864	}
865	spin_unlock(lock: &dev->parent->power.lock);
866	}
867
868	/* Carry out an asynchronous or a synchronous resume. */
869	if (rpmflags & RPM_ASYNC) {
870	dev->power.request = RPM_REQ_RESUME;
871	if (!dev->power.request_pending) {
872	dev->power.request_pending = true;
873	queue_work(wq: pm_wq, work: &dev->power.work);
874	}
875	retval = 0;
876	goto out;
877	}
878
879	if (!parent && dev->parent) {
880	/*
881	* Increment the parent's usage counter and resume it if
882	* necessary. Not needed if dev is irq-safe; then the
883	* parent is permanently resumed.
884	*/
885	parent = dev->parent;
886	if (dev->power.irq_safe)
887	goto skip_parent;
888
889	spin_unlock(lock: &dev->power.lock);
890
891	pm_runtime_get_noresume(dev: parent);
892
893	spin_lock(lock: &parent->power.lock);
894	/*
895	* Resume the parent if it has runtime PM enabled and not been
896	* set to ignore its children.
897	*/
898	if (!parent->power.disable_depth &&
899	!parent->power.ignore_children) {
900	rpm_resume(dev: parent, rpmflags: 0);
901	if (parent->power.runtime_status != RPM_ACTIVE)
902	retval = -EBUSY;
903	}
904	spin_unlock(lock: &parent->power.lock);
905
906	spin_lock(lock: &dev->power.lock);
907	if (retval)
908	goto out;
909
910	goto repeat;
911	}
912	skip_parent:
913
914	if (dev->power.no_callbacks)
915	goto no_callback; /* Assume success. */
916
917	__update_runtime_status(dev, status: RPM_RESUMING);
918
919	callback = RPM_GET_CALLBACK(dev, runtime_resume);
920
921	dev_pm_disable_wake_irq_check(dev, cond_disable: false);
922	retval = rpm_callback(cb: callback, dev);
923	if (retval) {
924	__update_runtime_status(dev, status: RPM_SUSPENDED);
925	pm_runtime_cancel_pending(dev);
926	dev_pm_enable_wake_irq_check(dev, can_change_status: false);
927	} else {
928	no_callback:
929	__update_runtime_status(dev, status: RPM_ACTIVE);
930	pm_runtime_mark_last_busy(dev);
931	if (parent)
932	atomic_inc(v: &parent->power.child_count);
933	}
934	wake_up_all(&dev->power.wait_queue);
935
936	if (retval >= 0)
937	rpm_idle(dev, RPM_ASYNC);
938
939	out:
940	if (parent && !dev->power.irq_safe) {
941	spin_unlock_irq(lock: &dev->power.lock);
942
943	pm_runtime_put(dev: parent);
944
945	spin_lock_irq(lock: &dev->power.lock);
946	}
947
948	trace_rpm_return_int(dev, _THIS_IP_, ret: retval);
949
950	return retval;
951	}
952
953	/**
954	* pm_runtime_work - Universal runtime PM work function.
955	* @work: Work structure used for scheduling the execution of this function.
956	*
957	* Use @work to get the device object the work is to be done for, determine what
958	* is to be done and execute the appropriate runtime PM function.
959	*/
960	static void pm_runtime_work(struct work_struct *work)
961	{
962	struct device *dev = container_of(work, struct device, power.work);
963	enum rpm_request req;
964
965	spin_lock_irq(lock: &dev->power.lock);
966
967	if (!dev->power.request_pending)
968	goto out;
969
970	req = dev->power.request;
971	dev->power.request = RPM_REQ_NONE;
972	dev->power.request_pending = false;
973
974	switch (req) {
975	case RPM_REQ_NONE:
976	break;
977	case RPM_REQ_IDLE:
978	rpm_idle(dev, RPM_NOWAIT);
979	break;
980	case RPM_REQ_SUSPEND:
981	rpm_suspend(dev, RPM_NOWAIT);
982	break;
983	case RPM_REQ_AUTOSUSPEND:
984	rpm_suspend(dev, RPM_NOWAIT | RPM_AUTO);
985	break;
986	case RPM_REQ_RESUME:
987	rpm_resume(dev, RPM_NOWAIT);
988	break;
989	}
990
991	out:
992	spin_unlock_irq(lock: &dev->power.lock);
993	}
994
995	/**
996	* pm_suspend_timer_fn - Timer function for pm_schedule_suspend().
997	* @timer: hrtimer used by pm_schedule_suspend().
998	*
999	* Check if the time is right and queue a suspend request.
1000	*/
1001	static enum hrtimer_restart pm_suspend_timer_fn(struct hrtimer *timer)
1002	{
1003	struct device *dev = container_of(timer, struct device, power.suspend_timer);
1004	unsigned long flags;
1005	u64 expires;
1006
1007	spin_lock_irqsave(&dev->power.lock, flags);
1008
1009	expires = dev->power.timer_expires;
1010	/*
1011	* If 'expires' is after the current time, we've been called
1012	* too early.
1013	*/
1014	if (expires > 0 && expires <= ktime_get_mono_fast_ns()) {
1015	dev->power.timer_expires = 0;
1016	rpm_suspend(dev, rpmflags: dev->power.timer_autosuspends ?
1017	(RPM_ASYNC | RPM_AUTO) : RPM_ASYNC);
1018	}
1019
1020	spin_unlock_irqrestore(lock: &dev->power.lock, flags);
1021
1022	return HRTIMER_NORESTART;
1023	}
1024
1025	/**
1026	* pm_schedule_suspend - Set up a timer to submit a suspend request in future.
1027	* @dev: Device to suspend.
1028	* @delay: Time to wait before submitting a suspend request, in milliseconds.
1029	*/
1030	int pm_schedule_suspend(struct device *dev, unsigned int delay)
1031	{
1032	unsigned long flags;
1033	u64 expires;
1034	int retval;
1035
1036	spin_lock_irqsave(&dev->power.lock, flags);
1037
1038	if (!delay) {
1039	retval = rpm_suspend(dev, RPM_ASYNC);
1040	goto out;
1041	}
1042
1043	retval = rpm_check_suspend_allowed(dev);
1044	if (retval)
1045	goto out;
1046
1047	/* Other scheduled or pending requests need to be canceled. */
1048	pm_runtime_cancel_pending(dev);
1049
1050	expires = ktime_get_mono_fast_ns() + (u64)delay * NSEC_PER_MSEC;
1051	dev->power.timer_expires = expires;
1052	dev->power.timer_autosuspends = 0;
1053	hrtimer_start(timer: &dev->power.suspend_timer, tim: expires, mode: HRTIMER_MODE_ABS);
1054
1055	out:
1056	spin_unlock_irqrestore(lock: &dev->power.lock, flags);
1057
1058	return retval;
1059	}
1060	EXPORT_SYMBOL_GPL(pm_schedule_suspend);
1061
1062	static int rpm_drop_usage_count(struct device *dev)
1063	{
1064	int ret;
1065
1066	ret = atomic_sub_return(i: 1, v: &dev->power.usage_count);
1067	if (ret >= 0)
1068	return ret;
1069
1070	/*
1071	* Because rpm_resume() does not check the usage counter, it will resume
1072	* the device even if the usage counter is 0 or negative, so it is
1073	* sufficient to increment the usage counter here to reverse the change
1074	* made above.
1075	*/
1076	atomic_inc(v: &dev->power.usage_count);
1077	dev_warn(dev, "Runtime PM usage count underflow!\n");
1078	return -EINVAL;
1079	}
1080
1081	/**
1082	* __pm_runtime_idle - Entry point for runtime idle operations.
1083	* @dev: Device to send idle notification for.
1084	* @rpmflags: Flag bits.
1085	*
1086	* If the RPM_GET_PUT flag is set, decrement the device's usage count and
1087	* return immediately if it is larger than zero (if it becomes negative, log a
1088	* warning, increment it, and return an error). Then carry out an idle
1089	* notification, either synchronous or asynchronous.
1090	*
1091	* This routine may be called in atomic context if the RPM_ASYNC flag is set,
1092	* or if pm_runtime_irq_safe() has been called.
1093	*/
1094	int __pm_runtime_idle(struct device *dev, int rpmflags)
1095	{
1096	unsigned long flags;
1097	int retval;
1098
1099	if (rpmflags & RPM_GET_PUT) {
1100	retval = rpm_drop_usage_count(dev);
1101	if (retval < 0) {
1102	return retval;
1103	} else if (retval > 0) {
1104	trace_rpm_usage(dev, flags: rpmflags);
1105	return 0;
1106	}
1107	}
1108
1109	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
1110
1111	spin_lock_irqsave(&dev->power.lock, flags);
1112	retval = rpm_idle(dev, rpmflags);
1113	spin_unlock_irqrestore(lock: &dev->power.lock, flags);
1114
1115	return retval;
1116	}
1117	EXPORT_SYMBOL_GPL(__pm_runtime_idle);
1118
1119	/**
1120	* __pm_runtime_suspend - Entry point for runtime put/suspend operations.
1121	* @dev: Device to suspend.
1122	* @rpmflags: Flag bits.
1123	*
1124	* If the RPM_GET_PUT flag is set, decrement the device's usage count and
1125	* return immediately if it is larger than zero (if it becomes negative, log a
1126	* warning, increment it, and return an error). Then carry out a suspend,
1127	* either synchronous or asynchronous.
1128	*
1129	* This routine may be called in atomic context if the RPM_ASYNC flag is set,
1130	* or if pm_runtime_irq_safe() has been called.
1131	*/
1132	int __pm_runtime_suspend(struct device *dev, int rpmflags)
1133	{
1134	unsigned long flags;
1135	int retval;
1136
1137	if (rpmflags & RPM_GET_PUT) {
1138	retval = rpm_drop_usage_count(dev);
1139	if (retval < 0) {
1140	return retval;
1141	} else if (retval > 0) {
1142	trace_rpm_usage(dev, flags: rpmflags);
1143	return 0;
1144	}
1145	}
1146
1147	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
1148
1149	spin_lock_irqsave(&dev->power.lock, flags);
1150	retval = rpm_suspend(dev, rpmflags);
1151	spin_unlock_irqrestore(lock: &dev->power.lock, flags);
1152
1153	return retval;
1154	}
1155	EXPORT_SYMBOL_GPL(__pm_runtime_suspend);
1156
1157	/**
1158	* __pm_runtime_resume - Entry point for runtime resume operations.
1159	* @dev: Device to resume.
1160	* @rpmflags: Flag bits.
1161	*
1162	* If the RPM_GET_PUT flag is set, increment the device's usage count. Then
1163	* carry out a resume, either synchronous or asynchronous.
1164	*
1165	* This routine may be called in atomic context if the RPM_ASYNC flag is set,
1166	* or if pm_runtime_irq_safe() has been called.
1167	*/
1168	int __pm_runtime_resume(struct device *dev, int rpmflags)
1169	{
1170	unsigned long flags;
1171	int retval;
1172
1173	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe &&
1174	dev->power.runtime_status != RPM_ACTIVE);
1175
1176	if (rpmflags & RPM_GET_PUT)
1177	atomic_inc(v: &dev->power.usage_count);
1178
1179	spin_lock_irqsave(&dev->power.lock, flags);
1180	retval = rpm_resume(dev, rpmflags);
1181	spin_unlock_irqrestore(lock: &dev->power.lock, flags);
1182
1183	return retval;
1184	}
1185	EXPORT_SYMBOL_GPL(__pm_runtime_resume);
1186
1187	/**
1188	* pm_runtime_get_conditional - Conditionally bump up device usage counter.
1189	* @dev: Device to handle.
1190	* @ign_usage_count: Whether or not to look at the current usage counter value.
1191	*
1192	* Return -EINVAL if runtime PM is disabled for @dev.
1193	*
1194	* Otherwise, if the runtime PM status of @dev is %RPM_ACTIVE and either
1195	* @ign_usage_count is %true or the runtime PM usage counter of @dev is not
1196	* zero, increment the usage counter of @dev and return 1. Otherwise, return 0
1197	* without changing the usage counter.
1198	*
1199	* If @ign_usage_count is %true, this function can be used to prevent suspending
1200	* the device when its runtime PM status is %RPM_ACTIVE.
1201	*
1202	* If @ign_usage_count is %false, this function can be used to prevent
1203	* suspending the device when both its runtime PM status is %RPM_ACTIVE and its
1204	* runtime PM usage counter is not zero.
1205	*
1206	* The caller is responsible for decrementing the runtime PM usage counter of
1207	* @dev after this function has returned a positive value for it.
1208	*/
1209	static int pm_runtime_get_conditional(struct device *dev, bool ign_usage_count)
1210	{
1211	unsigned long flags;
1212	int retval;
1213
1214	spin_lock_irqsave(&dev->power.lock, flags);
1215	if (dev->power.disable_depth > 0) {
1216	retval = -EINVAL;
1217	} else if (dev->power.runtime_status != RPM_ACTIVE) {
1218	retval = 0;
1219	} else if (ign_usage_count) {
1220	retval = 1;
1221	atomic_inc(v: &dev->power.usage_count);
1222	} else {
1223	retval = atomic_inc_not_zero(v: &dev->power.usage_count);
1224	}
1225	trace_rpm_usage(dev, flags: 0);
1226	spin_unlock_irqrestore(lock: &dev->power.lock, flags);
1227
1228	return retval;
1229	}
1230
1231	/**
1232	* pm_runtime_get_if_active - Bump up runtime PM usage counter if the device is
1233	* in active state
1234	* @dev: Target device.
1235	*
1236	* Increment the runtime PM usage counter of @dev if its runtime PM status is
1237	* %RPM_ACTIVE, in which case it returns 1. If the device is in a different
1238	* state, 0 is returned. -EINVAL is returned if runtime PM is disabled for the
1239	* device, in which case also the usage_count will remain unmodified.
1240	*/
1241	int pm_runtime_get_if_active(struct device *dev)
1242	{
1243	return pm_runtime_get_conditional(dev, ign_usage_count: true);
1244	}
1245	EXPORT_SYMBOL_GPL(pm_runtime_get_if_active);
1246
1247	/**
1248	* pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter.
1249	* @dev: Target device.
1250	*
1251	* Increment the runtime PM usage counter of @dev if its runtime PM status is
1252	* %RPM_ACTIVE and its runtime PM usage counter is greater than 0, in which case
1253	* it returns 1. If the device is in a different state or its usage_count is 0,
1254	* 0 is returned. -EINVAL is returned if runtime PM is disabled for the device,
1255	* in which case also the usage_count will remain unmodified.
1256	*/
1257	int pm_runtime_get_if_in_use(struct device *dev)
1258	{
1259	return pm_runtime_get_conditional(dev, ign_usage_count: false);
1260	}
1261	EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use);
1262
1263	/**
1264	* __pm_runtime_set_status - Set runtime PM status of a device.
1265	* @dev: Device to handle.
1266	* @status: New runtime PM status of the device.
1267	*
1268	* If runtime PM of the device is disabled or its power.runtime_error field is
1269	* different from zero, the status may be changed either to RPM_ACTIVE, or to
1270	* RPM_SUSPENDED, as long as that reflects the actual state of the device.
1271	* However, if the device has a parent and the parent is not active, and the
1272	* parent's power.ignore_children flag is unset, the device's status cannot be
1273	* set to RPM_ACTIVE, so -EBUSY is returned in that case.
1274	*
1275	* If successful, __pm_runtime_set_status() clears the power.runtime_error field
1276	* and the device parent's counter of unsuspended children is modified to
1277	* reflect the new status. If the new status is RPM_SUSPENDED, an idle
1278	* notification request for the parent is submitted.
1279	*
1280	* If @dev has any suppliers (as reflected by device links to them), and @status
1281	* is RPM_ACTIVE, they will be activated upfront and if the activation of one
1282	* of them fails, the status of @dev will be changed to RPM_SUSPENDED (instead
1283	* of the @status value) and the suppliers will be deacticated on exit. The
1284	* error returned by the failing supplier activation will be returned in that
1285	* case.
1286	*/
1287	int __pm_runtime_set_status(struct device *dev, unsigned int status)
1288	{
1289	struct device *parent = dev->parent;
1290	bool notify_parent = false;
1291	unsigned long flags;
1292	int error = 0;
1293
1294	if (status != RPM_ACTIVE && status != RPM_SUSPENDED)
1295	return -EINVAL;
1296
1297	spin_lock_irqsave(&dev->power.lock, flags);
1298
1299	/*
1300	* Prevent PM-runtime from being enabled for the device or return an
1301	* error if it is enabled already and working.
1302	*/
1303	if (dev->power.runtime_error || dev->power.disable_depth)
1304	dev->power.disable_depth++;
1305	else
1306	error = -EAGAIN;
1307
1308	spin_unlock_irqrestore(lock: &dev->power.lock, flags);
1309
1310	if (error)
1311	return error;
1312
1313	/*
1314	* If the new status is RPM_ACTIVE, the suppliers can be activated
1315	* upfront regardless of the current status, because next time
1316	* rpm_put_suppliers() runs, the rpm_active refcounts of the links
1317	* involved will be dropped down to one anyway.
1318	*/
1319	if (status == RPM_ACTIVE) {
1320	int idx = device_links_read_lock();
1321
1322	error = rpm_get_suppliers(dev);
1323	if (error)
1324	status = RPM_SUSPENDED;
1325
1326	device_links_read_unlock(idx);
1327	}
1328
1329	spin_lock_irqsave(&dev->power.lock, flags);
1330
1331	if (dev->power.runtime_status == status || !parent)
1332	goto out_set;
1333
1334	if (status == RPM_SUSPENDED) {
1335	atomic_add_unless(v: &parent->power.child_count, a: -1, u: 0);
1336	notify_parent = !parent->power.ignore_children;
1337	} else {
1338	spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING);
1339
1340	/*
1341	* It is invalid to put an active child under a parent that is
1342	* not active, has runtime PM enabled and the
1343	* 'power.ignore_children' flag unset.
1344	*/
1345	if (!parent->power.disable_depth &&
1346	!parent->power.ignore_children &&
1347	parent->power.runtime_status != RPM_ACTIVE) {
1348	dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n",
1349	dev_name(dev),
1350	dev_name(parent));
1351	error = -EBUSY;
1352	} else if (dev->power.runtime_status == RPM_SUSPENDED) {
1353	atomic_inc(v: &parent->power.child_count);
1354	}
1355
1356	spin_unlock(lock: &parent->power.lock);
1357
1358	if (error) {
1359	status = RPM_SUSPENDED;
1360	goto out;
1361	}
1362	}
1363
1364	out_set:
1365	__update_runtime_status(dev, status);
1366	if (!error)
1367	dev->power.runtime_error = 0;
1368
1369	out:
1370	spin_unlock_irqrestore(lock: &dev->power.lock, flags);
1371
1372	if (notify_parent)
1373	pm_request_idle(dev: parent);
1374
1375	if (status == RPM_SUSPENDED) {
1376	int idx = device_links_read_lock();
1377
1378	rpm_put_suppliers(dev);
1379
1380	device_links_read_unlock(idx);
1381	}
1382
1383	pm_runtime_enable(dev);
1384
1385	return error;
1386	}
1387	EXPORT_SYMBOL_GPL(__pm_runtime_set_status);
1388
1389	/**
1390	* __pm_runtime_barrier - Cancel pending requests and wait for completions.
1391	* @dev: Device to handle.
1392	*
1393	* Flush all pending requests for the device from pm_wq and wait for all
1394	* runtime PM operations involving the device in progress to complete.
1395	*
1396	* Should be called under dev->power.lock with interrupts disabled.
1397	*/
1398	static void __pm_runtime_barrier(struct device *dev)
1399	{
1400	pm_runtime_deactivate_timer(dev);
1401
1402	if (dev->power.request_pending) {
1403	dev->power.request = RPM_REQ_NONE;
1404	spin_unlock_irq(lock: &dev->power.lock);
1405
1406	cancel_work_sync(work: &dev->power.work);
1407
1408	spin_lock_irq(lock: &dev->power.lock);
1409	dev->power.request_pending = false;
1410	}
1411
1412	if (dev->power.runtime_status == RPM_SUSPENDING ||
1413	dev->power.runtime_status == RPM_RESUMING ||
1414	dev->power.idle_notification) {
1415	DEFINE_WAIT(wait);
1416
1417	/* Suspend, wake-up or idle notification in progress. */
1418	for (;;) {
1419	prepare_to_wait(wq_head: &dev->power.wait_queue, wq_entry: &wait,
1420	TASK_UNINTERRUPTIBLE);
1421	if (dev->power.runtime_status != RPM_SUSPENDING
1422	&& dev->power.runtime_status != RPM_RESUMING
1423	&& !dev->power.idle_notification)
1424	break;
1425	spin_unlock_irq(lock: &dev->power.lock);
1426
1427	schedule();
1428
1429	spin_lock_irq(lock: &dev->power.lock);
1430	}
1431	finish_wait(wq_head: &dev->power.wait_queue, wq_entry: &wait);
1432	}
1433	}
1434
1435	/**
1436	* pm_runtime_barrier - Flush pending requests and wait for completions.
1437	* @dev: Device to handle.
1438	*
1439	* Prevent the device from being suspended by incrementing its usage counter and
1440	* if there's a pending resume request for the device, wake the device up.
1441	* Next, make sure that all pending requests for the device have been flushed
1442	* from pm_wq and wait for all runtime PM operations involving the device in
1443	* progress to complete.
1444	*
1445	* Return value:
1446	* 1, if there was a resume request pending and the device had to be woken up,
1447	* 0, otherwise
1448	*/
1449	int pm_runtime_barrier(struct device *dev)
1450	{
1451	int retval = 0;
1452
1453	pm_runtime_get_noresume(dev);
1454	spin_lock_irq(lock: &dev->power.lock);
1455
1456	if (dev->power.request_pending
1457	&& dev->power.request == RPM_REQ_RESUME) {
1458	rpm_resume(dev, rpmflags: 0);
1459	retval = 1;
1460	}
1461
1462	__pm_runtime_barrier(dev);
1463
1464	spin_unlock_irq(lock: &dev->power.lock);
1465	pm_runtime_put_noidle(dev);
1466
1467	return retval;
1468	}
1469	EXPORT_SYMBOL_GPL(pm_runtime_barrier);
1470
1471	bool pm_runtime_block_if_disabled(struct device *dev)
1472	{
1473	bool ret;
1474
1475	spin_lock_irq(lock: &dev->power.lock);
1476
1477	ret = !pm_runtime_enabled(dev);
1478	if (ret && dev->power.last_status == RPM_INVALID)
1479	dev->power.last_status = RPM_BLOCKED;
1480
1481	spin_unlock_irq(lock: &dev->power.lock);
1482
1483	return ret;
1484	}
1485
1486	void pm_runtime_unblock(struct device *dev)
1487	{
1488	spin_lock_irq(lock: &dev->power.lock);
1489
1490	if (dev->power.last_status == RPM_BLOCKED)
1491	dev->power.last_status = RPM_INVALID;
1492
1493	spin_unlock_irq(lock: &dev->power.lock);
1494	}
1495
1496	void __pm_runtime_disable(struct device *dev, bool check_resume)
1497	{
1498	spin_lock_irq(lock: &dev->power.lock);
1499
1500	if (dev->power.disable_depth > 0) {
1501	dev->power.disable_depth++;
1502	goto out;
1503	}
1504
1505	/*
1506	* Wake up the device if there's a resume request pending, because that
1507	* means there probably is some I/O to process and disabling runtime PM
1508	* shouldn't prevent the device from processing the I/O.
1509	*/
1510	if (check_resume && dev->power.request_pending &&
1511	dev->power.request == RPM_REQ_RESUME) {
1512	/*
1513	* Prevent suspends and idle notifications from being carried
1514	* out after we have woken up the device.
1515	*/
1516	pm_runtime_get_noresume(dev);
1517
1518	rpm_resume(dev, rpmflags: 0);
1519
1520	pm_runtime_put_noidle(dev);
1521	}
1522
1523	/* Update time accounting before disabling PM-runtime. */
1524	update_pm_runtime_accounting(dev);
1525
1526	if (!dev->power.disable_depth++) {
1527	__pm_runtime_barrier(dev);
1528	dev->power.last_status = dev->power.runtime_status;
1529	}
1530
1531	out:
1532	spin_unlock_irq(lock: &dev->power.lock);
1533	}
1534	EXPORT_SYMBOL_GPL(__pm_runtime_disable);
1535
1536	/**
1537	* pm_runtime_enable - Enable runtime PM of a device.
1538	* @dev: Device to handle.
1539	*/
1540	void pm_runtime_enable(struct device *dev)
1541	{
1542	unsigned long flags;
1543
1544	spin_lock_irqsave(&dev->power.lock, flags);
1545
1546	if (!dev->power.disable_depth) {
1547	dev_warn(dev, "Unbalanced %s!\n", __func__);
1548	goto out;
1549	}
1550
1551	if (--dev->power.disable_depth > 0)
1552	goto out;
1553
1554	if (dev->power.last_status == RPM_BLOCKED) {
1555	dev_warn(dev, "Attempt to enable runtime PM when it is blocked\n");
1556	dump_stack();
1557	}
1558	dev->power.last_status = RPM_INVALID;
1559	dev->power.accounting_timestamp = ktime_get_mono_fast_ns();
1560
1561	if (dev->power.runtime_status == RPM_SUSPENDED &&
1562	!dev->power.ignore_children &&
1563	atomic_read(v: &dev->power.child_count) > 0)
1564	dev_warn(dev, "Enabling runtime PM for inactive device with active children\n");
1565
1566	out:
1567	spin_unlock_irqrestore(lock: &dev->power.lock, flags);
1568	}
1569	EXPORT_SYMBOL_GPL(pm_runtime_enable);
1570
1571	static void pm_runtime_set_suspended_action(void *data)
1572	{
1573	pm_runtime_set_suspended(dev: data);
1574	}
1575
1576	/**
1577	* devm_pm_runtime_set_active_enabled - set_active version of devm_pm_runtime_enable.
1578	*
1579	* @dev: Device to handle.
1580	*/
1581	int devm_pm_runtime_set_active_enabled(struct device *dev)
1582	{
1583	int err;
1584
1585	err = pm_runtime_set_active(dev);
1586	if (err)
1587	return err;
1588
1589	err = devm_add_action_or_reset(dev, pm_runtime_set_suspended_action, dev);
1590	if (err)
1591	return err;
1592
1593	return devm_pm_runtime_enable(dev);
1594	}
1595	EXPORT_SYMBOL_GPL(devm_pm_runtime_set_active_enabled);
1596
1597	static void pm_runtime_disable_action(void *data)
1598	{
1599	pm_runtime_dont_use_autosuspend(dev: data);
1600	pm_runtime_disable(dev: data);
1601	}
1602
1603	/**
1604	* devm_pm_runtime_enable - devres-enabled version of pm_runtime_enable.
1605	*
1606	* NOTE: this will also handle calling pm_runtime_dont_use_autosuspend() for
1607	* you at driver exit time if needed.
1608	*
1609	* @dev: Device to handle.
1610	*/
1611	int devm_pm_runtime_enable(struct device *dev)
1612	{
1613	pm_runtime_enable(dev);
1614
1615	return devm_add_action_or_reset(dev, pm_runtime_disable_action, dev);
1616	}
1617	EXPORT_SYMBOL_GPL(devm_pm_runtime_enable);
1618
1619	static void pm_runtime_put_noidle_action(void *data)
1620	{
1621	pm_runtime_put_noidle(dev: data);
1622	}
1623
1624	/**
1625	* devm_pm_runtime_get_noresume - devres-enabled version of pm_runtime_get_noresume.
1626	*
1627	* @dev: Device to handle.
1628	*/
1629	int devm_pm_runtime_get_noresume(struct device *dev)
1630	{
1631	pm_runtime_get_noresume(dev);
1632
1633	return devm_add_action_or_reset(dev, pm_runtime_put_noidle_action, dev);
1634	}
1635	EXPORT_SYMBOL_GPL(devm_pm_runtime_get_noresume);
1636
1637	/**
1638	* pm_runtime_forbid - Block runtime PM of a device.
1639	* @dev: Device to handle.
1640	*
1641	* Increase the device's usage count and clear its power.runtime_auto flag,
1642	* so that it cannot be suspended at run time until pm_runtime_allow() is called
1643	* for it.
1644	*/
1645	void pm_runtime_forbid(struct device *dev)
1646	{
1647	spin_lock_irq(lock: &dev->power.lock);
1648	if (!dev->power.runtime_auto)
1649	goto out;
1650
1651	dev->power.runtime_auto = false;
1652	atomic_inc(v: &dev->power.usage_count);
1653	rpm_resume(dev, rpmflags: 0);
1654
1655	out:
1656	spin_unlock_irq(lock: &dev->power.lock);
1657	}
1658	EXPORT_SYMBOL_GPL(pm_runtime_forbid);
1659
1660	/**
1661	* pm_runtime_allow - Unblock runtime PM of a device.
1662	* @dev: Device to handle.
1663	*
1664	* Decrease the device's usage count and set its power.runtime_auto flag.
1665	*/
1666	void pm_runtime_allow(struct device *dev)
1667	{
1668	int ret;
1669
1670	spin_lock_irq(lock: &dev->power.lock);
1671	if (dev->power.runtime_auto)
1672	goto out;
1673
1674	dev->power.runtime_auto = true;
1675	ret = rpm_drop_usage_count(dev);
1676	if (ret == 0)
1677	rpm_idle(dev, RPM_AUTO | RPM_ASYNC);
1678	else if (ret > 0)
1679	trace_rpm_usage(dev, RPM_AUTO | RPM_ASYNC);
1680
1681	out:
1682	spin_unlock_irq(lock: &dev->power.lock);
1683	}
1684	EXPORT_SYMBOL_GPL(pm_runtime_allow);
1685
1686	/**
1687	* pm_runtime_no_callbacks - Ignore runtime PM callbacks for a device.
1688	* @dev: Device to handle.
1689	*
1690	* Set the power.no_callbacks flag, which tells the PM core that this
1691	* device is power-managed through its parent and has no runtime PM
1692	* callbacks of its own. The runtime sysfs attributes will be removed.
1693	*/
1694	void pm_runtime_no_callbacks(struct device *dev)
1695	{
1696	spin_lock_irq(lock: &dev->power.lock);
1697	dev->power.no_callbacks = 1;
1698	spin_unlock_irq(lock: &dev->power.lock);
1699	if (device_is_registered(dev))
1700	rpm_sysfs_remove(dev);
1701	}
1702	EXPORT_SYMBOL_GPL(pm_runtime_no_callbacks);
1703
1704	/**
1705	* pm_runtime_irq_safe - Leave interrupts disabled during callbacks.
1706	* @dev: Device to handle
1707	*
1708	* Set the power.irq_safe flag, which tells the PM core that the
1709	* ->runtime_suspend() and ->runtime_resume() callbacks for this device should
1710	* always be invoked with the spinlock held and interrupts disabled. It also
1711	* causes the parent's usage counter to be permanently incremented, preventing
1712	* the parent from runtime suspending -- otherwise an irq-safe child might have
1713	* to wait for a non-irq-safe parent.
1714	*/
1715	void pm_runtime_irq_safe(struct device *dev)
1716	{
1717	if (dev->parent)
1718	pm_runtime_get_sync(dev: dev->parent);
1719
1720	spin_lock_irq(lock: &dev->power.lock);
1721	dev->power.irq_safe = 1;
1722	spin_unlock_irq(lock: &dev->power.lock);
1723	}
1724	EXPORT_SYMBOL_GPL(pm_runtime_irq_safe);
1725
1726	/**
1727	* update_autosuspend - Handle a change to a device's autosuspend settings.
1728	* @dev: Device to handle.
1729	* @old_delay: The former autosuspend_delay value.
1730	* @old_use: The former use_autosuspend value.
1731	*
1732	* Prevent runtime suspend if the new delay is negative and use_autosuspend is
1733	* set; otherwise allow it. Send an idle notification if suspends are allowed.
1734	*
1735	* This function must be called under dev->power.lock with interrupts disabled.
1736	*/
1737	static void update_autosuspend(struct device *dev, int old_delay, int old_use)
1738	{
1739	int delay = dev->power.autosuspend_delay;
1740
1741	/* Should runtime suspend be prevented now? */
1742	if (dev->power.use_autosuspend && delay < 0) {
1743
1744	/* If it used to be allowed then prevent it. */
1745	if (!old_use || old_delay >= 0) {
1746	atomic_inc(v: &dev->power.usage_count);
1747	rpm_resume(dev, rpmflags: 0);
1748	} else {
1749	trace_rpm_usage(dev, flags: 0);
1750	}
1751	}
1752
1753	/* Runtime suspend should be allowed now. */
1754	else {
1755
1756	/* If it used to be prevented then allow it. */
1757	if (old_use && old_delay < 0)
1758	atomic_dec(v: &dev->power.usage_count);
1759
1760	/* Maybe we can autosuspend now. */
1761	rpm_idle(dev, RPM_AUTO);
1762	}
1763	}
1764
1765	/**
1766	* pm_runtime_set_autosuspend_delay - Set a device's autosuspend_delay value.
1767	* @dev: Device to handle.
1768	* @delay: Value of the new delay in milliseconds.
1769	*
1770	* Set the device's power.autosuspend_delay value. If it changes to negative
1771	* and the power.use_autosuspend flag is set, prevent runtime suspends. If it
1772	* changes the other way, allow runtime suspends.
1773	*/
1774	void pm_runtime_set_autosuspend_delay(struct device *dev, int delay)
1775	{
1776	int old_delay, old_use;
1777
1778	spin_lock_irq(lock: &dev->power.lock);
1779	old_delay = dev->power.autosuspend_delay;
1780	old_use = dev->power.use_autosuspend;
1781	dev->power.autosuspend_delay = delay;
1782	update_autosuspend(dev, old_delay, old_use);
1783	spin_unlock_irq(lock: &dev->power.lock);
1784	}
1785	EXPORT_SYMBOL_GPL(pm_runtime_set_autosuspend_delay);
1786
1787	/**
1788	* __pm_runtime_use_autosuspend - Set a device's use_autosuspend flag.
1789	* @dev: Device to handle.
1790	* @use: New value for use_autosuspend.
1791	*
1792	* Set the device's power.use_autosuspend flag, and allow or prevent runtime
1793	* suspends as needed.
1794	*/
1795	void __pm_runtime_use_autosuspend(struct device *dev, bool use)
1796	{
1797	int old_delay, old_use;
1798
1799	spin_lock_irq(lock: &dev->power.lock);
1800	old_delay = dev->power.autosuspend_delay;
1801	old_use = dev->power.use_autosuspend;
1802	dev->power.use_autosuspend = use;
1803	update_autosuspend(dev, old_delay, old_use);
1804	spin_unlock_irq(lock: &dev->power.lock);
1805	}
1806	EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend);
1807
1808	/**
1809	* pm_runtime_init - Initialize runtime PM fields in given device object.
1810	* @dev: Device object to initialize.
1811	*/
1812	void pm_runtime_init(struct device *dev)
1813	{
1814	dev->power.runtime_status = RPM_SUSPENDED;
1815	dev->power.last_status = RPM_INVALID;
1816	dev->power.idle_notification = false;
1817
1818	dev->power.disable_depth = 1;
1819	atomic_set(v: &dev->power.usage_count, i: 0);
1820
1821	dev->power.runtime_error = 0;
1822
1823	atomic_set(v: &dev->power.child_count, i: 0);
1824	pm_suspend_ignore_children(dev, enable: false);
1825	dev->power.runtime_auto = true;
1826
1827	dev->power.request_pending = false;
1828	dev->power.request = RPM_REQ_NONE;
1829	dev->power.deferred_resume = false;
1830	dev->power.needs_force_resume = 0;
1831	INIT_WORK(&dev->power.work, pm_runtime_work);
1832
1833	dev->power.timer_expires = 0;
1834	hrtimer_setup(timer: &dev->power.suspend_timer, function: pm_suspend_timer_fn, CLOCK_MONOTONIC,
1835	mode: HRTIMER_MODE_ABS);
1836
1837	init_waitqueue_head(&dev->power.wait_queue);
1838	}
1839
1840	/**
1841	* pm_runtime_reinit - Re-initialize runtime PM fields in given device object.
1842	* @dev: Device object to re-initialize.
1843	*/
1844	void pm_runtime_reinit(struct device *dev)
1845	{
1846	if (!pm_runtime_enabled(dev)) {
1847	if (dev->power.runtime_status == RPM_ACTIVE)
1848	pm_runtime_set_suspended(dev);
1849	if (dev->power.irq_safe) {
1850	spin_lock_irq(lock: &dev->power.lock);
1851	dev->power.irq_safe = 0;
1852	spin_unlock_irq(lock: &dev->power.lock);
1853	if (dev->parent)
1854	pm_runtime_put(dev: dev->parent);
1855	}
1856	}
1857	}
1858
1859	/**
1860	* pm_runtime_remove - Prepare for removing a device from device hierarchy.
1861	* @dev: Device object being removed from device hierarchy.
1862	*/
1863	void pm_runtime_remove(struct device *dev)
1864	{
1865	__pm_runtime_disable(dev, false);
1866	pm_runtime_reinit(dev);
1867	}
1868
1869	/**
1870	* pm_runtime_get_suppliers - Resume and reference-count supplier devices.
1871	* @dev: Consumer device.
1872	*/
1873	void pm_runtime_get_suppliers(struct device *dev)
1874	{
1875	struct device_link *link;
1876	int idx;
1877
1878	idx = device_links_read_lock();
1879
1880	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
1881	device_links_read_lock_held())
1882	if (link->flags & DL_FLAG_PM_RUNTIME) {
1883	link->supplier_preactivated = true;
1884	pm_runtime_get_sync(dev: link->supplier);
1885	}
1886
1887	device_links_read_unlock(idx);
1888	}
1889
1890	/**
1891	* pm_runtime_put_suppliers - Drop references to supplier devices.
1892	* @dev: Consumer device.
1893	*/
1894	void pm_runtime_put_suppliers(struct device *dev)
1895	{
1896	struct device_link *link;
1897	int idx;
1898
1899	idx = device_links_read_lock();
1900
1901	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
1902	device_links_read_lock_held())
1903	if (link->supplier_preactivated) {
1904	link->supplier_preactivated = false;
1905	pm_runtime_put(dev: link->supplier);
1906	}
1907
1908	device_links_read_unlock(idx);
1909	}
1910
1911	void pm_runtime_new_link(struct device *dev)
1912	{
1913	spin_lock_irq(lock: &dev->power.lock);
1914	dev->power.links_count++;
1915	spin_unlock_irq(lock: &dev->power.lock);
1916	}
1917
1918	static void pm_runtime_drop_link_count(struct device *dev)
1919	{
1920	spin_lock_irq(lock: &dev->power.lock);
1921	WARN_ON(dev->power.links_count == 0);
1922	dev->power.links_count--;
1923	spin_unlock_irq(lock: &dev->power.lock);
1924	}
1925
1926	/**
1927	* pm_runtime_drop_link - Prepare for device link removal.
1928	* @link: Device link going away.
1929	*
1930	* Drop the link count of the consumer end of @link and decrement the supplier
1931	* device's runtime PM usage counter as many times as needed to drop all of the
1932	* PM runtime reference to it from the consumer.
1933	*/
1934	void pm_runtime_drop_link(struct device_link *link)
1935	{
1936	if (!(link->flags & DL_FLAG_PM_RUNTIME))
1937	return;
1938
1939	pm_runtime_drop_link_count(dev: link->consumer);
1940	pm_runtime_release_supplier(link);
1941	pm_request_idle(dev: link->supplier);
1942	}
1943
1944	bool pm_runtime_need_not_resume(struct device *dev)
1945	{
1946	return atomic_read(v: &dev->power.usage_count) <= 1 &&
1947	(atomic_read(v: &dev->power.child_count) == 0 ||
1948	dev->power.ignore_children);
1949	}
1950
1951	/**
1952	* pm_runtime_force_suspend - Force a device into suspend state if needed.
1953	* @dev: Device to suspend.
1954	*
1955	* Disable runtime PM so we safely can check the device's runtime PM status and
1956	* if it is active, invoke its ->runtime_suspend callback to suspend it and
1957	* change its runtime PM status field to RPM_SUSPENDED. Also, if the device's
1958	* usage and children counters don't indicate that the device was in use before
1959	* the system-wide transition under way, decrement its parent's children counter
1960	* (if there is a parent). Keep runtime PM disabled to preserve the state
1961	* unless we encounter errors.
1962	*
1963	* Typically this function may be invoked from a system suspend callback to make
1964	* sure the device is put into low power state and it should only be used during
1965	* system-wide PM transitions to sleep states. It assumes that the analogous
1966	* pm_runtime_force_resume() will be used to resume the device.
1967	*
1968	* Do not use with DPM_FLAG_SMART_SUSPEND as this can lead to an inconsistent
1969	* state where this function has called the ->runtime_suspend callback but the
1970	* PM core marks the driver as runtime active.
1971	*/
1972	int pm_runtime_force_suspend(struct device *dev)
1973	{
1974	int (*callback)(struct device *);
1975	int ret;
1976
1977	pm_runtime_disable(dev);
1978	if (pm_runtime_status_suspended(dev))
1979	return 0;
1980
1981	callback = RPM_GET_CALLBACK(dev, runtime_suspend);
1982
1983	dev_pm_enable_wake_irq_check(dev, can_change_status: true);
1984	ret = callback ? callback(dev) : 0;
1985	if (ret)
1986	goto err;
1987
1988	dev_pm_enable_wake_irq_complete(dev);
1989
1990	/*
1991	* If the device can stay in suspend after the system-wide transition
1992	* to the working state that will follow, drop the children counter of
1993	* its parent, but set its status to RPM_SUSPENDED anyway in case this
1994	* function will be called again for it in the meantime.
1995	*/
1996	if (pm_runtime_need_not_resume(dev)) {
1997	pm_runtime_set_suspended(dev);
1998	} else {
1999	__update_runtime_status(dev, status: RPM_SUSPENDED);
2000	dev->power.needs_force_resume = 1;
2001	}
2002
2003	return 0;
2004
2005	err:
2006	dev_pm_disable_wake_irq_check(dev, cond_disable: true);
2007	pm_runtime_enable(dev);
2008	return ret;
2009	}
2010	EXPORT_SYMBOL_GPL(pm_runtime_force_suspend);
2011
2012	/**
2013	* pm_runtime_force_resume - Force a device into resume state if needed.
2014	* @dev: Device to resume.
2015	*
2016	* Prior invoking this function we expect the user to have brought the device
2017	* into low power state by a call to pm_runtime_force_suspend(). Here we reverse
2018	* those actions and bring the device into full power, if it is expected to be
2019	* used on system resume. In the other case, we defer the resume to be managed
2020	* via runtime PM.
2021	*
2022	* Typically this function may be invoked from a system resume callback.
2023	*/
2024	int pm_runtime_force_resume(struct device *dev)
2025	{
2026	int (*callback)(struct device *);
2027	int ret = 0;
2028
2029	if (!dev->power.needs_force_resume)
2030	goto out;
2031
2032	/*
2033	* The value of the parent's children counter is correct already, so
2034	* just update the status of the device.
2035	*/
2036	__update_runtime_status(dev, status: RPM_ACTIVE);
2037
2038	callback = RPM_GET_CALLBACK(dev, runtime_resume);
2039
2040	dev_pm_disable_wake_irq_check(dev, cond_disable: false);
2041	ret = callback ? callback(dev) : 0;
2042	if (ret) {
2043	pm_runtime_set_suspended(dev);
2044	dev_pm_enable_wake_irq_check(dev, can_change_status: false);
2045	goto out;
2046	}
2047
2048	pm_runtime_mark_last_busy(dev);
2049	out:
2050	dev->power.needs_force_resume = 0;
2051	pm_runtime_enable(dev);
2052	return ret;
2053	}
2054	EXPORT_SYMBOL_GPL(pm_runtime_force_resume);
2055