drivers/base/power/main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * drivers/base/power/main.c - Where the driver meets power management.
   4  *
   5  * Copyright (c) 2003 Patrick Mochel
   6  * Copyright (c) 2003 Open Source Development Lab
   7  *
   8  * The driver model core calls device_pm_add() when a device is registered.
   9  * This will initialize the embedded device_pm_info object in the device
  10  * and add it to the list of power-controlled devices. sysfs entries for
  11  * controlling device power management will also be added.
  12  *
  13  * A separate list is used for keeping track of power info, because the power
  14  * domain dependencies may differ from the ancestral dependencies that the
  15  * subsystem list maintains.
  16  */
  17
  18 #define pr_fmt(fmt) "PM: " fmt
  19
  20 #include <linux/device.h>
  21 #include <linux/export.h>
  22 #include <linux/mutex.h>
  23 #include <linux/pm.h>
  24 #include <linux/pm_runtime.h>
  25 #include <linux/pm-trace.h>
  26 #include <linux/pm_wakeirq.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/sched.h>
  29 #include <linux/sched/debug.h>
  30 #include <linux/async.h>
  31 #include <linux/suspend.h>
  32 #include <trace/events/power.h>
  33 #include <linux/cpufreq.h>
  34 #include <linux/cpuidle.h>
  35 #include <linux/devfreq.h>
  36 #include <linux/timer.h>
  37
  38 #include "../base.h"
  39 #include "power.h"
  40
  41 typedef int (*pm_callback_t)(struct device *);
  42
  43 #define list_for_each_entry_rcu_locked(pos, head, member) \
  44         list_for_each_entry_rcu(pos, head, member, \
  45                         device_links_read_lock_held())
  46
  47 /*
  48  * The entries in the dpm_list list are in a depth first order, simply
  49  * because children are guaranteed to be discovered after parents, and
  50  * are inserted at the back of the list on discovery.
  51  *
  52  * Since device_pm_add() may be called with a device lock held,
  53  * we must never try to acquire a device lock while holding
  54  * dpm_list_mutex.
  55  */
  56
  57 LIST_HEAD(dpm_list);
  58 static LIST_HEAD(dpm_prepared_list);
  59 static LIST_HEAD(dpm_suspended_list);
  60 static LIST_HEAD(dpm_late_early_list);
  61 static LIST_HEAD(dpm_noirq_list);
  62
  63 struct suspend_stats suspend_stats;
  64 static DEFINE_MUTEX(dpm_list_mtx);
  65 static pm_message_t pm_transition;
  66
  67 static int async_error;
  68
  69 static const char *pm_verb(int event)
  70 {
  71         switch (event) {
  72         case PM_EVENT_SUSPEND:
  73                 return "suspend";
  74         case PM_EVENT_RESUME:
  75                 return "resume";
  76         case PM_EVENT_FREEZE:
  77                 return "freeze";
  78         case PM_EVENT_QUIESCE:
  79                 return "quiesce";
  80         case PM_EVENT_HIBERNATE:
  81                 return "hibernate";
  82         case PM_EVENT_THAW:
  83                 return "thaw";
  84         case PM_EVENT_RESTORE:
  85                 return "restore";
  86         case PM_EVENT_RECOVER:
  87                 return "recover";
  88         default:
  89                 return "(unknown PM event)";
  90         }
  91 }
  92
  93 /**
  94  * device_pm_sleep_init - Initialize system suspend-related device fields.
  95  * @dev: Device object being initialized.
  96  */
  97 void device_pm_sleep_init(struct device *dev)
  98 {
  99         dev->power.is_prepared = false;
 100         dev->power.is_suspended = false;
 101         dev->power.is_noirq_suspended = false;
 102         dev->power.is_late_suspended = false;
 103         init_completion(&dev->power.completion);
 104         complete_all(&dev->power.completion);
 105         dev->power.wakeup = NULL;
 106         INIT_LIST_HEAD(&dev->power.entry);
 107 }
 108
 109 /**
 110  * device_pm_lock - Lock the list of active devices used by the PM core.
 111  */
 112 void device_pm_lock(void)
 113 {
 114         mutex_lock(&dpm_list_mtx);
 115 }
 116
 117 /**
 118  * device_pm_unlock - Unlock the list of active devices used by the PM core.
 119  */
 120 void device_pm_unlock(void)
 121 {
 122         mutex_unlock(&dpm_list_mtx);
 123 }
 124
 125 /**
 126  * device_pm_add - Add a device to the PM core's list of active devices.
 127  * @dev: Device to add to the list.
 128  */
 129 void device_pm_add(struct device *dev)
 130 {
 131         /* Skip PM setup/initialization. */
 132         if (device_pm_not_required(dev))
 133                 return;
 134
 135         pr_debug("Adding info for %s:%s\n",
 136                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 137         device_pm_check_callbacks(dev);
 138         mutex_lock(&dpm_list_mtx);
 139         if (dev->parent && dev->parent->power.is_prepared)
 140                 dev_warn(dev, "parent %s should not be sleeping\n",
 141                         dev_name(dev->parent));
 142         list_add_tail(&dev->power.entry, &dpm_list);
 143         dev->power.in_dpm_list = true;
 144         mutex_unlock(&dpm_list_mtx);
 145 }
 146
 147 /**
 148  * device_pm_remove - Remove a device from the PM core's list of active devices.
 149  * @dev: Device to be removed from the list.
 150  */
 151 void device_pm_remove(struct device *dev)
 152 {
 153         if (device_pm_not_required(dev))
 154                 return;
 155
 156         pr_debug("Removing info for %s:%s\n",
 157                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 158         complete_all(&dev->power.completion);
 159         mutex_lock(&dpm_list_mtx);
 160         list_del_init(&dev->power.entry);
 161         dev->power.in_dpm_list = false;
 162         mutex_unlock(&dpm_list_mtx);
 163         device_wakeup_disable(dev);
 164         pm_runtime_remove(dev);
 165         device_pm_check_callbacks(dev);
 166 }
 167
 168 /**
 169  * device_pm_move_before - Move device in the PM core's list of active devices.
 170  * @deva: Device to move in dpm_list.
 171  * @devb: Device @deva should come before.
 172  */
 173 void device_pm_move_before(struct device *deva, struct device *devb)
 174 {
 175         pr_debug("Moving %s:%s before %s:%s\n",
 176                  deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
 177                  devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
 178         /* Delete deva from dpm_list and reinsert before devb. */
 179         list_move_tail(&deva->power.entry, &devb->power.entry);
 180 }
 181
 182 /**
 183  * device_pm_move_after - Move device in the PM core's list of active devices.
 184  * @deva: Device to move in dpm_list.
 185  * @devb: Device @deva should come after.
 186  */
 187 void device_pm_move_after(struct device *deva, struct device *devb)
 188 {
 189         pr_debug("Moving %s:%s after %s:%s\n",
 190                  deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
 191                  devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
 192         /* Delete deva from dpm_list and reinsert after devb. */
 193         list_move(&deva->power.entry, &devb->power.entry);
 194 }
 195
 196 /**
 197  * device_pm_move_last - Move device to end of the PM core's list of devices.
 198  * @dev: Device to move in dpm_list.
 199  */
 200 void device_pm_move_last(struct device *dev)
 201 {
 202         pr_debug("Moving %s:%s to end of list\n",
 203                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 204         list_move_tail(&dev->power.entry, &dpm_list);
 205 }
 206
 207 static ktime_t initcall_debug_start(struct device *dev, void *cb)
 208 {
 209         if (!pm_print_times_enabled)
 210                 return 0;
 211
 212         dev_info(dev, "calling %pS @ %i, parent: %s\n", cb,
 213                  task_pid_nr(current),
 214                  dev->parent ? dev_name(dev->parent) : "none");
 215         return ktime_get();
 216 }
 217
 218 static void initcall_debug_report(struct device *dev, ktime_t calltime,
 219                                   void *cb, int error)
 220 {
 221         ktime_t rettime;
 222         s64 nsecs;
 223
 224         if (!pm_print_times_enabled)
 225                 return;
 226
 227         rettime = ktime_get();
 228         nsecs = (s64) ktime_to_ns(ktime_sub(rettime, calltime));
 229
 230         dev_info(dev, "%pS returned %d after %Ld usecs\n", cb, error,
 231                  (unsigned long long)nsecs >> 10);
 232 }
 233
 234 /**
 235  * dpm_wait - Wait for a PM operation to complete.
 236  * @dev: Device to wait for.
 237  * @async: If unset, wait only if the device's power.async_suspend flag is set.
 238  */
 239 static void dpm_wait(struct device *dev, bool async)
 240 {
 241         if (!dev)
 242                 return;
 243
 244         if (async || (pm_async_enabled && dev->power.async_suspend))
 245                 wait_for_completion(&dev->power.completion);
 246 }
 247
 248 static int dpm_wait_fn(struct device *dev, void *async_ptr)
 249 {
 250         dpm_wait(dev, *((bool *)async_ptr));
 251         return 0;
 252 }
 253
 254 static void dpm_wait_for_children(struct device *dev, bool async)
 255 {
 256        device_for_each_child(dev, &async, dpm_wait_fn);
 257 }
 258
 259 static void dpm_wait_for_suppliers(struct device *dev, bool async)
 260 {
 261         struct device_link *link;
 262         int idx;
 263
 264         idx = device_links_read_lock();
 265
 266         /*
 267          * If the supplier goes away right after we've checked the link to it,
 268          * we'll wait for its completion to change the state, but that's fine,
 269          * because the only things that will block as a result are the SRCU
 270          * callbacks freeing the link objects for the links in the list we're
 271          * walking.
 272          */
 273         list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
 274                 if (READ_ONCE(link->status) != DL_STATE_DORMANT)
 275                         dpm_wait(link->supplier, async);
 276
 277         device_links_read_unlock(idx);
 278 }
 279
 280 static bool dpm_wait_for_superior(struct device *dev, bool async)
 281 {
 282         struct device *parent;
 283
 284         /*
 285          * If the device is resumed asynchronously and the parent's callback
 286          * deletes both the device and the parent itself, the parent object may
 287          * be freed while this function is running, so avoid that by reference
 288          * counting the parent once more unless the device has been deleted
 289          * already (in which case return right away).
 290          */
 291         mutex_lock(&dpm_list_mtx);
 292
 293         if (!device_pm_initialized(dev)) {
 294                 mutex_unlock(&dpm_list_mtx);
 295                 return false;
 296         }
 297
 298         parent = get_device(dev->parent);
 299
 300         mutex_unlock(&dpm_list_mtx);
 301
 302         dpm_wait(parent, async);
 303         put_device(parent);
 304
 305         dpm_wait_for_suppliers(dev, async);
 306
 307         /*
 308          * If the parent's callback has deleted the device, attempting to resume
 309          * it would be invalid, so avoid doing that then.
 310          */
 311         return device_pm_initialized(dev);
 312 }
 313
 314 static void dpm_wait_for_consumers(struct device *dev, bool async)
 315 {
 316         struct device_link *link;
 317         int idx;
 318
 319         idx = device_links_read_lock();
 320
 321         /*
 322          * The status of a device link can only be changed from "dormant" by a
 323          * probe, but that cannot happen during system suspend/resume.  In
 324          * theory it can change to "dormant" at that time, but then it is
 325          * reasonable to wait for the target device anyway (eg. if it goes
 326          * away, it's better to wait for it to go away completely and then
 327          * continue instead of trying to continue in parallel with its
 328          * unregistration).
 329          */
 330         list_for_each_entry_rcu_locked(link, &dev->links.consumers, s_node)
 331                 if (READ_ONCE(link->status) != DL_STATE_DORMANT)
 332                         dpm_wait(link->consumer, async);
 333
 334         device_links_read_unlock(idx);
 335 }
 336
 337 static void dpm_wait_for_subordinate(struct device *dev, bool async)
 338 {
 339         dpm_wait_for_children(dev, async);
 340         dpm_wait_for_consumers(dev, async);
 341 }
 342
 343 /**
 344  * pm_op - Return the PM operation appropriate for given PM event.
 345  * @ops: PM operations to choose from.
 346  * @state: PM transition of the system being carried out.
 347  */
 348 static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
 349 {
 350         switch (state.event) {
 351 #ifdef CONFIG_SUSPEND
 352         case PM_EVENT_SUSPEND:
 353                 return ops->suspend;
 354         case PM_EVENT_RESUME:
 355                 return ops->resume;
 356 #endif /* CONFIG_SUSPEND */
 357 #ifdef CONFIG_HIBERNATE_CALLBACKS
 358         case PM_EVENT_FREEZE:
 359         case PM_EVENT_QUIESCE:
 360                 return ops->freeze;
 361         case PM_EVENT_HIBERNATE:
 362                 return ops->poweroff;
 363         case PM_EVENT_THAW:
 364         case PM_EVENT_RECOVER:
 365                 return ops->thaw;
 366                 break;
 367         case PM_EVENT_RESTORE:
 368                 return ops->restore;
 369 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 370         }
 371
 372         return NULL;
 373 }
 374
 375 /**
 376  * pm_late_early_op - Return the PM operation appropriate for given PM event.
 377  * @ops: PM operations to choose from.
 378  * @state: PM transition of the system being carried out.
 379  *
 380  * Runtime PM is disabled for @dev while this function is being executed.
 381  */
 382 static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
 383                                       pm_message_t state)
 384 {
 385         switch (state.event) {
 386 #ifdef CONFIG_SUSPEND
 387         case PM_EVENT_SUSPEND:
 388                 return ops->suspend_late;
 389         case PM_EVENT_RESUME:
 390                 return ops->resume_early;
 391 #endif /* CONFIG_SUSPEND */
 392 #ifdef CONFIG_HIBERNATE_CALLBACKS
 393         case PM_EVENT_FREEZE:
 394         case PM_EVENT_QUIESCE:
 395                 return ops->freeze_late;
 396         case PM_EVENT_HIBERNATE:
 397                 return ops->poweroff_late;
 398         case PM_EVENT_THAW:
 399         case PM_EVENT_RECOVER:
 400                 return ops->thaw_early;
 401         case PM_EVENT_RESTORE:
 402                 return ops->restore_early;
 403 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 404         }
 405
 406         return NULL;
 407 }
 408
 409 /**
 410  * pm_noirq_op - Return the PM operation appropriate for given PM event.
 411  * @ops: PM operations to choose from.
 412  * @state: PM transition of the system being carried out.
 413  *
 414  * The driver of @dev will not receive interrupts while this function is being
 415  * executed.
 416  */
 417 static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
 418 {
 419         switch (state.event) {
 420 #ifdef CONFIG_SUSPEND
 421         case PM_EVENT_SUSPEND:
 422                 return ops->suspend_noirq;
 423         case PM_EVENT_RESUME:
 424                 return ops->resume_noirq;
 425 #endif /* CONFIG_SUSPEND */
 426 #ifdef CONFIG_HIBERNATE_CALLBACKS
 427         case PM_EVENT_FREEZE:
 428         case PM_EVENT_QUIESCE:
 429                 return ops->freeze_noirq;
 430         case PM_EVENT_HIBERNATE:
 431                 return ops->poweroff_noirq;
 432         case PM_EVENT_THAW:
 433         case PM_EVENT_RECOVER:
 434                 return ops->thaw_noirq;
 435         case PM_EVENT_RESTORE:
 436                 return ops->restore_noirq;
 437 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 438         }
 439
 440         return NULL;
 441 }
 442
 443 static void pm_dev_dbg(struct device *dev, pm_message_t state, const char *info)
 444 {
 445         dev_dbg(dev, "%s%s%s\n", info, pm_verb(state.event),
 446                 ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
 447                 ", may wakeup" : "");
 448 }
 449
 450 static void pm_dev_err(struct device *dev, pm_message_t state, const char *info,
 451                         int error)
 452 {
 453         pr_err("Device %s failed to %s%s: error %d\n",
 454                dev_name(dev), pm_verb(state.event), info, error);
 455 }
 456
 457 static void dpm_show_time(ktime_t starttime, pm_message_t state, int error,
 458                           const char *info)
 459 {
 460         ktime_t calltime;
 461         u64 usecs64;
 462         int usecs;
 463
 464         calltime = ktime_get();
 465         usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
 466         do_div(usecs64, NSEC_PER_USEC);
 467         usecs = usecs64;
 468         if (usecs == 0)
 469                 usecs = 1;
 470
 471         pm_pr_dbg("%s%s%s of devices %s after %ld.%03ld msecs\n",
 472                   info ?: "", info ? " " : "", pm_verb(state.event),
 473                   error ? "aborted" : "complete",
 474                   usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
 475 }
 476
 477 static int dpm_run_callback(pm_callback_t cb, struct device *dev,
 478                             pm_message_t state, const char *info)
 479 {
 480         ktime_t calltime;
 481         int error;
 482
 483         if (!cb)
 484                 return 0;
 485
 486         calltime = initcall_debug_start(dev, cb);
 487
 488         pm_dev_dbg(dev, state, info);
 489         trace_device_pm_callback_start(dev, info, state.event);
 490         error = cb(dev);
 491         trace_device_pm_callback_end(dev, error);
 492         suspend_report_result(cb, error);
 493
 494         initcall_debug_report(dev, calltime, cb, error);
 495
 496         return error;
 497 }
 498
 499 #ifdef CONFIG_DPM_WATCHDOG
 500 struct dpm_watchdog {
 501         struct device           *dev;
 502         struct task_struct      *tsk;
 503         struct timer_list       timer;
 504 };
 505
 506 #define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
 507         struct dpm_watchdog wd
 508
 509 /**
 510  * dpm_watchdog_handler - Driver suspend / resume watchdog handler.
 511  * @t: The timer that PM watchdog depends on.
 512  *
 513  * Called when a driver has timed out suspending or resuming.
 514  * There's not much we can do here to recover so panic() to
 515  * capture a crash-dump in pstore.
 516  */
 517 static void dpm_watchdog_handler(struct timer_list *t)
 518 {
 519         struct dpm_watchdog *wd = from_timer(wd, t, timer);
 520
 521         dev_emerg(wd->dev, "**** DPM device timeout ****\n");
 522         show_stack(wd->tsk, NULL);
 523         panic("%s %s: unrecoverable failure\n",
 524                 dev_driver_string(wd->dev), dev_name(wd->dev));
 525 }
 526
 527 /**
 528  * dpm_watchdog_set - Enable pm watchdog for given device.
 529  * @wd: Watchdog. Must be allocated on the stack.
 530  * @dev: Device to handle.
 531  */
 532 static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
 533 {
 534         struct timer_list *timer = &wd->timer;
 535
 536         wd->dev = dev;
 537         wd->tsk = current;
 538
 539         timer_setup_on_stack(timer, dpm_watchdog_handler, 0);
 540         /* use same timeout value for both suspend and resume */
 541         timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_TIMEOUT;
 542         add_timer(timer);
 543 }
 544
 545 /**
 546  * dpm_watchdog_clear - Disable suspend/resume watchdog.
 547  * @wd: Watchdog to disable.
 548  */
 549 static void dpm_watchdog_clear(struct dpm_watchdog *wd)
 550 {
 551         struct timer_list *timer = &wd->timer;
 552
 553         del_timer_sync(timer);
 554         destroy_timer_on_stack(timer);
 555 }
 556 #else
 557 #define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
 558 #define dpm_watchdog_set(x, y)
 559 #define dpm_watchdog_clear(x)
 560 #endif
 561
 562 /*------------------------- Resume routines -------------------------*/
 563
 564 /**
 565  * dev_pm_may_skip_resume - System-wide device resume optimization check.
 566  * @dev: Target device.
 567  *
 568  * Return:
 569  * - %false if the transition under way is RESTORE.
 570  * - The return value of dev_pm_smart_suspend_and_suspended() if the transition
 571  *   under way is THAW.
 572  * - The logical negation of %power.must_resume otherwise (that is, when the
 573  *   transition under way is RESUME).
 574  */
 575 bool dev_pm_may_skip_resume(struct device *dev)
 576 {
 577         if (pm_transition.event == PM_EVENT_RESTORE)
 578                 return false;
 579
 580         if (pm_transition.event == PM_EVENT_THAW)
 581                 return dev_pm_smart_suspend_and_suspended(dev);
 582
 583         return !dev->power.must_resume;
 584 }
 585
 586 /**
 587  * device_resume_noirq - Execute a "noirq resume" callback for given device.
 588  * @dev: Device to handle.
 589  * @state: PM transition of the system being carried out.
 590  * @async: If true, the device is being resumed asynchronously.
 591  *
 592  * The driver of @dev will not receive interrupts while this function is being
 593  * executed.
 594  */
 595 static int device_resume_noirq(struct device *dev, pm_message_t state, bool async)
 596 {
 597         pm_callback_t callback = NULL;
 598         const char *info = NULL;
 599         bool skip_resume;
 600         int error = 0;
 601
 602         TRACE_DEVICE(dev);
 603         TRACE_RESUME(0);
 604
 605         if (dev->power.syscore || dev->power.direct_complete)
 606                 goto Out;
 607
 608         if (!dev->power.is_noirq_suspended)
 609                 goto Out;
 610
 611         if (!dpm_wait_for_superior(dev, async))
 612                 goto Out;
 613
 614         skip_resume = dev_pm_may_skip_resume(dev);
 615         /*
 616          * If the driver callback is skipped below or by the middle layer
 617          * callback and device_resume_early() also skips the driver callback for
 618          * this device later, it needs to appear as "suspended" to PM-runtime,
 619          * so change its status accordingly.
 620          *
 621          * Otherwise, the device is going to be resumed, so set its PM-runtime
 622          * status to "active", but do that only if DPM_FLAG_SMART_SUSPEND is set
 623          * to avoid confusing drivers that don't use it.
 624          */
 625         if (skip_resume)
 626                 pm_runtime_set_suspended(dev);
 627         else if (dev_pm_smart_suspend_and_suspended(dev))
 628                 pm_runtime_set_active(dev);
 629
 630         if (dev->pm_domain) {
 631                 info = "noirq power domain ";
 632                 callback = pm_noirq_op(&dev->pm_domain->ops, state);
 633         } else if (dev->type && dev->type->pm) {
 634                 info = "noirq type ";
 635                 callback = pm_noirq_op(dev->type->pm, state);
 636         } else if (dev->class && dev->class->pm) {
 637                 info = "noirq class ";
 638                 callback = pm_noirq_op(dev->class->pm, state);
 639         } else if (dev->bus && dev->bus->pm) {
 640                 info = "noirq bus ";
 641                 callback = pm_noirq_op(dev->bus->pm, state);
 642         }
 643         if (callback)
 644                 goto Run;
 645
 646         if (skip_resume)
 647                 goto Skip;
 648
 649         if (dev->driver && dev->driver->pm) {
 650                 info = "noirq driver ";
 651                 callback = pm_noirq_op(dev->driver->pm, state);
 652         }
 653
 654 Run:
 655         error = dpm_run_callback(callback, dev, state, info);
 656
 657 Skip:
 658         dev->power.is_noirq_suspended = false;
 659
 660 Out:
 661         complete_all(&dev->power.completion);
 662         TRACE_RESUME(error);
 663         return error;
 664 }
 665
 666 static bool is_async(struct device *dev)
 667 {
 668         return dev->power.async_suspend && pm_async_enabled
 669                 && !pm_trace_is_enabled();
 670 }
 671
 672 static bool dpm_async_fn(struct device *dev, async_func_t func)
 673 {
 674         reinit_completion(&dev->power.completion);
 675
 676         if (is_async(dev)) {
 677                 get_device(dev);
 678                 async_schedule(func, dev);
 679                 return true;
 680         }
 681
 682         return false;
 683 }
 684
 685 static void async_resume_noirq(void *data, async_cookie_t cookie)
 686 {
 687         struct device *dev = (struct device *)data;
 688         int error;
 689
 690         error = device_resume_noirq(dev, pm_transition, true);
 691         if (error)
 692                 pm_dev_err(dev, pm_transition, " async", error);
 693
 694         put_device(dev);
 695 }
 696
 697 static void dpm_noirq_resume_devices(pm_message_t state)
 698 {
 699         struct device *dev;
 700         ktime_t starttime = ktime_get();
 701
 702         trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true);
 703         mutex_lock(&dpm_list_mtx);
 704         pm_transition = state;
 705
 706         /*
 707          * Advanced the async threads upfront,
 708          * in case the starting of async threads is
 709          * delayed by non-async resuming devices.
 710          */
 711         list_for_each_entry(dev, &dpm_noirq_list, power.entry)
 712                 dpm_async_fn(dev, async_resume_noirq);
 713
 714         while (!list_empty(&dpm_noirq_list)) {
 715                 dev = to_device(dpm_noirq_list.next);
 716                 get_device(dev);
 717                 list_move_tail(&dev->power.entry, &dpm_late_early_list);
 718                 mutex_unlock(&dpm_list_mtx);
 719
 720                 if (!is_async(dev)) {
 721                         int error;
 722
 723                         error = device_resume_noirq(dev, state, false);
 724                         if (error) {
 725                                 suspend_stats.failed_resume_noirq++;
 726                                 dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);
 727                                 dpm_save_failed_dev(dev_name(dev));
 728                                 pm_dev_err(dev, state, " noirq", error);
 729                         }
 730                 }
 731
 732                 mutex_lock(&dpm_list_mtx);
 733                 put_device(dev);
 734         }
 735         mutex_unlock(&dpm_list_mtx);
 736         async_synchronize_full();
 737         dpm_show_time(starttime, state, 0, "noirq");
 738         trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
 739 }
 740
 741 /**
 742  * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
 743  * @state: PM transition of the system being carried out.
 744  *
 745  * Invoke the "noirq" resume callbacks for all devices in dpm_noirq_list and
 746  * allow device drivers' interrupt handlers to be called.
 747  */
 748 void dpm_resume_noirq(pm_message_t state)
 749 {
 750         dpm_noirq_resume_devices(state);
 751
 752         resume_device_irqs();
 753         device_wakeup_disarm_wake_irqs();
 754
 755         cpuidle_resume();
 756 }
 757
 758 /**
 759  * device_resume_early - Execute an "early resume" callback for given device.
 760  * @dev: Device to handle.
 761  * @state: PM transition of the system being carried out.
 762  * @async: If true, the device is being resumed asynchronously.
 763  *
 764  * Runtime PM is disabled for @dev while this function is being executed.
 765  */
 766 static int device_resume_early(struct device *dev, pm_message_t state, bool async)
 767 {
 768         pm_callback_t callback = NULL;
 769         const char *info = NULL;
 770         int error = 0;
 771
 772         TRACE_DEVICE(dev);
 773         TRACE_RESUME(0);
 774
 775         if (dev->power.syscore || dev->power.direct_complete)
 776                 goto Out;
 777
 778         if (!dev->power.is_late_suspended)
 779                 goto Out;
 780
 781         if (!dpm_wait_for_superior(dev, async))
 782                 goto Out;
 783
 784         if (dev->pm_domain) {
 785                 info = "early power domain ";
 786                 callback = pm_late_early_op(&dev->pm_domain->ops, state);
 787         } else if (dev->type && dev->type->pm) {
 788                 info = "early type ";
 789                 callback = pm_late_early_op(dev->type->pm, state);
 790         } else if (dev->class && dev->class->pm) {
 791                 info = "early class ";
 792                 callback = pm_late_early_op(dev->class->pm, state);
 793         } else if (dev->bus && dev->bus->pm) {
 794                 info = "early bus ";
 795                 callback = pm_late_early_op(dev->bus->pm, state);
 796         }
 797         if (callback)
 798                 goto Run;
 799
 800         if (dev_pm_may_skip_resume(dev))
 801                 goto Skip;
 802
 803         if (dev->driver && dev->driver->pm) {
 804                 info = "early driver ";
 805                 callback = pm_late_early_op(dev->driver->pm, state);
 806         }
 807
 808 Run:
 809         error = dpm_run_callback(callback, dev, state, info);
 810
 811 Skip:
 812         dev->power.is_late_suspended = false;
 813
 814 Out:
 815         TRACE_RESUME(error);
 816
 817         pm_runtime_enable(dev);
 818         complete_all(&dev->power.completion);
 819         return error;
 820 }
 821
 822 static void async_resume_early(void *data, async_cookie_t cookie)
 823 {
 824         struct device *dev = (struct device *)data;
 825         int error;
 826
 827         error = device_resume_early(dev, pm_transition, true);
 828         if (error)
 829                 pm_dev_err(dev, pm_transition, " async", error);
 830
 831         put_device(dev);
 832 }
 833
 834 /**
 835  * dpm_resume_early - Execute "early resume" callbacks for all devices.
 836  * @state: PM transition of the system being carried out.
 837  */
 838 void dpm_resume_early(pm_message_t state)
 839 {
 840         struct device *dev;
 841         ktime_t starttime = ktime_get();
 842
 843         trace_suspend_resume(TPS("dpm_resume_early"), state.event, true);
 844         mutex_lock(&dpm_list_mtx);
 845         pm_transition = state;
 846
 847         /*
 848          * Advanced the async threads upfront,
 849          * in case the starting of async threads is
 850          * delayed by non-async resuming devices.
 851          */
 852         list_for_each_entry(dev, &dpm_late_early_list, power.entry)
 853                 dpm_async_fn(dev, async_resume_early);
 854
 855         while (!list_empty(&dpm_late_early_list)) {
 856                 dev = to_device(dpm_late_early_list.next);
 857                 get_device(dev);
 858                 list_move_tail(&dev->power.entry, &dpm_suspended_list);
 859                 mutex_unlock(&dpm_list_mtx);
 860
 861                 if (!is_async(dev)) {
 862                         int error;
 863
 864                         error = device_resume_early(dev, state, false);
 865                         if (error) {
 866                                 suspend_stats.failed_resume_early++;
 867                                 dpm_save_failed_step(SUSPEND_RESUME_EARLY);
 868                                 dpm_save_failed_dev(dev_name(dev));
 869                                 pm_dev_err(dev, state, " early", error);
 870                         }
 871                 }
 872                 mutex_lock(&dpm_list_mtx);
 873                 put_device(dev);
 874         }
 875         mutex_unlock(&dpm_list_mtx);
 876         async_synchronize_full();
 877         dpm_show_time(starttime, state, 0, "early");
 878         trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
 879 }
 880
 881 /**
 882  * dpm_resume_start - Execute "noirq" and "early" device callbacks.
 883  * @state: PM transition of the system being carried out.
 884  */
 885 void dpm_resume_start(pm_message_t state)
 886 {
 887         dpm_resume_noirq(state);
 888         dpm_resume_early(state);
 889 }
 890 EXPORT_SYMBOL_GPL(dpm_resume_start);
 891
 892 /**
 893  * device_resume - Execute "resume" callbacks for given device.
 894  * @dev: Device to handle.
 895  * @state: PM transition of the system being carried out.
 896  * @async: If true, the device is being resumed asynchronously.
 897  */
 898 static int device_resume(struct device *dev, pm_message_t state, bool async)
 899 {
 900         pm_callback_t callback = NULL;
 901         const char *info = NULL;
 902         int error = 0;
 903         DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 904
 905         TRACE_DEVICE(dev);
 906         TRACE_RESUME(0);
 907
 908         if (dev->power.syscore)
 909                 goto Complete;
 910
 911         if (dev->power.direct_complete) {
 912                 /* Match the pm_runtime_disable() in __device_suspend(). */
 913                 pm_runtime_enable(dev);
 914                 goto Complete;
 915         }
 916
 917         if (!dpm_wait_for_superior(dev, async))
 918                 goto Complete;
 919
 920         dpm_watchdog_set(&wd, dev);
 921         device_lock(dev);
 922
 923         /*
 924          * This is a fib.  But we'll allow new children to be added below
 925          * a resumed device, even if the device hasn't been completed yet.
 926          */
 927         dev->power.is_prepared = false;
 928
 929         if (!dev->power.is_suspended)
 930                 goto Unlock;
 931
 932         if (dev->pm_domain) {
 933                 info = "power domain ";
 934                 callback = pm_op(&dev->pm_domain->ops, state);
 935                 goto Driver;
 936         }
 937
 938         if (dev->type && dev->type->pm) {
 939                 info = "type ";
 940                 callback = pm_op(dev->type->pm, state);
 941                 goto Driver;
 942         }
 943
 944         if (dev->class && dev->class->pm) {
 945                 info = "class ";
 946                 callback = pm_op(dev->class->pm, state);
 947                 goto Driver;
 948         }
 949
 950         if (dev->bus) {
 951                 if (dev->bus->pm) {
 952                         info = "bus ";
 953                         callback = pm_op(dev->bus->pm, state);
 954                 } else if (dev->bus->resume) {
 955                         info = "legacy bus ";
 956                         callback = dev->bus->resume;
 957                         goto End;
 958                 }
 959         }
 960
 961  Driver:
 962         if (!callback && dev->driver && dev->driver->pm) {
 963                 info = "driver ";
 964                 callback = pm_op(dev->driver->pm, state);
 965         }
 966
 967  End:
 968         error = dpm_run_callback(callback, dev, state, info);
 969         dev->power.is_suspended = false;
 970
 971  Unlock:
 972         device_unlock(dev);
 973         dpm_watchdog_clear(&wd);
 974
 975  Complete:
 976         complete_all(&dev->power.completion);
 977
 978         TRACE_RESUME(error);
 979
 980         return error;
 981 }
 982
 983 static void async_resume(void *data, async_cookie_t cookie)
 984 {
 985         struct device *dev = (struct device *)data;
 986         int error;
 987
 988         error = device_resume(dev, pm_transition, true);
 989         if (error)
 990                 pm_dev_err(dev, pm_transition, " async", error);
 991         put_device(dev);
 992 }
 993
 994 /**
 995  * dpm_resume - Execute "resume" callbacks for non-sysdev devices.
 996  * @state: PM transition of the system being carried out.
 997  *
 998  * Execute the appropriate "resume" callback for all devices whose status
 999  * indicates that they are suspended.
1000  */
1001 void dpm_resume(pm_message_t state)
1002 {
1003         struct device *dev;
1004         ktime_t starttime = ktime_get();
1005
1006         trace_suspend_resume(TPS("dpm_resume"), state.event, true);
1007         might_sleep();
1008
1009         mutex_lock(&dpm_list_mtx);
1010         pm_transition = state;
1011         async_error = 0;
1012
1013         list_for_each_entry(dev, &dpm_suspended_list, power.entry)
1014                 dpm_async_fn(dev, async_resume);
1015
1016         while (!list_empty(&dpm_suspended_list)) {
1017                 dev = to_device(dpm_suspended_list.next);
1018                 get_device(dev);
1019                 if (!is_async(dev)) {
1020                         int error;
1021
1022                         mutex_unlock(&dpm_list_mtx);
1023
1024                         error = device_resume(dev, state, false);
1025                         if (error) {
1026                                 suspend_stats.failed_resume++;
1027                                 dpm_save_failed_step(SUSPEND_RESUME);
1028                                 dpm_save_failed_dev(dev_name(dev));
1029                                 pm_dev_err(dev, state, "", error);
1030                         }
1031
1032                         mutex_lock(&dpm_list_mtx);
1033                 }
1034                 if (!list_empty(&dev->power.entry))
1035                         list_move_tail(&dev->power.entry, &dpm_prepared_list);
1036                 put_device(dev);
1037         }
1038         mutex_unlock(&dpm_list_mtx);
1039         async_synchronize_full();
1040         dpm_show_time(starttime, state, 0, NULL);
1041
1042         cpufreq_resume();
1043         devfreq_resume();
1044         trace_suspend_resume(TPS("dpm_resume"), state.event, false);
1045 }
1046
1047 /**
1048  * device_complete - Complete a PM transition for given device.
1049  * @dev: Device to handle.
1050  * @state: PM transition of the system being carried out.
1051  */
1052 static void device_complete(struct device *dev, pm_message_t state)
1053 {
1054         void (*callback)(struct device *) = NULL;
1055         const char *info = NULL;
1056
1057         if (dev->power.syscore)
1058                 return;
1059
1060         device_lock(dev);
1061
1062         if (dev->pm_domain) {
1063                 info = "completing power domain ";
1064                 callback = dev->pm_domain->ops.complete;
1065         } else if (dev->type && dev->type->pm) {
1066                 info = "completing type ";
1067                 callback = dev->type->pm->complete;
1068         } else if (dev->class && dev->class->pm) {
1069                 info = "completing class ";
1070                 callback = dev->class->pm->complete;
1071         } else if (dev->bus && dev->bus->pm) {
1072                 info = "completing bus ";
1073                 callback = dev->bus->pm->complete;
1074         }
1075
1076         if (!callback && dev->driver && dev->driver->pm) {
1077                 info = "completing driver ";
1078                 callback = dev->driver->pm->complete;
1079         }
1080
1081         if (callback) {
1082                 pm_dev_dbg(dev, state, info);
1083                 callback(dev);
1084         }
1085
1086         device_unlock(dev);
1087
1088         pm_runtime_put(dev);
1089 }
1090
1091 /**
1092  * dpm_complete - Complete a PM transition for all non-sysdev devices.
1093  * @state: PM transition of the system being carried out.
1094  *
1095  * Execute the ->complete() callbacks for all devices whose PM status is not
1096  * DPM_ON (this allows new devices to be registered).
1097  */
1098 void dpm_complete(pm_message_t state)
1099 {
1100         struct list_head list;
1101
1102         trace_suspend_resume(TPS("dpm_complete"), state.event, true);
1103         might_sleep();
1104
1105         INIT_LIST_HEAD(&list);
1106         mutex_lock(&dpm_list_mtx);
1107         while (!list_empty(&dpm_prepared_list)) {
1108                 struct device *dev = to_device(dpm_prepared_list.prev);
1109
1110                 get_device(dev);
1111                 dev->power.is_prepared = false;
1112                 list_move(&dev->power.entry, &list);
1113                 mutex_unlock(&dpm_list_mtx);
1114
1115                 trace_device_pm_callback_start(dev, "", state.event);
1116                 device_complete(dev, state);
1117                 trace_device_pm_callback_end(dev, 0);
1118
1119                 mutex_lock(&dpm_list_mtx);
1120                 put_device(dev);
1121         }
1122         list_splice(&list, &dpm_list);
1123         mutex_unlock(&dpm_list_mtx);
1124
1125         /* Allow device probing and trigger re-probing of deferred devices */
1126         device_unblock_probing();
1127         trace_suspend_resume(TPS("dpm_complete"), state.event, false);
1128 }
1129
1130 /**
1131  * dpm_resume_end - Execute "resume" callbacks and complete system transition.
1132  * @state: PM transition of the system being carried out.
1133  *
1134  * Execute "resume" callbacks for all devices and complete the PM transition of
1135  * the system.
1136  */
1137 void dpm_resume_end(pm_message_t state)
1138 {
1139         dpm_resume(state);
1140         dpm_complete(state);
1141 }
1142 EXPORT_SYMBOL_GPL(dpm_resume_end);
1143
1144
1145 /*------------------------- Suspend routines -------------------------*/
1146
1147 /**
1148  * resume_event - Return a "resume" message for given "suspend" sleep state.
1149  * @sleep_state: PM message representing a sleep state.
1150  *
1151  * Return a PM message representing the resume event corresponding to given
1152  * sleep state.
1153  */
1154 static pm_message_t resume_event(pm_message_t sleep_state)
1155 {
1156         switch (sleep_state.event) {
1157         case PM_EVENT_SUSPEND:
1158                 return PMSG_RESUME;
1159         case PM_EVENT_FREEZE:
1160         case PM_EVENT_QUIESCE:
1161                 return PMSG_RECOVER;
1162         case PM_EVENT_HIBERNATE:
1163                 return PMSG_RESTORE;
1164         }
1165         return PMSG_ON;
1166 }
1167
1168 static void dpm_superior_set_must_resume(struct device *dev)
1169 {
1170         struct device_link *link;
1171         int idx;
1172
1173         if (dev->parent)
1174                 dev->parent->power.must_resume = true;
1175
1176         idx = device_links_read_lock();
1177
1178         list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
1179                 link->supplier->power.must_resume = true;
1180
1181         device_links_read_unlock(idx);
1182 }
1183
1184 /**
1185  * __device_suspend_noirq - Execute a "noirq suspend" callback for given device.
1186  * @dev: Device to handle.
1187  * @state: PM transition of the system being carried out.
1188  * @async: If true, the device is being suspended asynchronously.
1189  *
1190  * The driver of @dev will not receive interrupts while this function is being
1191  * executed.
1192  */
1193 static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
1194 {
1195         pm_callback_t callback = NULL;
1196         const char *info = NULL;
1197         int error = 0;
1198
1199         TRACE_DEVICE(dev);
1200         TRACE_SUSPEND(0);
1201
1202         dpm_wait_for_subordinate(dev, async);
1203
1204         if (async_error)
1205                 goto Complete;
1206
1207         if (dev->power.syscore || dev->power.direct_complete)
1208                 goto Complete;
1209
1210         if (dev->pm_domain) {
1211                 info = "noirq power domain ";
1212                 callback = pm_noirq_op(&dev->pm_domain->ops, state);
1213         } else if (dev->type && dev->type->pm) {
1214                 info = "noirq type ";
1215                 callback = pm_noirq_op(dev->type->pm, state);
1216         } else if (dev->class && dev->class->pm) {
1217                 info = "noirq class ";
1218                 callback = pm_noirq_op(dev->class->pm, state);
1219         } else if (dev->bus && dev->bus->pm) {
1220                 info = "noirq bus ";
1221                 callback = pm_noirq_op(dev->bus->pm, state);
1222         }
1223         if (callback)
1224                 goto Run;
1225
1226         if (dev_pm_smart_suspend_and_suspended(dev))
1227                 goto Skip;
1228
1229         if (dev->driver && dev->driver->pm) {
1230                 info = "noirq driver ";
1231                 callback = pm_noirq_op(dev->driver->pm, state);
1232         }
1233
1234 Run:
1235         error = dpm_run_callback(callback, dev, state, info);
1236         if (error) {
1237                 async_error = error;
1238                 goto Complete;
1239         }
1240
1241 Skip:
1242         dev->power.is_noirq_suspended = true;
1243
1244         /*
1245          * Skipping the resume of devices that were in use right before the
1246          * system suspend (as indicated by their PM-runtime usage counters)
1247          * would be suboptimal.  Also resume them if doing that is not allowed
1248          * to be skipped.
1249          */
1250         if (atomic_read(&dev->power.usage_count) > 1 ||
1251             !(dev_pm_test_driver_flags(dev, DPM_FLAG_LEAVE_SUSPENDED) &&
1252               dev->power.may_skip_resume))
1253                 dev->power.must_resume = true;
1254
1255         if (dev->power.must_resume)
1256                 dpm_superior_set_must_resume(dev);
1257
1258 Complete:
1259         complete_all(&dev->power.completion);
1260         TRACE_SUSPEND(error);
1261         return error;
1262 }
1263
1264 static void async_suspend_noirq(void *data, async_cookie_t cookie)
1265 {
1266         struct device *dev = (struct device *)data;
1267         int error;
1268
1269         error = __device_suspend_noirq(dev, pm_transition, true);
1270         if (error) {
1271                 dpm_save_failed_dev(dev_name(dev));
1272                 pm_dev_err(dev, pm_transition, " async", error);
1273         }
1274
1275         put_device(dev);
1276 }
1277
1278 static int device_suspend_noirq(struct device *dev)
1279 {
1280         if (dpm_async_fn(dev, async_suspend_noirq))
1281                 return 0;
1282
1283         return __device_suspend_noirq(dev, pm_transition, false);
1284 }
1285
1286 static int dpm_noirq_suspend_devices(pm_message_t state)
1287 {
1288         ktime_t starttime = ktime_get();
1289         int error = 0;
1290
1291         trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);
1292         mutex_lock(&dpm_list_mtx);
1293         pm_transition = state;
1294         async_error = 0;
1295
1296         while (!list_empty(&dpm_late_early_list)) {
1297                 struct device *dev = to_device(dpm_late_early_list.prev);
1298
1299                 get_device(dev);
1300                 mutex_unlock(&dpm_list_mtx);
1301
1302                 error = device_suspend_noirq(dev);
1303
1304                 mutex_lock(&dpm_list_mtx);
1305                 if (error) {
1306                         pm_dev_err(dev, state, " noirq", error);
1307                         dpm_save_failed_dev(dev_name(dev));
1308                         put_device(dev);
1309                         break;
1310                 }
1311                 if (!list_empty(&dev->power.entry))
1312                         list_move(&dev->power.entry, &dpm_noirq_list);
1313                 put_device(dev);
1314
1315                 if (async_error)
1316                         break;
1317         }
1318         mutex_unlock(&dpm_list_mtx);
1319         async_synchronize_full();
1320         if (!error)
1321                 error = async_error;
1322
1323         if (error) {
1324                 suspend_stats.failed_suspend_noirq++;
1325                 dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
1326         }
1327         dpm_show_time(starttime, state, error, "noirq");
1328         trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
1329         return error;
1330 }
1331
1332 /**
1333  * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
1334  * @state: PM transition of the system being carried out.
1335  *
1336  * Prevent device drivers' interrupt handlers from being called and invoke
1337  * "noirq" suspend callbacks for all non-sysdev devices.
1338  */
1339 int dpm_suspend_noirq(pm_message_t state)
1340 {
1341         int ret;
1342
1343         cpuidle_pause();
1344
1345         device_wakeup_arm_wake_irqs();
1346         suspend_device_irqs();
1347
1348         ret = dpm_noirq_suspend_devices(state);
1349         if (ret)
1350                 dpm_resume_noirq(resume_event(state));
1351
1352         return ret;
1353 }
1354
1355 static void dpm_propagate_wakeup_to_parent(struct device *dev)
1356 {
1357         struct device *parent = dev->parent;
1358
1359         if (!parent)
1360                 return;
1361
1362         spin_lock_irq(&parent->power.lock);
1363
1364         if (dev->power.wakeup_path && !parent->power.ignore_children)
1365                 parent->power.wakeup_path = true;
1366
1367         spin_unlock_irq(&parent->power.lock);
1368 }
1369
1370 /**
1371  * __device_suspend_late - Execute a "late suspend" callback for given device.
1372  * @dev: Device to handle.
1373  * @state: PM transition of the system being carried out.
1374  * @async: If true, the device is being suspended asynchronously.
1375  *
1376  * Runtime PM is disabled for @dev while this function is being executed.
1377  */
1378 static int __device_suspend_late(struct device *dev, pm_message_t state, bool async)
1379 {
1380         pm_callback_t callback = NULL;
1381         const char *info = NULL;
1382         int error = 0;
1383
1384         TRACE_DEVICE(dev);
1385         TRACE_SUSPEND(0);
1386
1387         __pm_runtime_disable(dev, false);
1388
1389         dpm_wait_for_subordinate(dev, async);
1390
1391         if (async_error)
1392                 goto Complete;
1393
1394         if (pm_wakeup_pending()) {
1395                 async_error = -EBUSY;
1396                 goto Complete;
1397         }
1398
1399         if (dev->power.syscore || dev->power.direct_complete)
1400                 goto Complete;
1401
1402         if (dev->pm_domain) {
1403                 info = "late power domain ";
1404                 callback = pm_late_early_op(&dev->pm_domain->ops, state);
1405         } else if (dev->type && dev->type->pm) {
1406                 info = "late type ";
1407                 callback = pm_late_early_op(dev->type->pm, state);
1408         } else if (dev->class && dev->class->pm) {
1409                 info = "late class ";
1410                 callback = pm_late_early_op(dev->class->pm, state);
1411         } else if (dev->bus && dev->bus->pm) {
1412                 info = "late bus ";
1413                 callback = pm_late_early_op(dev->bus->pm, state);
1414         }
1415         if (callback)
1416                 goto Run;
1417
1418         if (dev_pm_smart_suspend_and_suspended(dev)) {
1419                 /*
1420                  * In principle, the resume of the device may be skippend if it
1421                  * remains in runtime suspend at this point.
1422                  */
1423                 dev->power.may_skip_resume = true;
1424                 goto Skip;
1425         }
1426
1427         if (dev->driver && dev->driver->pm) {
1428                 info = "late driver ";
1429                 callback = pm_late_early_op(dev->driver->pm, state);
1430         }
1431
1432 Run:
1433         error = dpm_run_callback(callback, dev, state, info);
1434         if (error) {
1435                 async_error = error;
1436                 goto Complete;
1437         }
1438         dpm_propagate_wakeup_to_parent(dev);
1439
1440 Skip:
1441         dev->power.is_late_suspended = true;
1442
1443 Complete:
1444         TRACE_SUSPEND(error);
1445         complete_all(&dev->power.completion);
1446         return error;
1447 }
1448
1449 static void async_suspend_late(void *data, async_cookie_t cookie)
1450 {
1451         struct device *dev = (struct device *)data;
1452         int error;
1453
1454         error = __device_suspend_late(dev, pm_transition, true);
1455         if (error) {
1456                 dpm_save_failed_dev(dev_name(dev));
1457                 pm_dev_err(dev, pm_transition, " async", error);
1458         }
1459         put_device(dev);
1460 }
1461
1462 static int device_suspend_late(struct device *dev)
1463 {
1464         if (dpm_async_fn(dev, async_suspend_late))
1465                 return 0;
1466
1467         return __device_suspend_late(dev, pm_transition, false);
1468 }
1469
1470 /**
1471  * dpm_suspend_late - Execute "late suspend" callbacks for all devices.
1472  * @state: PM transition of the system being carried out.
1473  */
1474 int dpm_suspend_late(pm_message_t state)
1475 {
1476         ktime_t starttime = ktime_get();
1477         int error = 0;
1478
1479         trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true);
1480         mutex_lock(&dpm_list_mtx);
1481         pm_transition = state;
1482         async_error = 0;
1483
1484         while (!list_empty(&dpm_suspended_list)) {
1485                 struct device *dev = to_device(dpm_suspended_list.prev);
1486
1487                 get_device(dev);
1488                 mutex_unlock(&dpm_list_mtx);
1489
1490                 error = device_suspend_late(dev);
1491
1492                 mutex_lock(&dpm_list_mtx);
1493                 if (!list_empty(&dev->power.entry))
1494                         list_move(&dev->power.entry, &dpm_late_early_list);
1495
1496                 if (error) {
1497                         pm_dev_err(dev, state, " late", error);
1498                         dpm_save_failed_dev(dev_name(dev));
1499                         put_device(dev);
1500                         break;
1501                 }
1502                 put_device(dev);
1503
1504                 if (async_error)
1505                         break;
1506         }
1507         mutex_unlock(&dpm_list_mtx);
1508         async_synchronize_full();
1509         if (!error)
1510                 error = async_error;
1511         if (error) {
1512                 suspend_stats.failed_suspend_late++;
1513                 dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
1514                 dpm_resume_early(resume_event(state));
1515         }
1516         dpm_show_time(starttime, state, error, "late");
1517         trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
1518         return error;
1519 }
1520
1521 /**
1522  * dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
1523  * @state: PM transition of the system being carried out.
1524  */
1525 int dpm_suspend_end(pm_message_t state)
1526 {
1527         ktime_t starttime = ktime_get();
1528         int error;
1529
1530         error = dpm_suspend_late(state);
1531         if (error)
1532                 goto out;
1533
1534         error = dpm_suspend_noirq(state);
1535         if (error)
1536                 dpm_resume_early(resume_event(state));
1537
1538 out:
1539         dpm_show_time(starttime, state, error, "end");
1540         return error;
1541 }
1542 EXPORT_SYMBOL_GPL(dpm_suspend_end);
1543
1544 /**
1545  * legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
1546  * @dev: Device to suspend.
1547  * @state: PM transition of the system being carried out.
1548  * @cb: Suspend callback to execute.
1549  * @info: string description of caller.
1550  */
1551 static int legacy_suspend(struct device *dev, pm_message_t state,
1552                           int (*cb)(struct device *dev, pm_message_t state),
1553                           const char *info)
1554 {
1555         int error;
1556         ktime_t calltime;
1557
1558         calltime = initcall_debug_start(dev, cb);
1559
1560         trace_device_pm_callback_start(dev, info, state.event);
1561         error = cb(dev, state);
1562         trace_device_pm_callback_end(dev, error);
1563         suspend_report_result(cb, error);
1564
1565         initcall_debug_report(dev, calltime, cb, error);
1566
1567         return error;
1568 }
1569
1570 static void dpm_clear_superiors_direct_complete(struct device *dev)
1571 {
1572         struct device_link *link;
1573         int idx;
1574
1575         if (dev->parent) {
1576                 spin_lock_irq(&dev->parent->power.lock);
1577                 dev->parent->power.direct_complete = false;
1578                 spin_unlock_irq(&dev->parent->power.lock);
1579         }
1580
1581         idx = device_links_read_lock();
1582
1583         list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node) {
1584                 spin_lock_irq(&link->supplier->power.lock);
1585                 link->supplier->power.direct_complete = false;
1586                 spin_unlock_irq(&link->supplier->power.lock);
1587         }
1588
1589         device_links_read_unlock(idx);
1590 }
1591
1592 /**
1593  * __device_suspend - Execute "suspend" callbacks for given device.
1594  * @dev: Device to handle.
1595  * @state: PM transition of the system being carried out.
1596  * @async: If true, the device is being suspended asynchronously.
1597  */
1598 static int __device_suspend(struct device *dev, pm_message_t state, bool async)
1599 {
1600         pm_callback_t callback = NULL;
1601         const char *info = NULL;
1602         int error = 0;
1603         DECLARE_DPM_WATCHDOG_ON_STACK(wd);
1604
1605         TRACE_DEVICE(dev);
1606         TRACE_SUSPEND(0);
1607
1608         dpm_wait_for_subordinate(dev, async);
1609
1610         if (async_error) {
1611                 dev->power.direct_complete = false;
1612                 goto Complete;
1613         }
1614
1615         /*
1616          * If a device configured to wake up the system from sleep states
1617          * has been suspended at run time and there's a resume request pending
1618          * for it, this is equivalent to the device signaling wakeup, so the
1619          * system suspend operation should be aborted.
1620          */
1621         if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
1622                 pm_wakeup_event(dev, 0);
1623
1624         if (pm_wakeup_pending()) {
1625                 dev->power.direct_complete = false;
1626                 async_error = -EBUSY;
1627                 goto Complete;
1628         }
1629
1630         if (dev->power.syscore)
1631                 goto Complete;
1632
1633         /* Avoid direct_complete to let wakeup_path propagate. */
1634         if (device_may_wakeup(dev) || dev->power.wakeup_path)
1635                 dev->power.direct_complete = false;
1636
1637         if (dev->power.direct_complete) {
1638                 if (pm_runtime_status_suspended(dev)) {
1639                         pm_runtime_disable(dev);
1640                         if (pm_runtime_status_suspended(dev)) {
1641                                 pm_dev_dbg(dev, state, "direct-complete ");
1642                                 goto Complete;
1643                         }
1644
1645                         pm_runtime_enable(dev);
1646                 }
1647                 dev->power.direct_complete = false;
1648         }
1649
1650         dev->power.may_skip_resume = false;
1651         dev->power.must_resume = false;
1652
1653         dpm_watchdog_set(&wd, dev);
1654         device_lock(dev);
1655
1656         if (dev->pm_domain) {
1657                 info = "power domain ";
1658                 callback = pm_op(&dev->pm_domain->ops, state);
1659                 goto Run;
1660         }
1661
1662         if (dev->type && dev->type->pm) {
1663                 info = "type ";
1664                 callback = pm_op(dev->type->pm, state);
1665                 goto Run;
1666         }
1667
1668         if (dev->class && dev->class->pm) {
1669                 info = "class ";
1670                 callback = pm_op(dev->class->pm, state);
1671                 goto Run;
1672         }
1673
1674         if (dev->bus) {
1675                 if (dev->bus->pm) {
1676                         info = "bus ";
1677                         callback = pm_op(dev->bus->pm, state);
1678                 } else if (dev->bus->suspend) {
1679                         pm_dev_dbg(dev, state, "legacy bus ");
1680                         error = legacy_suspend(dev, state, dev->bus->suspend,
1681                                                 "legacy bus ");
1682                         goto End;
1683                 }
1684         }
1685
1686  Run:
1687         if (!callback && dev->driver && dev->driver->pm) {
1688                 info = "driver ";
1689                 callback = pm_op(dev->driver->pm, state);
1690         }
1691
1692         error = dpm_run_callback(callback, dev, state, info);
1693
1694  End:
1695         if (!error) {
1696                 dev->power.is_suspended = true;
1697                 if (device_may_wakeup(dev))
1698                         dev->power.wakeup_path = true;
1699
1700                 dpm_propagate_wakeup_to_parent(dev);
1701                 dpm_clear_superiors_direct_complete(dev);
1702         }
1703
1704         device_unlock(dev);
1705         dpm_watchdog_clear(&wd);
1706
1707  Complete:
1708         if (error)
1709                 async_error = error;
1710
1711         complete_all(&dev->power.completion);
1712         TRACE_SUSPEND(error);
1713         return error;
1714 }
1715
1716 static void async_suspend(void *data, async_cookie_t cookie)
1717 {
1718         struct device *dev = (struct device *)data;
1719         int error;
1720
1721         error = __device_suspend(dev, pm_transition, true);
1722         if (error) {
1723                 dpm_save_failed_dev(dev_name(dev));
1724                 pm_dev_err(dev, pm_transition, " async", error);
1725         }
1726
1727         put_device(dev);
1728 }
1729
1730 static int device_suspend(struct device *dev)
1731 {
1732         if (dpm_async_fn(dev, async_suspend))
1733                 return 0;
1734
1735         return __device_suspend(dev, pm_transition, false);
1736 }
1737
1738 /**
1739  * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
1740  * @state: PM transition of the system being carried out.
1741  */
1742 int dpm_suspend(pm_message_t state)
1743 {
1744         ktime_t starttime = ktime_get();
1745         int error = 0;
1746
1747         trace_suspend_resume(TPS("dpm_suspend"), state.event, true);
1748         might_sleep();
1749
1750         devfreq_suspend();
1751         cpufreq_suspend();
1752
1753         mutex_lock(&dpm_list_mtx);
1754         pm_transition = state;
1755         async_error = 0;
1756         while (!list_empty(&dpm_prepared_list)) {
1757                 struct device *dev = to_device(dpm_prepared_list.prev);
1758
1759                 get_device(dev);
1760                 mutex_unlock(&dpm_list_mtx);
1761
1762                 error = device_suspend(dev);
1763
1764                 mutex_lock(&dpm_list_mtx);
1765                 if (error) {
1766                         pm_dev_err(dev, state, "", error);
1767                         dpm_save_failed_dev(dev_name(dev));
1768                         put_device(dev);
1769                         break;
1770                 }
1771                 if (!list_empty(&dev->power.entry))
1772                         list_move(&dev->power.entry, &dpm_suspended_list);
1773                 put_device(dev);
1774                 if (async_error)
1775                         break;
1776         }
1777         mutex_unlock(&dpm_list_mtx);
1778         async_synchronize_full();
1779         if (!error)
1780                 error = async_error;
1781         if (error) {
1782                 suspend_stats.failed_suspend++;
1783                 dpm_save_failed_step(SUSPEND_SUSPEND);
1784         }
1785         dpm_show_time(starttime, state, error, NULL);
1786         trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
1787         return error;
1788 }
1789
1790 /**
1791  * device_prepare - Prepare a device for system power transition.
1792  * @dev: Device to handle.
1793  * @state: PM transition of the system being carried out.
1794  *
1795  * Execute the ->prepare() callback(s) for given device.  No new children of the
1796  * device may be registered after this function has returned.
1797  */
1798 static int device_prepare(struct device *dev, pm_message_t state)
1799 {
1800         int (*callback)(struct device *) = NULL;
1801         int ret = 0;
1802
1803         if (dev->power.syscore)
1804                 return 0;
1805
1806         /*
1807          * If a device's parent goes into runtime suspend at the wrong time,
1808          * it won't be possible to resume the device.  To prevent this we
1809          * block runtime suspend here, during the prepare phase, and allow
1810          * it again during the complete phase.
1811          */
1812         pm_runtime_get_noresume(dev);
1813
1814         device_lock(dev);
1815
1816         dev->power.wakeup_path = false;
1817
1818         if (dev->power.no_pm_callbacks)
1819                 goto unlock;
1820
1821         if (dev->pm_domain)
1822                 callback = dev->pm_domain->ops.prepare;
1823         else if (dev->type && dev->type->pm)
1824                 callback = dev->type->pm->prepare;
1825         else if (dev->class && dev->class->pm)
1826                 callback = dev->class->pm->prepare;
1827         else if (dev->bus && dev->bus->pm)
1828                 callback = dev->bus->pm->prepare;
1829
1830         if (!callback && dev->driver && dev->driver->pm)
1831                 callback = dev->driver->pm->prepare;
1832
1833         if (callback)
1834                 ret = callback(dev);
1835
1836 unlock:
1837         device_unlock(dev);
1838
1839         if (ret < 0) {
1840                 suspend_report_result(callback, ret);
1841                 pm_runtime_put(dev);
1842                 return ret;
1843         }
1844         /*
1845          * A positive return value from ->prepare() means "this device appears
1846          * to be runtime-suspended and its state is fine, so if it really is
1847          * runtime-suspended, you can leave it in that state provided that you
1848          * will do the same thing with all of its descendants".  This only
1849          * applies to suspend transitions, however.
1850          */
1851         spin_lock_irq(&dev->power.lock);
1852         dev->power.direct_complete = state.event == PM_EVENT_SUSPEND &&
1853                 (ret > 0 || dev->power.no_pm_callbacks) &&
1854                 !dev_pm_test_driver_flags(dev, DPM_FLAG_NEVER_SKIP);
1855         spin_unlock_irq(&dev->power.lock);
1856         return 0;
1857 }
1858
1859 /**
1860  * dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
1861  * @state: PM transition of the system being carried out.
1862  *
1863  * Execute the ->prepare() callback(s) for all devices.
1864  */
1865 int dpm_prepare(pm_message_t state)
1866 {
1867         int error = 0;
1868
1869         trace_suspend_resume(TPS("dpm_prepare"), state.event, true);
1870         might_sleep();
1871
1872         /*
1873          * Give a chance for the known devices to complete their probes, before
1874          * disable probing of devices. This sync point is important at least
1875          * at boot time + hibernation restore.
1876          */
1877         wait_for_device_probe();
1878         /*
1879          * It is unsafe if probing of devices will happen during suspend or
1880          * hibernation and system behavior will be unpredictable in this case.
1881          * So, let's prohibit device's probing here and defer their probes
1882          * instead. The normal behavior will be restored in dpm_complete().
1883          */
1884         device_block_probing();
1885
1886         mutex_lock(&dpm_list_mtx);
1887         while (!list_empty(&dpm_list)) {
1888                 struct device *dev = to_device(dpm_list.next);
1889
1890                 get_device(dev);
1891                 mutex_unlock(&dpm_list_mtx);
1892
1893                 trace_device_pm_callback_start(dev, "", state.event);
1894                 error = device_prepare(dev, state);
1895                 trace_device_pm_callback_end(dev, error);
1896
1897                 mutex_lock(&dpm_list_mtx);
1898                 if (error) {
1899                         if (error == -EAGAIN) {
1900                                 put_device(dev);
1901                                 error = 0;
1902                                 continue;
1903                         }
1904                         pr_info("Device %s not prepared for power transition: code %d\n",
1905                                 dev_name(dev), error);
1906                         put_device(dev);
1907                         break;
1908                 }
1909                 dev->power.is_prepared = true;
1910                 if (!list_empty(&dev->power.entry))
1911                         list_move_tail(&dev->power.entry, &dpm_prepared_list);
1912                 put_device(dev);
1913         }
1914         mutex_unlock(&dpm_list_mtx);
1915         trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
1916         return error;
1917 }
1918
1919 /**
1920  * dpm_suspend_start - Prepare devices for PM transition and suspend them.
1921  * @state: PM transition of the system being carried out.
1922  *
1923  * Prepare all non-sysdev devices for system PM transition and execute "suspend"
1924  * callbacks for them.
1925  */
1926 int dpm_suspend_start(pm_message_t state)
1927 {
1928         ktime_t starttime = ktime_get();
1929         int error;
1930
1931         error = dpm_prepare(state);
1932         if (error) {
1933                 suspend_stats.failed_prepare++;
1934                 dpm_save_failed_step(SUSPEND_PREPARE);
1935         } else
1936                 error = dpm_suspend(state);
1937         dpm_show_time(starttime, state, error, "start");
1938         return error;
1939 }
1940 EXPORT_SYMBOL_GPL(dpm_suspend_start);
1941
1942 void __suspend_report_result(const char *function, void *fn, int ret)
1943 {
1944         if (ret)
1945                 pr_err("%s(): %pS returns %d\n", function, fn, ret);
1946 }
1947 EXPORT_SYMBOL_GPL(__suspend_report_result);
1948
1949 /**
1950  * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
1951  * @subordinate: Device that needs to wait for @dev.
1952  * @dev: Device to wait for.
1953  */
1954 int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
1955 {
1956         dpm_wait(dev, subordinate->power.async_suspend);
1957         return async_error;
1958 }
1959 EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);
1960
1961 /**
1962  * dpm_for_each_dev - device iterator.
1963  * @data: data for the callback.
1964  * @fn: function to be called for each device.
1965  *
1966  * Iterate over devices in dpm_list, and call @fn for each device,
1967  * passing it @data.
1968  */
1969 void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
1970 {
1971         struct device *dev;
1972
1973         if (!fn)
1974                 return;
1975
1976         device_pm_lock();
1977         list_for_each_entry(dev, &dpm_list, power.entry)
1978                 fn(dev, data);
1979         device_pm_unlock();
1980 }
1981 EXPORT_SYMBOL_GPL(dpm_for_each_dev);
1982
1983 static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
1984 {
1985         if (!ops)
1986                 return true;
1987
1988         return !ops->prepare &&
1989                !ops->suspend &&
1990                !ops->suspend_late &&
1991                !ops->suspend_noirq &&
1992                !ops->resume_noirq &&
1993                !ops->resume_early &&
1994                !ops->resume &&
1995                !ops->complete;
1996 }
1997
1998 void device_pm_check_callbacks(struct device *dev)
1999 {
2000         spin_lock_irq(&dev->power.lock);
2001         dev->power.no_pm_callbacks =
2002                 (!dev->bus || (pm_ops_is_empty(dev->bus->pm) &&
2003                  !dev->bus->suspend && !dev->bus->resume)) &&
2004                 (!dev->class || pm_ops_is_empty(dev->class->pm)) &&
2005                 (!dev->type || pm_ops_is_empty(dev->type->pm)) &&
2006                 (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) &&
2007                 (!dev->driver || (pm_ops_is_empty(dev->driver->pm) &&
2008                  !dev->driver->suspend && !dev->driver->resume));
2009         spin_unlock_irq(&dev->power.lock);
2010 }
2011
2012 bool dev_pm_smart_suspend_and_suspended(struct device *dev)
2013 {
2014         return dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) &&
2015                 pm_runtime_status_suspended(dev);
2016 }